diff --git a/AGENTS.md b/AGENTS.md index f95bf9bbc..ac047121c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,7 +71,7 @@ submitqueue/ # repo root (Go module github.com/uber/submi └── doc/ # Documentation ``` -The `platform/` tree holds code reused across domains (infrastructure, shared entities, shared extension contracts). A multi-service **domain** (e.g. `submitqueue/`) keeps the same internal layout (`gateway/`, `orchestrator/`, `entity/`, `extension/`, `core/`); a domain's own `core/` (e.g. `submitqueue/core/`) holds infra shared only between that domain's services. A **single-service domain** collapses that split — the domain *is* the service, so its controllers live directly under the domain root (e.g. `runway/controller/`, `stovepipe/controller/`) with no `gateway/`/`orchestrator/` segment, and its wire contract is service-segment-free (`api/{domain}/`). `runway` is a consumer-only landing service with no gateway. `stovepipe` exposes ingestion RPC behavior and runs its own process, build, build-signal, record, hook, and DLQ queue stages. +The `platform/` tree holds code reused across domains (infrastructure, shared entities, shared extension contracts). A multi-service **domain** (e.g. `submitqueue/`) keeps the same internal layout (`gateway/`, `orchestrator/`, `entity/`, `extension/`, `core/`); a domain's own `core/` (e.g. `submitqueue/core/`) holds infra shared only between that domain's services. A **single-service domain** collapses that split — the domain *is* the service, so its controllers live directly under the domain root (e.g. `runway/controller/`, `stovepipe/controller/`) with no `gateway/`/`orchestrator/` segment, and its wire contract is service-segment-free (`api/{domain}/`). `runway` is a consumer-only merge execution service with no gateway. `stovepipe` exposes ingestion RPC behavior and runs its own process, build, build-signal, record, hook, and DLQ queue stages. The `api/` tree holds **published** wire contracts — those depended on from outside the owning domain. RPC contracts live at `api/{domain}/{service}/` (`proto/` for `.proto` sources, `protopb/` for committed generated Go); for a single-service domain the service segment is dropped, so the contract lives directly at `api/{domain}/` (e.g. `api/runway/{proto,protopb}/`). A service package may hold multiple `.proto` files, all generating into the same `protopb/`. External message-queue contracts live at `api/{domain}/messagequeue/` (see Message Queue Contracts below). Internal queue contracts do **not** go here — they live under `{domain}/core/messagequeue/`. @@ -113,7 +113,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er Controllers receive `consumer.Delivery` (a subset interface without Ack/Nack) to enforce separation of business logic from queue mechanics. `delivery.Hold(delayMs)` requests delayed redelivery without consuming retry budget; the controller must then return `nil`. -**Queue payloads: IDs within a boundary, full payloads across one.** When producer and consumer share a store (for example SubmitQueue's `build`→`buildsignal` flow), put only the entity **ID** on the queue and reload from storage (the store is the source of truth, messages stay small, redelivery is idempotent). Reloading from storage is what makes the publish ordering load-bearing — see "Persist before you publish" above. When a queue **crosses a service boundary** (for example SubmitQueue validation or merge handing work to Runway), publish the **full payload** the consumer needs, and have the **client own the correlation ID** so it can match the asynchronous result back to the work it is tracking. The queue's **owner defines the wire contract and topic keys** (in its own domain package); the other side imports them. +**Queue payloads: IDs within a boundary, full payloads across one.** When producer and consumer share a store (same service — e.g. `build`→`buildsignal`, `validate`→`landconflict`), put only the entity **ID** on the queue and reload from storage (the store is the source of truth, messages stay small, redelivery is idempotent). Reloading from storage is what makes the publish ordering load-bearing — see "Persist before you publish" above. When a queue **crosses a service boundary** (the consumer cannot read the producer's store — e.g. orchestrator→runway), publish the **full payload** the consumer needs, and have the **client own the correlation ID** so it can match the async result back to the work it is tracking. The queue's **owner defines the wire contract and topic keys** (in its own domain package); the other side imports them. ### Entities @@ -198,7 +198,7 @@ To add a new `.proto` to a service, drop it in the service's `api/{domain}/{serv New queue contracts are defined in **proto3** (`.proto` under `proto/`, generated Go in `protopb/` as the binding) and serialized as **protobuf JSON** (protojson) so the queue keeps storing self-describing JSON. Location follows audience: external/cross-domain contracts go under `api/{domain}/messagequeue/`; internal contracts (used only within the owning domain) go under `{domain}/core/messagequeue/`. Bazel `visibility` enforces the split — internal targets are domain-scoped, `api/` targets are public. -For proto-backed contracts, the message types are generated and the contract package adds generic `protojson` glue — `Marshal(m)` / `Unmarshal[T](b, m)` — owning the wire conventions: `UseProtoNames` (snake_case fields), UPPER_SNAKE enum values, int64-as-string, and unknown fields discarded on read (additive evolution). The topic key(s) carrying a message are declared on the message via the `topic_keys` proto option — a `google.protobuf.MessageOptions` extension defined in `api/base/messagequeue`. A topic key is a stable logical name, not a concrete wire topic; each implementer maps it to its backend's topic name, and a `TopicKeys(msg)` reflection helper reads the option back. It is contract metadata, not the hot path — publish/consume still routes on `consumer.TopicKey` + `TopicRegistry`. The contract package owns both halves: the proto payload and the `TopicKey` constants for its topic keys. A contract test round-trips the payloads and asserts every topic key is bound to exactly one message. Shared field types (`Change`, `Strategy`) are shared protos under `api/base/{change,mergestrategy}`. `api/runway/messagequeue/` and `stovepipe/core/messagequeue/` are current examples. +The message types are generated; the contract package adds only generic `protojson` glue — `Marshal(m)` / `Unmarshal[T](b, m)` — owning the wire conventions: `UseProtoNames` (snake_case fields), UPPER_SNAKE enum values, int64-as-string, unknown fields discarded on read (additive evolution). The topic key(s) carrying a message are declared on the message via the `topic_keys` proto option — a `google.protobuf.MessageOptions` extension defined in `api/base/messagequeue`. A topic key is a stable logical name, not a concrete wire topic; each implementer maps it to its backend's topic name, and a `TopicKeys(msg)` reflection helper reads the option back. It is contract metadata, not the hot path — publish/consume still routes on `consumer.TopicKey` + `TopicRegistry`. The contract package owns both halves: the proto payload and the `TopicKey` constants for its topic keys. A contract test round-trips the payloads and asserts every topic key is bound to exactly one message. Shared field types (`Change`, `Strategy`) are shared protos under `api/base/{change,mergestrategy}`. `api/runway/messagequeue/` is the reference example. SubmitQueue's internal pipeline predates the proto-backed convention. It continues to serialize domain entities with `encoding/json` and declares its logical keys in `submitqueue/core/topickey/`. Do not convert or mix these wire formats incidentally; treat migration as an explicit compatibility change. @@ -376,6 +376,6 @@ Errors are classified by origin (user vs infra) and retryability. The framework **Key rules:** 1. **Non-retryable by default** — a plain `fmt.Errorf(...)` is non-retryable. Retryability is opted into explicitly, but that decision is almost always made by a classifier, not a controller (see rule 4). 2. **Infra by default** — any error not wrapped with `NewUserError` is infra. There is no `NewInfraError`. -3. **Extensions return plain errors** — extension interfaces (`MergeChecker`, `Storage`, `Publisher`) return standard `error` values with their own domain sentinels (e.g. `storage.ErrNotFound`). They do NOT classify errors as user or infra. +3. **Extensions return plain errors** — extension interfaces (`ChangeProvider`, `Storage`, `Publisher`) return standard `error` values with their own domain sentinels (e.g. `storage.ErrNotFound`). They do NOT classify errors as user or infra. 4. **Classifiers do the bulk of classification; controllers override only with knowledge a classifier lacks** — primary pipeline consumers compose per-backend classifiers into `errs.NewClassifierProcessor(...)`; the processor runs once per chain in the consumer and decides retryability from the raw error. So the common case is a controller returning the raw error (`fmt.Errorf("...: %w", err)`) and letting the classifier verdict stand. Reserve an explicit `errs.New*Error` wrap for the rare case where the controller knows something the classifier cannot infer from the error value alone (e.g. `storage.ErrNotFound` meaning "user asked for a missing resource" *in this call site*). Do **not** wrap a failure as retryable just because replaying it is convenient (e.g. a failed queue publish) — that turns permanent failures into infinite retries instead of dead-lettering. DLQ reconciliation consumers use `errs.AlwaysRetryableProcessor` instead. See [platform/errs/README.md](platform/errs/README.md). 5. **Error chain works end-to-end** — extensions wrap custom errors, controllers wrap with `errs.New*Error`, and `errors.Is`/`errors.As` walks the full chain. diff --git a/Makefile b/Makefile index 25e5097ed..d137f3067 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ export REPO_ROOT := $(shell pwd) # path, so adding a provider is mostly adding a directory — see # service/submitqueue/demo/provider/README.md. # -# fake a change is a URI; nothing merges anywhere. Needs nothing. +# fake a change is a URI; nothing lands anywhere. Needs nothing. # git branches in a bare repository on disk; real fetch, cherry-pick, push. # github real pull requests. Needs a repository and GITHUB_TOKEN. PROVIDER ?= fake @@ -60,7 +60,7 @@ PROVIDER_COMPOSE_FILE_git = service/submitqueue/docker-compose.git.yml PROVIDER_COMPOSE_FILE_github = service/submitqueue/docker-compose.provider.yml PROVIDER_COMPOSE_FILE = $(PROVIDER_COMPOSE_FILE_$(PROVIDER)) -# Where PROVIDER=git keeps the bare repository it merges into. Outside the +# Where PROVIDER=git keeps the bare repository it lands into. Outside the # repository, so a demo leaves nothing in a checkout, and bind-mounted rather # than kept in a volume so `git log` on the host can show what landed. # @@ -266,7 +266,7 @@ deps: tidy-go ## Download and tidy Go dependencies e2e-git-test: ## Run the hermetic git E2E (real merger against a bare repo; no credentials) @echo "Running hermetic git end-to-end tests..." @$(BAZEL) test //test/e2e/submitqueue:go_default_test --test_output=errors \ - --test_filter='TestGitMergeE2E' + --test_filter='TestGitLandE2E' e2e-test: ## Run end-to-end tests (hermetic; Bazel builds all inputs; runs in parallel) @echo "Running end-to-end tests (parallel)..." @@ -519,7 +519,7 @@ local-submitqueue-start: build-all-linux ## Start full stack (PROVIDER=fake|git| @echo "" @echo "Gateway gRPC port: $$(docker port $(SUBMITQUEUE_LOCAL_PROJECT)-gateway-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'unknown')" @if [ "$(PROVIDER)" = "git" ]; then \ - echo "Merge target: $(SQ_GIT_SANDBOX_DIR)/sandbox.git"; \ + echo "Land target: $(SQ_GIT_SANDBOX_DIR)/sandbox.git"; \ fi @echo "" @echo "Generate traffic with:" @@ -579,7 +579,7 @@ local-stovepipe-stop: ## Stop the Stovepipe service mocks: ## Generate mock files using mockgen @echo "Generating mocks..." - @$(BAZEL) run @rules_go//go -- generate ./submitqueue/extension/storage/... ./submitqueue/extension/buildrunner/... ./submitqueue/extension/changeprovider/... ./platform/extension/counter/... ./platform/extension/consumergate/... ./platform/extension/hook/... ./platform/extension/messagequeue/... ./submitqueue/extension/queueconfig/... ./submitqueue/extension/mergechecker/... ./submitqueue/extension/conflict/... ./submitqueue/extension/speculation/... ./submitqueue/extension/validator/... ./platform/consumer/... ./stovepipe/core/requestlog/... ./stovepipe/extension/storage/... ./stovepipe/extension/sourcecontrol/... + @$(BAZEL) run @rules_go//go -- generate ./submitqueue/extension/storage/... ./submitqueue/extension/buildrunner/... ./submitqueue/extension/changeprovider/... ./platform/extension/counter/... ./platform/extension/consumergate/... ./platform/extension/hook/... ./platform/extension/messagequeue/... ./submitqueue/extension/queueconfig/... ./runway/extension/merger/... ./submitqueue/extension/conflict/... ./submitqueue/extension/speculation/... ./submitqueue/extension/validator/... ./platform/consumer/... ./stovepipe/core/requestlog/... ./stovepipe/extension/storage/... ./stovepipe/extension/sourcecontrol/... @echo "Mocks generated successfully!" proto: ## Generate protobuf files from .proto definitions diff --git a/README.md b/README.md index 6c7fb86a7..130407284 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE) [![Slack](https://img.shields.io/badge/Slack-join%20the%20community-4A154B?logo=slack&logoColor=white)](https://join.slack.com/t/submitqueue/shared_invite/zt-46gkqj682-7zcQphxm2pYqkjDo9lbmYA) -SubmitQueue is a high-performance speculative merge queue that keeps your trunk consistently green at scale. Rather than validating changes one at a time, SubmitQueue speculatively rebases and validates multiple changes in parallel against predicted future states of HEAD. When validations pass, changes land automatically. When they fail, SubmitQueue isolates the offending change and retries the rest — all without human intervention. +SubmitQueue is a high-performance speculative submission queue that keeps your trunk consistently green at scale. Rather than validating changes one at a time, SubmitQueue speculatively rebases and validates multiple changes in parallel against predicted future states of HEAD. When validations pass, changes land automatically. When they fail, SubmitQueue isolates the offending change and retries the rest — all without human intervention. Designed for large monorepos and fast-moving teams where concurrent changes can introduce subtle conflicts and destabilize builds. diff --git a/api/base/hook/protopb/hook.pb.go b/api/base/hook/protopb/hook.pb.go index cc0046c49..d84f9b0fc 100644 --- a/api/base/hook/protopb/hook.pb.go +++ b/api/base/hook/protopb/hook.pb.go @@ -40,19 +40,18 @@ const ( // HookEvent is one fire-and-forget lifecycle event. Every domain publishes this // same shape to its own hook topic, so a sink that consumes several domains -// reads one schema rather than one per producer. See api/base/hook/README.md. +// reads one schema rather than one per producer. type HookEvent struct { state protoimpl.MessageState `protogen:"open.v1"` // id is the opaque identity of this occurrence, derived from the transition // it describes so that replaying the transition mints the same id. It is // the queue's dedupe key and a hook's idempotency key, and is never parsed. Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` - // source is the domain that produced the event: "submitqueue", - // "stovepipe", ... An open string rather than an enum so a new producer - // does not break existing consumers. + // source is the domain that produced the event. An open string rather than + // an enum so a new producer does not break existing consumers. Source string `protobuf:"bytes,2,opt,name=source,proto3" json:"source,omitempty"` // type is what happened, as one dotted open string: "request.landed", - // "batch.failed", ... It is the only dimension a consumer filters on, and + // "batch.failed", etc. It is the only dimension a consumer filters on, and // open for the same reason as source. Type string `protobuf:"bytes,3,opt,name=type,proto3" json:"type,omitempty"` // timestamp_ms is when the occurrence happened, in milliseconds since the diff --git a/api/base/messagequeue/proto/messagequeue.proto b/api/base/messagequeue/proto/messagequeue.proto index 3dbabb705..bd18807d1 100644 --- a/api/base/messagequeue/proto/messagequeue.proto +++ b/api/base/messagequeue/proto/messagequeue.proto @@ -28,7 +28,7 @@ option java_package = "com.uber.submitqueue.base.messagequeue"; // in any domain — annotates itself with stable logical topic key(s), making the // key-to-payload binding part of the language-neutral proto contract rather than // out-of-band Go wiring. A single payload may list several keys (one shape can -// serve a queue pair, e.g. a dry-run check and a committing merge). Domains +// serve a queue pair, e.g. a dry-run check and a committing operation). Domains // import this rather than redefining their own. extend google.protobuf.MessageOptions { // topic_keys are the stable logical topic keys that carry this message — not diff --git a/api/submitqueue/gateway/proto/gateway.proto b/api/submitqueue/gateway/proto/gateway.proto index 272c7d013..e9a8dec4a 100644 --- a/api/submitqueue/gateway/proto/gateway.proto +++ b/api/submitqueue/gateway/proto/gateway.proto @@ -46,7 +46,7 @@ message PingResponse { string hostname = 4; } -// LandRequest defines a request to land (merge into target branch of the source control repository) a set of code changes. +// LandRequest defines a request to land a set of code changes on the source control repository's target branch. // // SubmitQueue guarantees changes are landed in order with no other changes in between. // SubmitQueue does not guarantee each change is individually valid, but produces a validity marker on such changes. @@ -261,7 +261,7 @@ service SubmitQueueGateway { // state transition is performed in the background by the orchestrator and may not have completed by the time the // caller receives a response. // - // Cancellation is NOT GUARANTEED: a request that has already merged, or that races to completion before the cancel + // Cancellation is NOT GUARANTEED: a request that has already landed, or that races to completion before the cancel // signal propagates through the pipeline, may still land (or end in an error). Callers must NOT assume that a // successful Cancel response means the request was cancelled — the actual terminal outcome (cancelled, landed, or // error) must be checked through the request-summary or request-history APIs. diff --git a/api/submitqueue/gateway/protopb/gateway.pb.go b/api/submitqueue/gateway/protopb/gateway.pb.go index 301a53930..135455a89 100644 --- a/api/submitqueue/gateway/protopb/gateway.pb.go +++ b/api/submitqueue/gateway/protopb/gateway.pb.go @@ -157,7 +157,7 @@ func (x *PingResponse) GetHostname() string { return "" } -// LandRequest defines a request to land (merge into target branch of the source control repository) a set of code changes. +// LandRequest defines a request to land a set of code changes on the source control repository's target branch. // // SubmitQueue guarantees changes are landed in order with no other changes in between. // SubmitQueue does not guarantee each change is individually valid, but produces a validity marker on such changes. diff --git a/api/submitqueue/gateway/protopb/gateway_grpc.pb.go b/api/submitqueue/gateway/protopb/gateway_grpc.pb.go index ac9a4a2fa..79f70f8e2 100644 --- a/api/submitqueue/gateway/protopb/gateway_grpc.pb.go +++ b/api/submitqueue/gateway/protopb/gateway_grpc.pb.go @@ -62,7 +62,7 @@ type SubmitQueueGatewayClient interface { // state transition is performed in the background by the orchestrator and may not have completed by the time the // caller receives a response. // - // Cancellation is NOT GUARANTEED: a request that has already merged, or that races to completion before the cancel + // Cancellation is NOT GUARANTEED: a request that has already landed, or that races to completion before the cancel // signal propagates through the pipeline, may still land (or end in an error). Callers must NOT assume that a // successful Cancel response means the request was cancelled — the actual terminal outcome (cancelled, landed, or // error) must be checked through the request-summary or request-history APIs. @@ -185,7 +185,7 @@ type SubmitQueueGatewayServer interface { // state transition is performed in the background by the orchestrator and may not have completed by the time the // caller receives a response. // - // Cancellation is NOT GUARANTEED: a request that has already merged, or that races to completion before the cancel + // Cancellation is NOT GUARANTEED: a request that has already landed, or that races to completion before the cancel // signal propagates through the pipeline, may still land (or end in an error). Callers must NOT assume that a // successful Cancel response means the request was cancelled — the actual terminal outcome (cancelled, landed, or // error) must be checked through the request-summary or request-history APIs. diff --git a/doc/howto/QUICKSTART.md b/doc/howto/QUICKSTART.md index da49f4896..fbe66d203 100644 --- a/doc/howto/QUICKSTART.md +++ b/doc/howto/QUICKSTART.md @@ -200,7 +200,7 @@ Still no credential. `PROVIDER=git` provisions a bare repository at `/tmp/sq-san ✅ Stack is running against provider 'git'. Gateway gRPC port: 55295 -Merge target: /tmp/sq-sandbox/sandbox.git +Land target: /tmp/sq-sandbox/sandbox.git ``` Then the same command as before, unchanged: @@ -364,7 +364,7 @@ make land QUEUE=demo-queue \ URI='git://demo.example.com/demo/refs%2Fheads%2Fbad/2222222222222222222222222222222222222222?sq-fake=build-fail' ``` -That request walks the same path as far as `speculating`, records `building`, and then goes terminal at `error` instead of landing. Other tokens follow the same `sq-fake=` convention and are documented on the fake they drive — `provider-error` on the change provider, `unmergeable` and `mergecheck-error` on the merge checker, `trigger-error` and `build-error` on the build runner. +That request walks the same path as far as `speculating`, records `building`, and then goes terminal at `error` instead of landing. Other tokens follow the same `sq-fake=` convention and are documented on the fake they drive — `provider-error` on the change provider, `trigger-error` and `build-error` on the build runner, and `merge-conflict`, `merge-invalid`, and `merge-error` on Runway's merger. A hand-written URI like the one above belongs to the `fake` rung alone. On `git` it names a commit the merger cannot fetch, and on `github` the change provider tries to resolve it as a pull request — both fail, but for reasons that have nothing to do with the marker. diff --git a/doc/rfc/change-uri.md b/doc/rfc/change-uri.md index 05fd0c959..74cb97686 100644 --- a/doc/rfc/change-uri.md +++ b/doc/rfc/change-uri.md @@ -6,7 +6,7 @@ A change URI is the system-wide identity of a code change — a Pull Request, a Every change URI is an RFC 3986 URI of the form `scheme://{host[:port]}/{path}`, with a uniform division of labor: -- **scheme** — the provider *model*: how to parse the path and which extension family (change provider, merge checker, pusher) can act on it. One scheme per model — deployment flavors of the same model (github.com vs. GitHub Enterprise) do **not** get their own schemes, because the flavor is derivable from the host and two spellings for one instance would break identity. +- **scheme** — the provider *model*: how to parse the path and which change-provider implementation can resolve it. One scheme per model — deployment flavors of the same model (github.com vs. GitHub Enterprise) do **not** get their own schemes, because the flavor is derivable from the host and two spellings for one instance would break identity. - **authority** — the provider *instance*: the `host[:port]` the change lives on. Mandatory. - **path** — the change within that instance, pinned to an exact code state (head SHA or diff ID), so staleness is detectable by comparing the pin against the provider's current state. diff --git a/doc/rfc/consumer-gate.md b/doc/rfc/consumer-gate.md index 342aac93e..9b26ceb39 100644 --- a/doc/rfc/consumer-gate.md +++ b/doc/rfc/consumer-gate.md @@ -76,7 +76,7 @@ If gate state cannot be read (directory missing, I/O error), the check logs, inc The cancellation scenario, expressed as stop → observe → start: 1. The test closes the gate for `runway-mergeconflictcheck` (all partitions, or scoped to the test queue's partition key), before landing. -2. It lands a request. The orchestrator runs it to the merge-conflict-check hand-off; runway's subscriber delivers the check message, and the gate parks it. +2. It lands a request. The orchestrator runs it to the merge-conflict-check hand-off; Runway's subscriber delivers the check message, and the gate parks it. 3. The test awaits the parked record — proof the controller is stopped *and* holding exactly this message. Runway itself is still running; its RPC surface and merge controller are untouched. 4. While stopped, the test observes and acts: it cancels the request, awaits the terminal `cancelled` status through the existing event plane, and asserts no batch ever enrolled the request. 5. The test opens the gate. Within a re-check tick the postponed delivery redelivers, clears the open gate, and proceeds into the controller as a fresh attempt (postponing resets retry accounting); runway answers the now-stale check, and the test asserts the signal is dropped for the halted request. diff --git a/doc/rfc/hook-framework.md b/doc/rfc/hook-framework.md index bd035a8ca..e4a5ae601 100644 --- a/doc/rfc/hook-framework.md +++ b/doc/rfc/hook-framework.md @@ -4,9 +4,9 @@ Fire-and-forget side effects for pipeline lifecycle events: one shared event con ## Problem -The pipelines emit lifecycle transitions — a request lands or fails, a batch merges, a build finishes — but nothing can react outside pipeline state: no warehouse export, no PR comments or closes on merge events, no notifications or audit trails. The log topic is not this seam: SubmitQueue request statuses only, consumed solely to build gateway read models. +The pipelines emit lifecycle transitions — a request lands or fails, a batch lands, a build finishes — but nothing can react outside pipeline state: no warehouse export, no PR comments or closes on land events, no notifications or audit trails. The log topic is not this seam: SubmitQueue request statuses only, consumed solely to build gateway read models. -Two requirements: side effects must never stall or fail the pipeline, and "fire and forget" must not mean lossy — a merge-failure comment that silently never posts is a support ticket. +Two requirements: side effects must never stall or fail the pipeline, and "fire and forget" must not mean lossy — a land-failure comment that silently never posts is a support ticket. ## Proposal @@ -54,7 +54,7 @@ Delivery promise: - `api/base/hook/`: no owning domain, so the message-queue location rule extends — platform-owned contracts live under `api/base/`. - Envelope = only fields every consumer keys on uniformly; subject, queue, and error are occurrence facts → payload. `source`/`type` are strings, not enums, for additive evolution. -- Payload (`Struct`): shaped per type, add-only, documented by its domain; must carry the subject's id and transient facts (merge step outcomes, build failure detail) — the event is their only durable record. Never entity snapshots; hooks resolve entities from stores. +- Payload (`Struct`): shaped per type, add-only, documented by its domain; must carry the subject's id and transient facts (land step outcomes, build failure detail) — the event is their only durable record. Never entity snapshots; hooks resolve entities from stores. ### Hooks and dispatch @@ -93,7 +93,7 @@ message HookEvent { } ``` -A failed batch, carrying merge-result facts persisted nowhere else (protojson: int64 as string, empty fields omitted): +A failed batch, carrying land-result facts persisted nowhere else (protojson: int64 as string, empty fields omitted): ```json { @@ -105,7 +105,7 @@ A failed batch, carrying merge-result facts persisted nowhere else (protojson: i "payload": { "batch_id": "batch-778", "queue": "go-monorepo", - "error": "merge conflict", + "error": "land conflict", "failed_step": "sq-12346", "conflict_paths": ["foo/bar.go"] } @@ -116,7 +116,7 @@ A failed batch, carrying merge-result facts persisted nowhere else (protojson: i - **A contract per domain.** N schemas, N hook shapes, N warehouse tables; one envelope absorbs differences additively. - **Inline hook calls.** Couples pipeline latency to integrations; a crash between write and call silently drops the notification. -- **A second consumer group on the log topic.** Request statuses only; no path to batch, build, merge, or other domains. +- **A second consumer group on the log topic.** Request statuses only; no path to batch, build, land, or other domains. - **Enums for source/type.** protojson rejects unknown enum values; every addition would break consumers. - **Subject, queue, or error on the envelope.** Occurrence facts; they live in the payload. No major event platform carries a top-level error. - **Entity snapshots as payload.** Stale on redelivery; competes with the store; drags domain schemas into the shared contract. diff --git a/doc/rfc/index.md b/doc/rfc/index.md index d3319e661..2e8319e25 100644 --- a/doc/rfc/index.md +++ b/doc/rfc/index.md @@ -13,7 +13,7 @@ Design documents and technical proposals, grouped by scope. Shared/cross-cutting ## SubmitQueue -- [Orchestrator Workflow](submitqueue/workflow.md) - Queue-driven controller pipeline from gateway entry through batching, scoring, build, merge, and conclude +- [Orchestrator Workflow](submitqueue/workflow.md) - Queue-driven controller pipeline from gateway entry through batching, scoring, build, land, and conclude - [Gateway History APIs](submitqueue/history-api.md) - Request lifecycle history exposed through separate request ID and change ID endpoints - [Build Runner](submitqueue/build-runner.md) - Vendor-agnostic BuildRunner interface, provider-neutral BuildStatus lifecycle, and how the orchestrator wires it into the build stage - [Extension Contract](submitqueue/extension-contract.md) - When extensions take orchestrator identity (request/batch) and resolve granular content themselves vs. take controller-resolved data; revises the BuildRunner base/head contract @@ -24,7 +24,7 @@ Design documents and technical proposals, grouped by scope. Shared/cross-cutting ## Stovepipe -- [Stovepipe Workflow](stovepipe/workflow.md) - Post-merge validation pipeline overview: ingest, process, build, record greenness, analyze projects, notify downstream +- [Stovepipe Workflow](stovepipe/workflow.md) - Post-land validation pipeline overview: ingest, process, build, record greenness, analyze projects, notify downstream - [Process stage](stovepipe/steps/process.md) - Build-strategy decision, per-queue concurrency gate, backlog coalescing, entity model, platform prerequisites - [Build stage](stovepipe/steps/build.md) - Trigger-only stage and Stovepipe's URI-based BuildRunner contract - [Buildsignal stage](stovepipe/steps/buildsignal.md) - Build polling, terminal status persistence, and the handoff to record @@ -34,4 +34,4 @@ Design documents and technical proposals, grouped by scope. Shared/cross-cutting ## Runway -- [Runway Workflow](runway/workflow.md) - Landing service: merge-conflict checking and merging on behalf of SubmitQueue +- [Runway Workflow](runway/workflow.md) - Merge service: merge-conflict checking and merging on behalf of SubmitQueue diff --git a/doc/rfc/messagequeue-contract.md b/doc/rfc/messagequeue-contract.md index c52201e53..7c0383650 100644 --- a/doc/rfc/messagequeue-contract.md +++ b/doc/rfc/messagequeue-contract.md @@ -57,7 +57,7 @@ So the option earns its place as the single, language-neutral source of truth fo ### Go binding: the generated `protopb` -The generated message types in `protopb` are the Go binding, sitting beside `proto/` exactly as for the RPC contracts. The contract package adds only thin helpers — `protojson` (de)serialization and the `topic_keys` reflection lookup. Shared field types (`change.Change`, `mergestrategy.MergeStrategy`) are themselves shared protos under `api/base/{change,mergestrategy}/proto`, imported by every contract that needs them. +The generated message types in `protopb` are the Go binding, sitting beside `proto/` exactly as for the RPC contracts. The contract package adds only thin helpers — `protojson` (de)serialization and the `topic_keys` reflection lookup. Shared field types (`change.Change`, `mergestrategy.Strategy`) are themselves shared protos under `api/base/{change,mergestrategy}/proto`, imported by every contract that needs them. ## Example @@ -82,9 +82,9 @@ message ExampleRequest { message ExampleResult { // One shape, two queues: the same result is published under the check-result - // key for a dry run and the merge-result key for a committing run. + // key for a dry run and the land-result key for a committing run. option (uber.base.messagequeue.topic_keys) = "example-check-result"; - option (uber.base.messagequeue.topic_keys) = "example-merge-result"; + option (uber.base.messagequeue.topic_keys) = "example-land-result"; string id = 1; // Echoes the request's correlation id. bool success = 2; diff --git a/doc/rfc/sql-queue-rfc.md b/doc/rfc/sql-queue-rfc.md index 34bd5c840..2c743e649 100644 --- a/doc/rfc/sql-queue-rfc.md +++ b/doc/rfc/sql-queue-rfc.md @@ -18,7 +18,7 @@ MySQL-based distributed message queue with immutable message log, per-consumer-g ### Motivation SubmitQueue needs a reliable message queue for coordinating asynchronous workflows: -- **Orchestrator** publishes merge jobs and speculative build requests to workers +- **Orchestrator** publishes land jobs and speculative build requests to workers - **Workers** need distributed coordination without duplicate processing - **Crash recovery** must preserve exactly where processing stopped @@ -223,7 +223,7 @@ See `platform/extension/messagequeue/mysql/schema/queue_subscriber_heartbeats.sq ### Dead Letter Queue -DLQ messages are stored in the same `queue_messages` table under a different topic name (original topic + DLQ suffix, e.g., `merge_queue_dlq`). This allows DLQ messages to be consumed using the normal subscriber with the DLQ topic name. DLQ-specific fields (`failed_at`, `failure_count`, `last_error`, `original_topic`) are populated when a message is moved to DLQ; they are zero/empty for normal messages. +DLQ messages are stored in the same `queue_messages` table under a different topic name (original topic + DLQ suffix, e.g., `land_queue_dlq`). This allows DLQ messages to be consumed using the normal subscriber with the DLQ topic name. DLQ-specific fields (`failed_at`, `failure_count`, `last_error`, `original_topic`) are populated when a message is moved to DLQ; they are zero/empty for normal messages. ## Message Flow @@ -387,7 +387,7 @@ For our use case, we need ordering per repository. With Watermill: With our custom implementation: - Single `queue_messages` table for all topics and partitions -- Rows like `('merge_events', 'repo-123', offset, ...)` provide ordering within partition +- Rows like `('land_events', 'repo-123', offset, ...)` provide ordering within partition - No schema migrations for new repos or topics - Ordering guaranteed within `(topic, partition_key)` @@ -453,7 +453,7 @@ The current design separates the immutable message log from per-consumer-group d **At-Least-Once vs Exactly-Once** - Simpler, better performance - Applications must handle duplicates -- Mitigation: Idempotency keys (e.g., merge request ID) +- Mitigation: Idempotency keys (e.g., land request ID) ## Appendix diff --git a/doc/rfc/stovepipe/steps/build.md b/doc/rfc/stovepipe/steps/build.md index 0501c9900..f630870b2 100644 --- a/doc/rfc/stovepipe/steps/build.md +++ b/doc/rfc/stovepipe/steps/build.md @@ -121,7 +121,7 @@ Both domains have a `build` controller that triggers via a build-runner extensio ### SubmitQueue -SubmitQueue validates **stacks of changes** before merging. Its `build` controller loads `base []entity.Batch` (ordered dependency batches) and `head entity.Batch` and triggers: +SubmitQueue validates **stacks of changes** before landing. Its `build` controller loads `base []entity.Batch` (ordered dependency batches) and `head entity.Batch` and triggers: ```go buildID, err := buildRunner.Trigger(ctx, base, head, metadata) diff --git a/doc/rfc/stovepipe/steps/process.md b/doc/rfc/stovepipe/steps/process.md index e541139d6..8f6962733 100644 --- a/doc/rfc/stovepipe/steps/process.md +++ b/doc/rfc/stovepipe/steps/process.md @@ -73,7 +73,7 @@ A slot is held from admit until the build goes terminal (`process → build → ## Raising `max_concurrent` (speculative validation) -Setting `max_concurrent = N > 1` overlaps validations to start work sooner, and it is **safe** — because Stovepipe validates **already-landed, linear trunk heads**, not pre-merge candidates. Successive commits (`G0 → A → B → C …`) each contain everything below them, so validating `G0..B` already tests A+B together. (A pre-merge queue serializes to catch "two changes green alone, broken combined"; that risk isn't present here.) A green result for head `H` on baseline `B` is an immutable property of `H`, true no matter where last-green moves afterward. +Setting `max_concurrent = N > 1` overlaps validations to start work sooner, and it is **safe** — because Stovepipe validates **already-landed, linear trunk heads**, not pre-land candidates. Successive commits (`G0 → A → B → C …`) each contain everything below them, so validating `G0..B` already tests A+B together. (A pre-land queue serializes to catch "two changes green alone, broken combined"; that risk isn't present here.) A green result for head `H` on baseline `B` is an immutable property of `H`, true no matter where last-green moves afterward. The scheme: each admit pins its baseline to last-green *at admit time*; a green head is adopted even if last-green has since advanced. A late green result is either the newest (adopt) or already behind the pointer (**moot** — dropped, never a regression). diff --git a/doc/rfc/stovepipe/workflow.md b/doc/rfc/stovepipe/workflow.md index bad91d8d8..00da86a97 100644 --- a/doc/rfc/stovepipe/workflow.md +++ b/doc/rfc/stovepipe/workflow.md @@ -1,6 +1,6 @@ # Stovepipe Workflow -Stovepipe answers one question for the rest of the company: **at which commit is this thing green?** It continuously polls a repository branch for its latest commit, validates that commit, works out which projects (if any) are broken at it, records the result, and notifies downstream systems so they can gate deployments on a known-good commit. It is a post-merge service: code lands first, Stovepipe finds out whether it was good. +Stovepipe answers one question for the rest of the company: **at which commit is this thing green?** It continuously polls a repository branch for its latest commit, validates that commit, works out which projects (if any) are broken at it, records the result, and notifies downstream systems so they can gate deployments on a known-good commit. It is a post-land service: code lands first, Stovepipe finds out whether it was good. The pipeline is a queue-driven chain of small, single-purpose controllers, in the same style as SubmitQueue (SQ). Each controller consumes one topic, advances one entity, and publishes to the next topic. Most hops carry only an **ID** and the controller reloads the entity from storage; the entry hop carries the caller's input because there is no row to load yet. The high-level shape is: @@ -23,7 +23,7 @@ Everything Stovepipe records greenness *about* is a URI: a specific commit on a ### Queue — the unit of identity for "what we validate" -Stovepipe reuses SQ's **Queue** concept for the same two reasons SQ does — to **namespace the generated IDs** and to give callers a **stable handle for the repo+ref being validated** — plus a third that is specific to a post-merge validator: a Queue **owns the last-known-good URI** and the greenness history for its branch. +Stovepipe reuses SQ's **Queue** concept for the same two reasons SQ does — to **namespace the generated IDs** and to give callers a **stable handle for the repo+ref being validated** — plus a third that is specific to a post-land validator: a Queue **owns the last-known-good URI** and the greenness history for its branch. A Queue is named by a **stable logical string** (e.g. `monorepo/main`), and that name is what the ingest API takes — *not* a raw URI. SourceControl/config resolves the Queue name to a concrete VCS URI base. This keeps callers (and the external poller) free of VCS detail: they say "the `monorepo/main` Queue has moved", and Stovepipe resolves what that means. diff --git a/doc/rfc/submitqueue/extension-contract.md b/doc/rfc/submitqueue/extension-contract.md index 80c241ff2..d45498a74 100644 --- a/doc/rfc/submitqueue/extension-contract.md +++ b/doc/rfc/submitqueue/extension-contract.md @@ -4,7 +4,7 @@ Design notes for what SubmitQueue's pluggable extensions accept: orchestrator ** ## Problem -Extension input granularity is inconsistent across the pipeline stages (see [workflow.md](workflow.md)). `conflict.Analyzer` takes identity (`entity.Batch`); `scorer`, `changeprovider`, `buildrunner`, `pusher` take controller-resolved `entity.Change`. The split caps what an extension can do: +Extension input granularity is inconsistent across the pipeline stages (see [workflow.md](workflow.md)). `conflict.Analyzer` takes identity (`entity.Batch`); `scorer`, `changeprovider`, and `buildrunner` take controller-resolved `entity.Change`. The split caps what an extension can do: - `ConflictType` already names `target_overlap`, but a real target-overlap analyzer **cannot be written** — the dependency-analysis stage hands it identity-level batches (no changed targets) and the contract has nowhere to put them. - `scorer` gets a URIs-only `Change`, so a heuristic scorer **cannot see** lines-changed / file-count. @@ -15,7 +15,7 @@ Both unblock with the shape `conflict` already uses: accept identity, resolve in - **Decision/action extensions** take orchestrator identity at their stage granularity and resolve granular content through narrowly-injected dependencies. Request stage → `entity.Request`; batch stage → `entity.Batch` / `[]entity.Batch`. Both are thin reference entities (a `Request` carries URIs, not diffs; a `Batch` carries IDs, not changes). - **Resolution targets** — `storage`, `changestore`, `queueconfig` — stay key/value-shaped. They are what the others resolve *through* (see [storage/README.md](../../../submitqueue/extension/storage/README.md) and [AGENTS.md](../../../AGENTS.md)). Refinement: the storage *aggregate* has since gained the same per-queue factory resolution every other seam has — the stores it hands back remain strictly key/value, bound to their queue, while the cross-queue read-model stores stay individually-injected singletons. -- **Output mirrors the input unit.** Each output element self-identifies with the input it corresponds to — `changeprovider`'s `ChangeInfo` carries its `URI`, `conflict`'s `Conflict` carries its `BatchID` — so a flat list suffices and the caller correlates results back to inputs without re-deriving boundaries. A *wrapper* entity (`entity.BatchChanges`) is introduced only to aggregate *up* to a coarser unit than the elements — the scorer needs batch-wide line/file totals, so the rollup earns its keep; no `RequestChanges` exists because nothing needs request-wide rollups. And when the input is a *collection* of independently-actioned units, the output groups by them: `pusher`, fed `[]entity.Batch`, returns outcomes grouped per batch, the same way `conflict` already tags each `Conflict` with its in-flight `BatchID`. +- **Output mirrors the input unit.** Each output element self-identifies with the input it corresponds to — `changeprovider`'s `ChangeInfo` carries its `URI`, `conflict`'s `Conflict` carries its `BatchID` — so a flat list suffices and the caller correlates results back to inputs without re-deriving boundaries. A *wrapper* entity (`entity.BatchChanges`) is introduced only to aggregate *up* to a coarser unit than the elements — the scorer needs batch-wide line/file totals, so the rollup earns its keep; no `RequestChanges` exists because nothing needs request-wide rollups. ### What each stage resolves today @@ -24,10 +24,9 @@ Both unblock with the shape `conflict` already uses: accept identity, resolve in | `validate` | `entity.Request` | nothing — `request.Change` is already in hand (the change-store reads here serve duplicate detection) | `request.Change` → `changeprovider` | | `dependency` | `entity.Batch` + active `[]entity.Batch` | **nothing** — the batch it analyzes is already persisted, with `Contains` set to `[requestID]` | `entity.Batch`, `[]entity.Batch` → `conflict` | | `score` | `entity.Batch`, then each `entity.Request` | batch → requests | `request.Change` per request, then multiplies the scores → `scorer` | -| `build` | `entity.Batch`, then `collectChanges` | batch → requests → changes, **flattening batch boundaries** | base `[]Change`, head `[]Change` → `buildrunner` | -| `merge` | `entity.Batch`, then `collectChanges` | batch → requests → changes | `[]Change` → `pusher` | +| `build` | head `entity.Batch` + path base `[]entity.Batch` | **nothing** — the build runner resolves each batch through its injected `changeset.Resolver` | base `[]entity.Batch`, head `entity.Batch` → `buildrunner` | -Two facts this grounds: `conflict` already resolves nothing (the baseline), and the batch→changes walk is **already duplicated** in `build`/`merge` `collectChanges` — the shared resolver below only consolidates it. +This grounds `conflict` as the baseline: it already resolves nothing because the controller passes the identity it needs. ## Verdict @@ -37,22 +36,20 @@ Two facts this grounds: `conflict` already resolves nothing (the baseline), and | `scorer.Scorer` | score | flat `Change`, per request | `entity.Batch` — resolve + reduce internally | one batch score (`float64`) — unchanged | request store + change provider | | `changeprovider.ChangeProvider` | validate | `Change` | `entity.Request` | per-URI change info (`[]ChangeInfo`, `URI`-tagged) — unchanged | none — it *is* the resolver | | `buildrunner.BuildRunner` | build | base/head `[]Change` | base `[]entity.Batch` + head `entity.Batch` | build id, then status/cancel (`BuildID`, `BuildStatus`) — unchanged | request store + change provider | -| `pusher.Pusher` *(removed)* | merge | — | **moved out-of-process to runway** (`merge` / `merge-signal`); see the note below the table | — | — | | `storage`, `changestore`, `queueconfig` | — | keys + entities | unchanged — resolution targets | entities | — | -**Outputs are unchanged.** This RFC moves the *input* toward identity; the four live return contracts — conflicts, score, change info, build id/status — are exactly what they are today. (The `pusher` row is not an in-process extension: merge runs out-of-process in runway, so its output is not part of this catalog — see the note below.) No other output shape changes. +**Outputs are unchanged.** This RFC moves the *input* toward identity; the four live return contracts — conflicts, score, change info, build id/status — are exactly what they are today. No output shape changes. -The validate-time mergeability **check** and the **merge** itself both run **asynchronously and out-of-process** in runway rather than as in-process extensions, over the one shared `MergeRequest`/`MergeResult` contract — a check is a dry run of a merge. `validate` hands off directly to runway (→ `merge-conflict-check`, result back via `mergeconflictsignal`); `merge` hands the batch to runway (→ `merge`, result back via `mergesignal`) rather than calling an in-process `pusher`. See [workflow.md](workflow.md). The in-process `mergechecker` and `pusher` packages are unused on the pipeline path. +The validate-time landability **check** and the **land** itself both run **asynchronously and out-of-process** in Runway rather than as in-process extensions. SubmitQueue adapts its land request to Runway's shared `MergeRequest`/`MergeResult` contract, where a conflict check is a dry run of a merge. `validate` hands off directly to Runway (→ `merge-conflict-check`, result back via `landconflictsignal`); `land` hands the batch to Runway (→ `runway-merge`, result back via `landsignal`). See [workflow.md](workflow.md). SubmitQueue retains no parallel in-process checking or pushing contract. Non-obvious points: - **scorer** — owning the batch moves batch-level reduction (today the controller's multiplicative product) into the scorer, where the `composite` reduce step already lives. -- **buildrunner** — this **revises** [build-runner.md](build-runner.md), which deliberately kept batches out of the boundary. The base/head split survives, expressed as batches; the provider still operates on changes (the shared resolver produces them inside the extension). Cost: a `buildrunner` / `pusher` implementation now depends on a request store + change provider. -- **pusher** — a *list* of batches (not one) designs for a merge-train: land several ready batches, or a batch with not-yet-landed deps, in one atomic push. Today merge pushes a single batch because deps are already on trunk. Since the input is now a list, the output groups outcomes per batch (`BatchID`-tagged, with per-change commit detail kept underneath) instead of one flat per-change list — the only output shape this RFC changes. Push atomicity is unchanged (all-or-nothing across the whole call), so a per-batch *status* is intentionally omitted: a partial-landing train would be a separate, larger change to the atomicity contract. +- **buildrunner** — this **revises** [build-runner.md](build-runner.md), which deliberately kept batches out of the boundary. The base/head split survives, expressed as batches; the provider still operates on changes (the shared resolver produces them inside the extension). Cost: a `buildrunner` implementation now depends on a request store + change provider. ## Mechanism -Dependencies are injected per-extension at the existing `Factory.For` (wiring: `service/submitqueue/orchestrator/server/main.go`) — only the handles a contract justifies, never the whole storage aggregator. The repeated batch→changes walk becomes one shared resolver (today's duplicated `collectChanges`, consolidated, and preserving the batch boundaries build's copy flattens). Controllers shrink to passing the identity entity they already load. +Dependencies are injected per-extension at the existing `Factory.For` (wiring: `service/submitqueue/orchestrator/server/main.go`) — only the handles a contract justifies, never the whole storage aggregator. Batch→changes resolution is centralized in `changeset.Resolver`, while `buildrunner.ResolveBatches` shares the ordered flattening needed by build-runner backends. Controllers pass the identity entities they already load. `entity.BatchChanges` is kept, not removed — it becomes the shared resolver's *detailed output* (URIs + provider details for a batch, what the scorer consumes) rather than a value the score controller assembles and passes in. Its line/file helpers move with it; only its producer changes. @@ -60,5 +57,5 @@ Dependencies are injected per-extension at the existing `Factory.For` (wiring: ` - **Status quo (controller resolves).** Keeps extensions pure and trivially testable, but thickens controllers and caps every extension at what the controller chose to pre-compute — the two blocked features are that ceiling. - **Literal string IDs.** An extra read per call when the controller already holds the entity; pass thin reference entities instead. -- **Per-implementation batch→changes resolution.** How the `build`/`merge` duplication arose; one shared resolver instead. +- **Per-implementation batch→changes traversal.** Duplicates storage and ordering rules across backends; use the shared resolver and build-runner helper instead. - *Acknowledged:* decision extensions gain dependencies and are no longer pure functions — mitigated by their existing mock packages and `Factory` injection. diff --git a/doc/rfc/submitqueue/modular-queue-wiring.md b/doc/rfc/submitqueue/modular-queue-wiring.md index 1acdddb89..a3e716e10 100644 --- a/doc/rfc/submitqueue/modular-queue-wiring.md +++ b/doc/rfc/submitqueue/modular-queue-wiring.md @@ -281,12 +281,12 @@ row appears on topic "start", partition key "monorepo/exp" gateway orchestrator stovepipe runway ──────────────────────────────────────────────────────────────────────────────────────────── Deps seams counter · storage · changeprovider · storage · counter · storage · - queueconfig.Store · buildrunner · scorer sourcecontrol. merger Factory + queueconfig.Store · buildrunner · scorer sourcecontrol. lander Factory requestlog store analyzer · validator Factory · (+7 speculation) queueconfig.Store - Stages log start · cancel · process mergeconflictcheck · - (rows) validate · batch · merge + Stages log start · cancel · process landconflictcheck · + (rows) validate · batch · land … (+ DLQ column) Controllers Gateway Orchestrator Stovepipe Runway diff --git a/doc/rfc/submitqueue/speculation-generator-best-first.md b/doc/rfc/submitqueue/speculation-generator-best-first.md index 458e5e007..5558109fc 100644 --- a/doc/rfc/submitqueue/speculation-generator-best-first.md +++ b/doc/rfc/submitqueue/speculation-generator-best-first.md @@ -447,7 +447,7 @@ The ordering stays the same. `CandidatePath.RankingScore` contains this logarith - `Succeeded` fixes an assumption to succeeds. - `Failed` or `Cancelled` fixes an assumption to fails. - `Cancelling` remains undecided because cancellation may lose a race with completion. -- `Merging` also remains undecided, because a merge can fail. It is tempting to treat it as committed to landing and skip the scorer call, but that puts a state-specific policy inside the search: whether a path betting against a merging batch is worth funding is a question of price, and price belongs to the scorer. The allocator draws the same line — "no batch state enters this decision" — and the generator holds it too. Nothing is lost by staying open: a single passed path still waits for the merge result, while passed paths covering every outcome let the controller bypass the dependency (see [speculation.md](speculation.md)). Funding the unlikely side spends budget, which is the allocator's to ration. +- `Landing` also remains undecided, because a land can fail. It is tempting to treat it as committed to landing and skip the scorer call, but that puts a state-specific policy inside the search: whether a path betting against a landing batch is worth funding is a question of price, and price belongs to the scorer. The allocator draws the same line — "no batch state enters this decision" — and the generator holds it too. Nothing is lost by staying open: a single passed path still waits for the land result, while passed paths covering every outcome let the controller bypass the dependency (see [speculation.md](speculation.md)). Funding the unlikely side spends budget, which is the allocator's to ration. - A fixed assumption stays in the returned path but contributes probability 1 and has no flip. - A shared dependency is scored once per run. diff --git a/doc/rfc/submitqueue/speculation.md b/doc/rfc/submitqueue/speculation.md index 1887cff7e..526bfa9c3 100644 --- a/doc/rfc/submitqueue/speculation.md +++ b/doc/rfc/submitqueue/speculation.md @@ -1,20 +1,20 @@ # Speculation -A merge queue that verifies one change at a time is limited by its slowest build. Speculation removes that limit: it builds a batch early, against an assumption about how the conflicting batches ahead of it will resolve, so a valid build is usually ready by the time they do. +A land queue that verifies one change at a time is limited by its slowest build. Speculation removes that limit: it builds a batch early, against an assumption about how the conflicting batches ahead of it will resolve, so a valid build is usually ready by the time they do. -Work enters SubmitQueue as **batches** — changes verified and merged together. Two batches **conflict** when they touch the same code, which makes the earlier one a **dependency** of the later. A **path** is one set of assumptions about how a batch's dependencies resolve, and the batch it builds is the path's **head**. +Work enters SubmitQueue as **batches** — changes verified and landed together. Two batches **conflict** when they touch the same code, which makes the earlier one a **dependency** of the later. A **path** is one set of assumptions about how a batch's dependencies resolve, and the batch it builds is the path's **head**. -On every queue update the **speculate controller** reruns from scratch: it reads the current state, applies the incoming signals, asks a pluggable **Speculator** which paths are worth building within the CI budget, and persists only those. Everything else is recomputed next time, never stored. A batch normally merges after its dependencies resolve and a matching build has passed; complete passed coverage of every unresolved outcome lets it bypass those dependencies. +On every queue update the **speculate controller** reruns from scratch: it reads the current state, applies the incoming signals, asks a pluggable **Speculator** which paths are worth building within the CI budget, and persists only those. Everything else is recomputed next time, never stored. A batch normally lands after its dependencies resolve and a matching build has passed; complete passed coverage of every unresolved outcome lets it bypass those dependencies. ## The speculation run -The speculate controller runs whenever the queue changes — after a new batch, a completed build, a merge result, or a cancel. Each publishes a **dirty signal** carrying the changed batch ID, partitioned by the queue so a queue's runs happen one at a time. The dirty signal is an internal queue contract — payload in `submitqueue/core/messagequeue`, topic key in `submitqueue/core/topickey`. +The speculate controller runs whenever the queue changes — after a new batch, a completed build, a land result, or a cancel. Each publishes a **dirty signal** carrying the changed batch ID, partitioned by the queue so a queue's runs happen one at a time. The dirty signal is an internal queue contract — payload in `submitqueue/core/messagequeue`, topic key in `submitqueue/core/topickey`. ``` dirty(queue) — "trigger a run" — published after: - a new batch - a build completes (success/failure/cancellation) - - a merge result arrives (success/failure) + - a land result arrives (success/failure) - a cancel │ │ carries the changed batch ID, partitioned by queue, so a @@ -31,15 +31,15 @@ The speculate controller runs whenever the queue changes — after a new batch, │ → paths to build, paths to preempt │ 4 check validate that output: drop actions it shouldn't propose │ (non-Speculating head, refuted, incoherent, terminal path) - │ 5 write record each head's decisions; send build / cancel / merge messages + │ 5 write record each head's decisions; send build / cancel / land messages │ (a head whose write loses is re-planned on the next run) │ ├─▶ build / cancel (path ID, attempt) → build (orchestrator/controller/build): │ reserve → BuildRunner.Trigger(base, head) → record build ID, mark path building │ CI runs → buildsignal marks path passed/failed/cancelled ─▶ dirty(queue) │ - └─▶ merge (batch) → Runway performs the merge - → mergesignal marks the batch succeeded/failed ─▶ dirty(queue) + └─▶ land (batch) → Runway performs the land + → landsignal marks the batch succeeded/failed ─▶ dirty(queue) ``` ### State reconciliation @@ -54,8 +54,8 @@ Every write is a compare-and-swap: a writer that loses re-reads on a later run. Verdicts are controller-owned facts: the Speculator can neither compute nor veto them. -- **Merge.** Each path carries an assumption about every dependency — *succeeds* (built on top of) or *fails* (built without). Normally, once a path's build has passed and every dependency has finished the way the path assumed — one assumed *succeeds* has merged, one assumed *fails* has failed or been cancelled — the speculate controller moves the head to Merging and hands it to Runway. A dependency that is merely *merging* has not finished, because a merge can fail, so a single matching path still waits for the answer. Complete passed coverage is the exception described in Bypass large diff: it lets a head merge before those answers arrive. If the hand-off is lost, the next run re-sends it. The same run sets the head's remaining in-flight paths *cancelling*: once the head can merge they cannot help, and they hold CI slots until they stop. The mergesignal controller records Runway's terminal result: success marks the head Succeeded, while failure marks it Failed. The result publishes a single dirty signal — no per-dependent fan-out — and the next run refutes paths whose assumption disagrees with the result: *fails* assumptions after success, *succeeds* assumptions after failure. The hand-off is idempotent, so Runway reports success without another merge when the change is already present. A chain ordinarily merges one at a time, but a fully covered head can bypass its unsettled predecessors. -- **Failure (no viable path).** A batch fails when every possible future has a failed build — no path can pass, so it can never merge. +- **Land.** Each path carries an assumption about every dependency — *succeeds* (built on top of) or *fails* (built without). Normally, once a path's build has passed and every dependency has finished the way the path assumed — one assumed *succeeds* has landed, one assumed *fails* has failed or been cancelled — the speculate controller moves the head to Landing and hands it to Runway. A dependency that is merely *landing* has not finished, because a land can fail, so a single matching path still waits for the answer. Complete passed coverage is the exception described in Bypass large diff: it lets a head land before those answers arrive. If the hand-off is lost, the next run re-sends it. The same run sets the head's remaining in-flight paths *cancelling*: once the head can land they cannot help, and they hold CI slots until they stop. The landsignal controller records Runway's terminal result: success marks the head Succeeded, while failure marks it Failed. The result publishes a single dirty signal — no per-dependent fan-out — and the next run refutes paths whose assumption disagrees with the result: *fails* assumptions after success, *succeeds* assumptions after failure. The hand-off is idempotent, so Runway reports success without another land when the change is already present. A chain ordinarily lands one at a time, but a fully covered head can bypass its unsettled predecessors. +- **Failure (no viable path).** A batch fails when every possible future has a failed build — no path can pass, so it can never land. - **Cancel.** A cancelled batch is driven terminal: its in-flight paths are set *cancelling*, then the batch is marked Cancelled once they stop (see Cancellation). ### Conflict relaxation @@ -64,23 +64,23 @@ Conflict analysis is conservative — it flags any *possible* conflict — so he **Not implemented.** An earlier design expressed it per path, with a third assumption value — *ignored* — meaning "this path makes no claim about this dependency". Nothing ever produced one, and the value has been removed rather than left as vocabulary the system could not create. -When relaxation is built, it belongs in the **controller**, as a trim of the dependency list before the snapshot is handed over: the Speculator then sees a head whose dependencies are exactly the ones that count, and a path stays a total function over them — one assumption per dependency, each *succeeds* or *fails*, with no third state to reason about. That keeps the decision where the other correctness decisions live, since dropping a dependency is a judgement about what may land untested, not about which candidate is most promising. It also keeps every consumer honest by construction: a merge gate, a refutation check, or a generator cannot forget to special-case a value that does not exist. +When relaxation is built, it belongs in the **controller**, as a trim of the dependency list before the snapshot is handed over: the Speculator then sees a head whose dependencies are exactly the ones that count, and a path stays a total function over them — one assumption per dependency, each *succeeds* or *fails*, with no third state to reason about. That keeps the decision where the other correctness decisions live, since dropping a dependency is a judgement about what may land untested, not about which candidate is most promising. It also keeps every consumer honest by construction: a land gate, a refutation check, or a generator cannot forget to special-case a value that does not exist. The open question that design has to answer is what a stored path means once the trim changes between runs — a path built against a trimmed list no longer lines up with a head whose list has grown back, and `isWellFormed` rejects it. The per-path marker made that case self-describing; a trim does not, so the trim has to be either stable for a head's lifetime or recorded alongside the path. -Example of the payoff either way: `H` conflicts with `B1` and weak `B2`. Relax `B2`, and `H` merges once `B1` merges and its build passes — even if `B2` later merges. Without it, `H` waits on both. +Example of the payoff either way: `H` conflicts with `B1` and weak `B2`. Relax `B2`, and `H` lands once `B1` lands and its build passes — even if `B2` later lands. Without it, `H` waits on both. ### Bypass large diff -If a batch's passed builds cover *every* way its dependencies could resolve, the outcome is the same either way — so it can merge now, ahead of them. Classic case: a small change stuck behind a slow one is built both with and without it; both pass, and it merges immediately. +If a batch's passed builds cover *every* way its dependencies could resolve, the outcome is the same either way — so it can land now, ahead of them. Classic case: a small change stuck behind a slow one is built both with and without it; both pass, and it lands immediately. The controller checks coverage over only the dependencies that have not settled yet. Settled dependencies pin each surviving path to the outcome that actually happened; for every combination of the remaining dependencies, the path set must contain a passed, unbroken path with that combination of assumptions. If any combination is missing, unbuilt, failed, or contradicted by a settled dependency, the head waits normally. The check only observes paths the Speculator already funded — it does not fund the exponential path space itself or alter the queue's build budget. -Coverage makes the bypass sound because whichever way the dependencies later resolve, a passed build already validated the resulting set of changes. The build order and merge order differ: a path assuming dependency `D` succeeds validates `D` then head `H`, while bypass lands `H` before `D`. SubmitQueue treats those orders as content-equivalent. Runway still performs the real merge, so if the reordered changes conflict textually, the older dependency can fail after the newer head has bypassed it; this is an accepted cost of landing the fully covered head early rather than a licence to put unmergeable content on the target. +Coverage makes the bypass sound because whichever way the dependencies later resolve, a passed build already validated the resulting set of changes. The build order and land order differ: a path assuming dependency `D` succeeds validates `D` then head `H`, while bypass lands `H` before `D`. SubmitQueue treats those orders as content-equivalent. Runway still performs the real land, so if the reordered changes conflict textually, the older dependency can fail after the newer head has bypassed it; this is an accepted cost of landing the fully covered head early rather than a licence to put unlandable content on the target. ### Cancellation -Cancellation is best-effort: a batch marked *cancelling* may still merge if a merge wins the race, so terminal states prevail. A cancel sets the intent; a later run drives it terminal. +Cancellation is best-effort: a batch marked *cancelling* may still land if a land wins the race, so terminal states prevail. A cancel sets the intent; a later run drives it terminal. Two kinds of cancel, split by owner: @@ -93,7 +93,7 @@ Cancelling a path sends a cancel (path ID, attempt) to the build controller, whi ## Speculator Extension -The one extension. It decides *which paths to build and which running ones to cancel* — nothing else; the controller handles the rest (reconciling facts, cancelling ruled-out paths, verdicts, checking output). A swapped-in Speculator changes which paths run, never whether a batch merges or fails. +The one extension. It decides *which paths to build and which running ones to cancel* — nothing else; the controller handles the rest (reconciling facts, cancelling ruled-out paths, verdicts, checking output). A swapped-in Speculator changes which paths run, never whether a batch lands or fails. **The contract** is `Speculate(batches, pathSets) → []Speculation`: @@ -116,6 +116,6 @@ Signatures live in code and are not copied here, so they cannot drift. This sect **Entities** — [`submitqueue/entity/speculation.go`](../../../submitqueue/entity/speculation.go). A `SpeculationPath` is a head batch plus one `PathDependency` per dependency in queue order, each carrying a `DependencyAssumption`: *succeeds* or *fails*. A `SpeculationPathEntry` is the stored record of one chosen path, keyed by a hash of its content, plus its status and attempt number; it holds no build reference — the execution record has that, keyed by (path ID, attempt) — and no ranking score, which means nothing outside the run that produced it. A `SpeculationPathSet` is one head's chosen paths, live and recently finished, under a single version for compare-and-swap. Every logical path is self-describing, but a store may encode the common head and ordered dependency IDs once per set and keep each path's assumptions positionally — one bit per dependency. -**Speculator** — [`submitqueue/extension/speculation/speculator`](../../../submitqueue/extension/speculation/speculator/README.md). `Speculate` takes one queue snapshot (the batches and their path sets) and returns the build and cancel actions it proposes; a path it wants left alone has no entry. Actions must target Speculating heads. Verdicts stay controller-owned, so there is no merge or fail action. +**Speculator** — [`submitqueue/extension/speculation/speculator`](../../../submitqueue/extension/speculation/speculator/README.md). `Speculate` takes one queue snapshot (the batches and their path sets) and returns the build and cancel actions it proposes; a path it wants left alone has no entry. Actions must target Speculating heads. Verdicts stay controller-owned, so there is no land or fail action. **Generator and Allocator** — [`generator`](../../../submitqueue/extension/speculation/generator/README.md) and [`allocator`](../../../submitqueue/extension/speculation/allocator/README.md), the two composition points inside the default Speculator. The Generator opens a pull-based stream of candidate paths over the batches; the Allocator spends the build budget over that stream, reconciling it against the path sets. Both abort on a cancelled context. diff --git a/doc/rfc/submitqueue/workflow.md b/doc/rfc/submitqueue/workflow.md index b50e3958c..b1d590948 100644 --- a/doc/rfc/submitqueue/workflow.md +++ b/doc/rfc/submitqueue/workflow.md @@ -1,8 +1,8 @@ # Orchestrator Workflow -The orchestrator processes land requests through a queue-driven pipeline of small, single-purpose controllers. The gateway accepts a request over RPC and hands it off asynchronously; from there each controller consumes one topic, advances the request or batch, and publishes to the next topic. Most hops carry only an ID — the controller fetches the entity from storage — while a few entry points (`start`, `buildsignal`, `log`) carry the full payload because there is no row to fetch yet. Some stages cross a service boundary: they publish a full payload to the other service's queue and consume a full payload back, because neither service can read the other's storage. (The `validate` and `merge` stages both hand work to runway — a merge-conflict check and the merge itself — and consume its result on `mergeconflictsignal` / `mergesignal`.) See the queue-payload-boundary rule in [AGENTS.md](../../../AGENTS.md). +The orchestrator processes land requests through a queue-driven pipeline of small, single-purpose controllers. The gateway accepts a request over RPC and hands it off asynchronously; from there each controller consumes one topic, advances the request or batch, and publishes to the next topic. Most hops carry only an ID — the controller fetches the entity from storage — while a few entry points (`start`, `buildsignal`, `log`) carry the full payload because there is no row to fetch yet. Some stages cross a service boundary: they publish a full payload to the other service's queue and consume a full payload back, because neither service can read the other's storage. The `validate` and `land` stages adapt SubmitQueue's land work to Runway's `MergeRequest` contract and consume `MergeResult` on `landconflictsignal` / `landsignal`. See the queue-payload-boundary rule in [AGENTS.md](../../../AGENTS.md). -The pipeline has two cycles: `speculate → build → buildsignal → speculate` (CI feedback loop) and `merge → runway → mergesignal → speculate` (land the batch out of process, then advance the next). `conclude` is the only stage that transitions a request to a terminal state; `log` is an append-only sink that any controller can publish to via `submitqueue/core/request.PublishLog`. +The pipeline has two cycles: `speculate → build → buildsignal → speculate` (CI feedback loop) and `land → runway → landsignal → speculate` (land the batch out of process, then advance the next). `conclude` is the only stage that transitions a request to a terminal state; `log` is an append-only sink that any controller can publish to via `submitqueue/core/request.PublishLog`. ## Diagram @@ -24,18 +24,18 @@ The pipeline has two cycles: `speculate → build → buildsignal → speculate` | | Dedup, fetch metadata, publish | | | check request to runway | | +----------------+-----------------+ - | MergeRequest - | v - | #################################### - | # runway (separate service) # - | # Dry-run merge, emit result # - | ####################+############### - | MergeResult - | v - | +----------------------------------+ - | | mergeconflictsignal | - | | Correlate result, gate request | - | +----------------+-----------------+ + | MergeRequest + | v + | #################################### + | # runway (separate service) # + | # Dry-run merge, emit result # + | ####################+############### + | MergeResult + | v + | +----------------------------------+ + | | landconflictsignal | + | | Correlate result, gate request | + | +----------------+-----------------+ | | RequestID | v | +----------------------------------+ @@ -50,20 +50,20 @@ The pipeline has two cycles: `speculate → build → buildsignal → speculate` | | +------+-----------------+---------+ | | | BatchID | | BatchID | | | v v | - | | +------------------+ +------------------+ | - | | | build | | merge | | - | | | Trigger CI build | | Publish to runway| | - | | +--------+---------+ +--------+---------+ | - | | Build | MergeRequest | - | | v v | - | | +------------------+ #################### | - | +--| buildsignal | # runway (sep.) # | - | BatchID | Feed CI result | # Merge, emit res. # | - | | back to spec. | ########+########### | - | +------------------+ MergeResult | - | ^ v | - | Build (ext.CI) | +------------------+ | - | | | mergesignal |--+ + | | +------------------+ +------------------+ | + | | | build | | land | | + | | | Trigger CI build | | Publish to runway| | + | | +--------+---------+ +--------+---------+ | + | | Build | MergeRequest | + | | v v | + | | +------------------+ #################### | + | +--| buildsignal | # runway (sep.) # | + | BatchID | Feed CI result | # Merge, emit res. # | + | | back to spec. | ########+########### | + | +------------------+ MergeResult | + | ^ v | + | Build (ext.CI) | +------------------+ | + | | | landsignal |--+ | | | Gate batch + fan | | | | +--------+---------+ | | | | BatchID | @@ -82,14 +82,14 @@ The pipeline has two cycles: `speculate → build → buildsignal → speculate` |---|---|---|---| | **gateway/Land** | RPC | start | Accept request, mint ID, log Accepted, hand off async | | **start** | LandRequest | validate, log | Persist Request and emit Started log | -| **validate** | RequestID | merge-conflict-check (runway) | Dedup, fetch change metadata, claim changes, then publish the full check request to runway (keyed by the request id, the correlation id) | -| **mergeconflictsignal** | MergeResult | batch | Correlate runway's result; advance if mergeable, fail if conflicted | +| **validate** | RequestID | merge-conflict-check (Runway) | Dedup, fetch change metadata, claim changes, then adapt and publish the full `MergeRequest` to Runway (keyed by the request id, the correlation id) | +| **landconflictsignal** | MergeResult | batch | Correlate Runway's result; advance if landable, fail if conflicted | | **batch** | RequestID | speculate | Group request into a Batch with dependencies | -| **speculate** | BatchID | build, merge | (stub) Decide whether to verify via CI or land | +| **speculate** | BatchID | build, land | (stub) Decide whether to verify via CI or land | | **build** | BatchID | buildsignal | Trigger CI build for the batch | | **buildsignal** | Build | speculate | Feed CI result back into speculation | -| **merge** | BatchID | merge (runway) | Build the full merge request from the batch's member requests and publish to runway, keyed by the batch id (the correlation id) | -| **mergesignal** | MergeResult | conclude, speculate | Correlate runway's result; mark the batch Succeeded/Failed and fan out | +| **land** | BatchID | runway-merge (Runway) | Build the full land request from the batch's member requests, adapt it to `MergeRequest`, and publish to Runway keyed by the batch id (the correlation id) | +| **landsignal** | MergeResult | conclude, speculate | Correlate Runway's result; mark the batch Succeeded/Failed and fan out | | **conclude** | BatchID | — | Map terminal batch state to request state | | **log** | RequestLog | — | Gateway-owned sink: persists request log events to storage | @@ -97,7 +97,7 @@ The pipeline has two cycles: `speculate → build → buildsignal → speculate` Every *consumed* primary pipeline topic above is paired with a `{topic}_dlq` subscription consumed by a dedicated DLQ controller. The `log` topic is the exception: the orchestrator only publishes to it (the gateway is the sole consumer that persists the request log), so it has no orchestrator-side subscription and therefore no DLQ. The consumer framework moves a message to its DLQ once the primary controller returns a non-retryable error or exhausts retries on a retryable one; without the DLQ side the affected request would stay in a non-terminal state forever and the gateway would still report it as "in progress". -The DLQ controllers do not re-attempt the failed work. They decode the payload to recover the affected request (`RequestID`) or batch (`BatchID`) and drive the entity to a terminal failed state — `RequestStateError` for requests, `BatchStateFailed` for batches, with fan-out to the member requests. A DLQ whose topic carries a full payload rather than a bare ID recovers the id from that payload instead — the `mergeconflictsignal` and `mergesignal` DLQs read it from the runway `MergeResult` the producer echoed back. State writes use the same optimistic-locking CAS as the primary pipeline, so a late primary-pipeline update wins cleanly and a version mismatch is asked back for redelivery. +The DLQ controllers do not re-attempt the failed work. They decode the payload to recover the affected request (`RequestID`) or batch (`BatchID`) and drive the entity to a terminal failed state — `RequestStateError` for requests, `BatchStateFailed` for batches, with fan-out to the member requests. A DLQ whose topic carries a full payload rather than a bare ID recovers the id from that payload instead — the `landconflictsignal` and `landsignal` DLQs read it from the Runway `MergeResult` the producer echoed back. State writes use the same optimistic-locking CAS as the primary pipeline, so a late primary-pipeline update wins cleanly and a version mismatch is asked back for redelivery. DLQ consumers are wired with `errs.AlwaysRetryableProcessor` and a very high `Retry.MaxAttempts`, with their own DLQ disabled. That combination makes reconciliation effectively non-droppable: any failure is forced retryable rather than escalating to a second-level dead-letter that nobody consumes. The trade-off is that a genuinely unprocessable DLQ message — typically a malformed payload — must be removed by an operator. diff --git a/platform/errs/README.md b/platform/errs/README.md index b86c3b104..95bc4d40a 100644 --- a/platform/errs/README.md +++ b/platform/errs/README.md @@ -178,7 +178,7 @@ In particular, **do not reach for `NewRetryableError` just because replaying the ## Extensions Return Go Errors -Extension interfaces (`MergeChecker`, `Storage`, `Publisher`) return `error` values and may define domain-specific sentinels. Most sentinels remain unclassified because their meaning depends on the call site; for example, `storage.ErrNotFound` might be a user error in one controller and an infrastructure error in another. A sentinel whose classification is intrinsic in every context may carry that classification at its declaration. `storage.ErrVersionMismatch`, for example, is always a retryable infrastructure error because it reports a lost optimistic-concurrency race. +Extension interfaces (`ChangeProvider`, `Storage`, `Publisher`) return `error` values and may define domain-specific sentinels. Most sentinels remain unclassified because their meaning depends on the call site; for example, `storage.ErrNotFound` might be a user error in one controller and an infrastructure error in another. A sentinel whose classification is intrinsic in every context may carry that classification at its declaration. `storage.ErrVersionMismatch`, for example, is always a retryable infrastructure error because it reports a lost optimistic-concurrency race. Controllers should return intrinsically classified sentinels without adding another framework wrapper. The declaration remains reusable across implementations while every caller observes the same classification. diff --git a/platform/extension/messagequeue/README.md b/platform/extension/messagequeue/README.md index ab3b30837..3a143f2e7 100644 --- a/platform/extension/messagequeue/README.md +++ b/platform/extension/messagequeue/README.md @@ -100,7 +100,7 @@ for delivery := range deliveries { A message ID is the deduplication key, scoped to its topic and partition key. A backend matches a publish against messages it still holds — including ones already consumed, since reclamation is lazy and may lag delivery by an unbounded interval — and a collision is reported to the publisher as a success that stored nothing. There is no error to retry and no row to deliver. -The ID therefore names the *occasion* to publish, not the entity published about. An entity's own ID buys exactly one message for that entity for as long as the backend remembers the first, so a stage that announces a batch at creation and another that wakes it after a merge would collide, and the wake-up would vanish. +The ID therefore names the *occasion* to publish, not the entity published about. An entity's own ID buys exactly one message for that entity for as long as the backend remembers the first, so a stage that announces a batch at creation and another that wakes it after a land would collide, and the wake-up would vanish. Producers do not choose IDs by hand. They publish through `platform/publish`, whose `IntentID(entityID, cause...)` composes the entity with the cause of this particular message: a retry of the same cause dedups, which is what makes redelivery safe, while a new cause about the same entity can never be swallowed. `UniqueID` is the fallback for a cause with nothing stable to name it by, and it trades that idempotency for guaranteed delivery. diff --git a/platform/extension/messagequeue/mysql/README.md b/platform/extension/messagequeue/mysql/README.md index c9fa8c69a..914665e64 100644 --- a/platform/extension/messagequeue/mysql/README.md +++ b/platform/extension/messagequeue/mysql/README.md @@ -26,11 +26,11 @@ defer q.Close() // Publish msg := entityqueue.NewMessage("msg-id", []byte(`{"data": "value"}`), "repo-123", nil) -q.Publisher().Publish(ctx, "merge_events", msg) +q.Publisher().Publish(ctx, "land_events", msg) // Subscribe with per-subscription config subConfig := extqueue.DefaultSubscriptionConfig("worker-1", "orchestrator") -deliveryCh, _ := q.Subscriber().Subscribe(ctx, "merge_events", subConfig) +deliveryCh, _ := q.Subscriber().Subscribe(ctx, "land_events", subConfig) for delivery := range deliveryCh { if err := process(delivery.Message()); err != nil { delivery.Nack(ctx) // Retry diff --git a/platform/extension/messagequeue/mysql/ctl/README.md b/platform/extension/messagequeue/mysql/ctl/README.md index e6cb64c2c..8cbbf1780 100644 --- a/platform/extension/messagequeue/mysql/ctl/README.md +++ b/platform/extension/messagequeue/mysql/ctl/README.md @@ -25,14 +25,14 @@ Via Make (uses Bazel): ```bash make run-queue-admin ARGS="list-topics" -make run-queue-admin ARGS="topic-stats --topic merge_queue" +make run-queue-admin ARGS="topic-stats --topic land_queue" ``` Via Bazel directly: ```bash bazel run //platform/extension/messagequeue/mysql/ctl -- list-topics -bazel run //platform/extension/messagequeue/mysql/ctl -- topic-stats --topic merge_queue +bazel run //platform/extension/messagequeue/mysql/ctl -- topic-stats --topic land_queue ``` ## Commands @@ -44,20 +44,20 @@ bazel run //platform/extension/messagequeue/mysql/ctl -- topic-stats --topic mer queue-admin list-topics # Detailed stats for a topic (total messages, DLQ count, partitions, consumer groups) -queue-admin topic-stats --topic merge_queue +queue-admin topic-stats --topic land_queue ``` ### Inspect Messages ```bash # List messages (default limit 50) -queue-admin list-messages --topic merge_queue +queue-admin list-messages --topic land_queue # Filter by partition, custom limit -queue-admin list-messages --topic merge_queue --partition uber/cadence --limit 10 +queue-admin list-messages --topic land_queue --partition uber/cadence --limit 10 # Full message details including payload and metadata -queue-admin inspect-message --topic merge_queue --message-id msg-123 +queue-admin inspect-message --topic land_queue --message-id msg-123 ``` ### Manage Messages @@ -66,13 +66,13 @@ Destructive commands prompt for confirmation by default. Use `--no-interactive` ```bash # Delete a single message -queue-admin delete-message --topic merge_queue --message-id msg-123 +queue-admin delete-message --topic land_queue --message-id msg-123 # Purge all messages from a topic -queue-admin purge-topic --topic merge_queue +queue-admin purge-topic --topic land_queue # Skip confirmation prompt (for scripting) -queue-admin purge-topic --topic merge_queue --no-interactive +queue-admin purge-topic --topic land_queue --no-interactive ``` ### Dead Letter Queue (DLQ) @@ -81,26 +81,26 @@ DLQ messages live in the same `queue_messages` table under `topic + "_dlq"` (def ```bash # List DLQ messages -queue-admin list-dlq --topic merge_queue +queue-admin list-dlq --topic land_queue # Inspect a DLQ message (use the DLQ topic name) -queue-admin inspect-message --topic merge_queue_dlq --message-id msg-456 +queue-admin inspect-message --topic land_queue_dlq --message-id msg-456 # Move a DLQ message back to the original topic -queue-admin requeue-dlq --topic merge_queue --message-id msg-456 +queue-admin requeue-dlq --topic land_queue --message-id msg-456 # Purge all DLQ messages -queue-admin purge-dlq --topic merge_queue +queue-admin purge-dlq --topic land_queue # Custom DLQ suffix (if not using default "_dlq") -queue-admin list-dlq --topic merge_queue --dlq-suffix _dead +queue-admin list-dlq --topic land_queue --dlq-suffix _dead ``` ### Consumer Lag ```bash # Per-partition lag for all consumer groups on a topic -queue-admin consumer-lag --topic merge_queue +queue-admin consumer-lag --topic land_queue ``` Output shows `ACKED` (last processed offset), `LATEST` (newest message offset), and `LAG` (unprocessed count) per partition per consumer group. @@ -115,10 +115,10 @@ queue-admin list-offsets queue-admin list-offsets --consumer-group orchestrator # Reset offset to 0 (reprocess all messages) -queue-admin reset-offset --consumer-group orchestrator --topic merge_queue --partition uber/cadence +queue-admin reset-offset --consumer-group orchestrator --topic land_queue --partition uber/cadence # Reset to a specific offset -queue-admin reset-offset --consumer-group orchestrator --topic merge_queue --partition uber/cadence --offset 42 +queue-admin reset-offset --consumer-group orchestrator --topic land_queue --partition uber/cadence --offset 42 ``` ### Partition Leases @@ -132,7 +132,7 @@ queue-admin stale-leases # default 60s threshold queue-admin stale-leases --threshold 30000 # 30s threshold # Force-release a stuck lease -queue-admin release-lease --consumer-group orchestrator --topic merge_queue --partition uber/cadence +queue-admin release-lease --consumer-group orchestrator --topic land_queue --partition uber/cadence ``` ### JSON Output @@ -141,6 +141,6 @@ Add `--json` to any read command for machine-readable output: ```bash queue-admin list-topics --json -queue-admin consumer-lag --topic merge_queue --json -queue-admin list-messages --topic merge_queue --json | jq '.[] | .ID' +queue-admin consumer-lag --topic land_queue --json +queue-admin list-messages --topic land_queue --json | jq '.[] | .ID' ``` diff --git a/platform/extension/messagequeue/mysql/schema/queue_messages.sql b/platform/extension/messagequeue/mysql/schema/queue_messages.sql index a3a655365..1bb5bf406 100644 --- a/platform/extension/messagequeue/mysql/schema/queue_messages.sql +++ b/platform/extension/messagequeue/mysql/schema/queue_messages.sql @@ -1,7 +1,7 @@ -- MESSAGES TABLE (Immutable Log) -- Single table for all topics. Partition key determines distribution across workers. -- Messages are append-only; per-consumer-group delivery tracking is in queue_delivery_state. --- Example: topic="merge_queue", partition_key="uber/cadence" +-- Example: topic="land_queue", partition_key="uber/cadence" CREATE TABLE IF NOT EXISTS queue_messages ( -- Auto-incrementing global offset for ordering diff --git a/platform/hook/README.md b/platform/hook/README.md index ec6a8a87d..12e1f5561 100644 --- a/platform/hook/README.md +++ b/platform/hook/README.md @@ -4,7 +4,7 @@ The consumer side of the hooks framework: the stage that turns hook events on a ## Why a stage at all -Side effects must never stall or fail the pipeline, and "fire and forget" must not mean lossy — a merge-failure comment that silently never posts is a support ticket. A durable queue between the two resolves the tension: the producer's obligation ends once the event is enqueued, which is fast and local, and everything after that gets real retry semantics and a dead-letter queue. +Side effects must never stall or fail the pipeline, and "fire and forget" must not mean lossy — a land-failure comment that silently never posts is a support ticket. A durable queue between the two resolves the tension: the producer's obligation ends once the event is enqueued, which is fast and local, and everything after that gets real retry semantics and a dead-letter queue. Calling hooks inline would give up both halves. It couples pipeline latency to whatever an integration talks to, and a crash between the state write and the call drops the notification with nothing to replay. diff --git a/platform/hook/dlq.go b/platform/hook/dlq.go index b34b1556a..f73e84fde 100644 --- a/platform/hook/dlq.go +++ b/platform/hook/dlq.go @@ -41,7 +41,7 @@ const reconcileOp = "reconcile" // // That is a deliberate step up from the log topic's DLQ, which warns and moves // on. Dropping an observability row costs a gap in a read model; dropping a -// merge-failure comment costs a support ticket, and nothing else in the system +// land-failure comment costs a support ticket, and nothing else in the system // will notice it is missing. type DLQController struct { logger *zap.SugaredLogger diff --git a/platform/metrics/README.md b/platform/metrics/README.md index b88cf6146..7a2426222 100644 --- a/platform/metrics/README.md +++ b/platform/metrics/README.md @@ -85,7 +85,7 @@ There is no default bucket set. The package exports four common sets: |-----|-------|---------| | `FastLatencyBuckets` | ~100µs – 5s | Fast in-process work such as scoring, cache lookups, and CPU-bound operations | | `StorageLatencyBuckets` | ~1ms – 1m | Storage and message-queue round trips such as database reads, writes, publishing, and consuming | -| `LongLatencyBuckets` | ~5ms – 4h | Long-running pipeline work and external calls such as builds, merges, pushes, and provider calls | +| `LongLatencyBuckets` | ~5ms – 4h | Long-running pipeline work and external calls such as builds, lands, pushes, and provider calls | | `ChangeAgeBuckets` | ~1m – 30d | Elapsed time measured from a source-control change's commit timestamp rather than from work this system started | Pass one of these sets or a custom `tally.DurationBuckets` to `Begin` or `NamedHistogram`. diff --git a/platform/metrics/metrics.go b/platform/metrics/metrics.go index 5bba9793f..aab5feb62 100644 --- a/platform/metrics/metrics.go +++ b/platform/metrics/metrics.go @@ -108,7 +108,7 @@ var ( } // LongLatencyBuckets suits long-running pipeline work and external calls - // (~5ms to hours): builds, merges, git pushes, and external provider calls. + // (~5ms to hours): builds, lands, git pushes, and external provider calls. LongLatencyBuckets = tally.DurationBuckets{ 5 * time.Millisecond, 10 * time.Millisecond, diff --git a/platform/publish/publish_test.go b/platform/publish/publish_test.go index 23ec08ae7..6dab1f773 100644 --- a/platform/publish/publish_test.go +++ b/platform/publish/publish_test.go @@ -143,8 +143,8 @@ func TestIntentID(t *testing.T) { { name: "single cause", entityID: "batch-1", - cause: []string{"merged"}, - want: "batch-1/merged", + cause: []string{"landed"}, + want: "batch-1/landed", }, { name: "multiple causes join in order", @@ -164,9 +164,9 @@ func TestIntentID(t *testing.T) { // The convention only works if the same cause is repeatable and a different // cause is distinguishable — the two properties every call site relies on. func TestIntentID_StableAcrossCallsAndDistinctPerCause(t *testing.T) { - assert.Equal(t, IntentID("batch-1", "merged"), IntentID("batch-1", "merged")) - assert.NotEqual(t, IntentID("batch-1", "merged"), IntentID("batch-1")) - assert.NotEqual(t, IntentID("batch-1", "merged"), IntentID("batch-1", "cancelling")) + assert.Equal(t, IntentID("batch-1", "landed"), IntentID("batch-1", "landed")) + assert.NotEqual(t, IntentID("batch-1", "landed"), IntentID("batch-1")) + assert.NotEqual(t, IntentID("batch-1", "landed"), IntentID("batch-1", "cancelling")) } func TestUniqueID(t *testing.T) { diff --git a/service/README.md b/service/README.md index 72c76bb46..0177846ba 100644 --- a/service/README.md +++ b/service/README.md @@ -6,14 +6,14 @@ Each domain has its own subdirectory with a dedicated README: - [`submitqueue/`](submitqueue/README.md) — the multi-service SubmitQueue domain (Gateway + Orchestrator). - [`stovepipe/`](stovepipe/README.md) — the single-service Stovepipe domain (ingest → process → build → buildsignal → record). -- [`runway/`](runway/README.md) — the single-service Runway landing service (consumes the merge queues). +- [`runway/`](runway/README.md) — the single-service Runway merge execution service. ## Services | Service | Port | Domain | RPCs | Backing stores | |---------|------|--------|------|----------------| | **SubmitQueue Gateway** | 8081 | `submitqueue` | `Ping`, `Land`, `Cancel`, `GetRequestSummaryByID`, `GetRequestSummaryByChangeURI`, `List`, `GetRequestHistoryByID`, `GetRequestHistoryByChangeURI` | MySQL app + queue | -| **SubmitQueue Orchestrator** | 8082 | `submitqueue` | `Ping` (+ consumes start, cancel, validate, merge-conflict-check-signal, batch, dependency-analysis, speculate, build, buildsignal, submitqueue-merge, merge-signal, conclude, submitqueue-hook, and paired DLQ topics) | MySQL app + queue | +| **SubmitQueue Orchestrator** | 8082 | `submitqueue` | `Ping` (+ consumes start, cancel, validate, merge-conflict-check-signal, batch, dependency-analysis, speculate, build, buildsignal, submitqueue-land, merge-signal, conclude, submitqueue-hook, and paired DLQ topics) | MySQL app + queue | | **Stovepipe** | 8083 | `stovepipe` | `Ping`, `Ingest` (+ consumes process, build, buildsignal, record, stovepipe-hook, and paired DLQ topics) | MySQL storage + queue | | **Runway** | 8086 | `runway` | `Ping` (+ consumes merge-conflict-check & runway-merge topics) | MySQL queue | diff --git a/service/submitqueue/README.md b/service/submitqueue/README.md index cc80c582b..9c94dee19 100644 --- a/service/submitqueue/README.md +++ b/service/submitqueue/README.md @@ -1,6 +1,6 @@ # SubmitQueue Services -Runnable wiring for the **SubmitQueue** domain's two services — the Gateway (entry point for land requests) and the Orchestrator (coordinates the pipeline) — wired with MySQL-backed extensions. The full Docker Compose workflow also starts Runway, which performs merge-conflict checks and merges. +Runnable wiring for the **SubmitQueue** domain's two services — the Gateway (entry point for land requests) and the Orchestrator (coordinates the pipeline) — wired with MySQL-backed extensions. The full Docker Compose workflow also starts Runway, which performs merge-conflict checks and merges on SubmitQueue's behalf. ## Starting diff --git a/service/submitqueue/demo/requests/main.go b/service/submitqueue/demo/requests/main.go index c1e3becfa..e978177b9 100644 --- a/service/submitqueue/demo/requests/main.go +++ b/service/submitqueue/demo/requests/main.go @@ -50,7 +50,7 @@ // // - fake (default): a change is a URI and nothing else. No repository, no // credential, no I/O — the fastest way to put traffic through the queue. -// - git: a branch pushed to the sandbox repository the stack merges into. +// - git: a branch pushed to the sandbox repository the stack lands into. // Real commits, still no credential. // - github: a real pull request over the REST API, which needs no clone and // no git binary, only GITHUB_TOKEN — the same credential the stack uses. @@ -130,7 +130,7 @@ func parseFlags() config { flag.BoolVar(&c.tls, "tls", false, "dial the gateway with transport security") flag.StringVar(&c.tokenEnv, "token-env", client.DefaultTokenEnv, "environment variable holding the gateway bearer token") flag.StringVar(&c.queue, "queue", "demo-queue", "queue to land on") - flag.StringVar(&c.strategy, "strategy", "SQUASH_REBASE", "merge strategy") + flag.StringVar(&c.strategy, "strategy", "SQUASH_REBASE", "land strategy") flag.Parse() // Only the GitHub source reads a credential; the other two must not fail, diff --git a/service/submitqueue/gateway/server/mapper/land.go b/service/submitqueue/gateway/server/mapper/land.go index 8c77d56fb..b33fc2c03 100644 --- a/service/submitqueue/gateway/server/mapper/land.go +++ b/service/submitqueue/gateway/server/mapper/land.go @@ -36,7 +36,7 @@ var errUnknownStrategy = errors.New("unknown land strategy in proto message") // ProtoToLandRequest maps the wire LandRequest to the entity.LandRequest the controller operates on. // The ID is left empty; the controller assigns it. func ProtoToLandRequest(req *pb.LandRequest) (entity.LandRequest, error) { - strategy, err := resolveMergeStrategy(req.GetStrategy()) + strategy, err := resolveLandStrategy(req.GetStrategy()) if err != nil { return entity.LandRequest{}, fmt.Errorf("failed to map land strategy: %w", err) } @@ -47,8 +47,8 @@ func ProtoToLandRequest(req *pb.LandRequest) (entity.LandRequest, error) { }, nil } -// resolveMergeStrategy maps a proto Strategy enum to the shared mergestrategy.MergeStrategy. -func resolveMergeStrategy(s mergestrategypb.Strategy) (mergestrategy.MergeStrategy, error) { +// resolveLandStrategy maps a proto Strategy enum to the shared mergestrategy.MergeStrategy. +func resolveLandStrategy(s mergestrategypb.Strategy) (mergestrategy.MergeStrategy, error) { switch s { case mergestrategypb.Strategy_DEFAULT: // TODO: resolve default strategy based on queue configuration diff --git a/service/submitqueue/gateway/server/mapper/land_test.go b/service/submitqueue/gateway/server/mapper/land_test.go index 348282856..f1e6ef253 100644 --- a/service/submitqueue/gateway/server/mapper/land_test.go +++ b/service/submitqueue/gateway/server/mapper/land_test.go @@ -83,7 +83,7 @@ func TestProtoToLandRequest(t *testing.T) { } } -func TestResolveMergeStrategy(t *testing.T) { +func TestResolveLandStrategy(t *testing.T) { tests := []struct { name string in mergestrategypb.Strategy @@ -100,7 +100,7 @@ func TestResolveMergeStrategy(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := resolveMergeStrategy(tt.in) + got, err := resolveLandStrategy(tt.in) if tt.errMsg != "" { assert.ErrorContains(t, err, tt.errMsg) return diff --git a/stovepipe/README.md b/stovepipe/README.md index 269e8388d..ec1a6c9cd 100644 --- a/stovepipe/README.md +++ b/stovepipe/README.md @@ -1,6 +1,6 @@ # Stovepipe -Stovepipe is a post-merge validation service. Its layout: +Stovepipe is a post-land validation service. Its layout: - `controller/` — business logic (transport-agnostic). Exposes the `Ping` and `Ingest` RPCs, and consumes the internal pipeline stages (`process`, `build`, `buildsignal`, `record`) plus a DLQ reconciler. diff --git a/submitqueue/core/batch/transition_test.go b/submitqueue/core/batch/transition_test.go index b99ff47ce..3bc814d42 100644 --- a/submitqueue/core/batch/transition_test.go +++ b/submitqueue/core/batch/transition_test.go @@ -147,7 +147,7 @@ func TestEnsureRecord(t *testing.T) { batch := entity.Batch{ ID: "monorepo/batch/7", Queue: "monorepo", - State: entity.BatchStateMerging, + State: entity.BatchStateLanding, Version: 5, } storeErr := errors.New("storage failed") diff --git a/submitqueue/core/changeset/README.md b/submitqueue/core/changeset/README.md index 4e8c9463c..fb32905f0 100644 --- a/submitqueue/core/changeset/README.md +++ b/submitqueue/core/changeset/README.md @@ -1,6 +1,6 @@ # changeset -`changeset` resolves batch identity into the changes a batch contains. Current consumers include build-runner and scorer implementations and the path-overlap conflict analyzer. The merge controller loads member requests directly because its Runway payload preserves one ordered merge step per request. +`changeset` resolves batch identity into the changes a batch contains. Current consumers include build-runner and scorer implementations and the path-overlap conflict analyzer. The land controller loads member requests directly because its Runway payload preserves one ordered merge step per request. ## Why it exists diff --git a/submitqueue/core/changeset/changeset.go b/submitqueue/core/changeset/changeset.go index 3edf21a8f..b3709d559 100644 --- a/submitqueue/core/changeset/changeset.go +++ b/submitqueue/core/changeset/changeset.go @@ -14,7 +14,7 @@ // Package changeset resolves batch identity into the changes a batch contains. // It is the single place the orchestrator walks batch -> requests -> changes, -// consolidating what the build and merge controllers each did privately. +// consolidating controller-side and backend-specific traversal. // Decision/action extensions (scorer, buildrunner, and future // detail-aware conflict analyzers) take thin identity entities and resolve their // granular content through an injected Resolver instead of being handed @@ -31,15 +31,15 @@ import ( ) // Resolver turns batch identity into the changes the batch contains. Both methods -// operate on a single batch — callers with several batches (a build's base, a -// merge train) loop and keep the per-batch boundary by holding a slice per batch. +// operate on a single batch — callers with several batches loop and keep the +// per-batch boundary by holding a slice per batch. // The two methods differ only in fidelity: ChangesForBatch is the cheap URI-only // view; DetailedForBatch reads the change store for provider details. type Resolver interface { // ChangesForBatch resolves a batch's contained requests into their raw // changes (URIs only; no change-store read), in batch.Contains order. A batch - // with no requests yields an empty slice. Used by the build (base/head) and - // merge stages. + // with no requests yields an empty slice. Used by build runners for their + // base and head batches. ChangesForBatch(ctx context.Context, batch entity.Batch) ([]change.Change, error) // DetailedForBatch resolves a batch into its normalized, batch-level view: diff --git a/submitqueue/core/request/request_test.go b/submitqueue/core/request/request_test.go index fe081ce4a..fd14736ca 100644 --- a/submitqueue/core/request/request_test.go +++ b/submitqueue/core/request/request_test.go @@ -54,9 +54,9 @@ func TestGetCurrentStateFromRequestLog(t *testing.T) { name: "terminal error status with last error", logs: []entity.RequestLog{ {RequestID: "q/1", TimestampMs: 1000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusStarted, RequestVersion: 1, LastError: "", Metadata: map[string]string{}}, - {RequestID: "q/1", TimestampMs: 2000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusError, RequestVersion: 4, LastError: "merge conflict", Metadata: map[string]string{"step": "merge"}}, + {RequestID: "q/1", TimestampMs: 2000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusError, RequestVersion: 4, LastError: "merge conflict", Metadata: map[string]string{"step": "land"}}, }, - expected: CurrentState{Status: entity.RequestStatusError, LastError: "merge conflict", Metadata: map[string]string{"step": "merge"}}, + expected: CurrentState{Status: entity.RequestStatusError, LastError: "merge conflict", Metadata: map[string]string{"step": "land"}}, }, { name: "multiple terminal records picks highest version", diff --git a/submitqueue/core/topickey/topickey.go b/submitqueue/core/topickey/topickey.go index fa31076fa..6bdae00f3 100644 --- a/submitqueue/core/topickey/topickey.go +++ b/submitqueue/core/topickey/topickey.go @@ -44,16 +44,16 @@ const ( // so the state machine re-evaluates, and holds the delivery for the next // poll when the build has not yet reached a terminal state. TopicKeyBuildSignal TopicKey = "buildsignal" - // TopicKeyMerge is the pipeline stage where speculated batches are published for merging. - TopicKeyMerge TopicKey = "submitqueue-merge" - // TopicKeyConclude is the pipeline stage where merged requests are published for conclusion. + // TopicKeyLand is the pipeline stage where speculated batches are published for landing. + TopicKeyLand TopicKey = "submitqueue-land" + // TopicKeyConclude is the pipeline stage where landed requests are published for conclusion. TopicKeyConclude TopicKey = "conclude" // TopicKeyLog is the pipeline stage where per-request logs are written. TopicKeyLog TopicKey = "log" ) // MetadataKeyFailureReason is the conclude message's metadata attribute carrying -// a failed batch's human-readable reason. Set by the failure sites (merge and +// a failed batch's human-readable reason. Set by the failure sites (land and // speculate) on the conclude publish and read by conclude to stamp the request's // terminal log; absent on the landed and cancelled paths. const MetadataKeyFailureReason = "failure_reason" diff --git a/submitqueue/entity/BUILD.bazel b/submitqueue/entity/BUILD.bazel index a2dc0a684..419a98a61 100644 --- a/submitqueue/entity/BUILD.bazel +++ b/submitqueue/entity/BUILD.bazel @@ -13,9 +13,7 @@ go_library( "conflict.go", "land.go", "list.go", - "merge_result.go", "path_build.go", - "push_result.go", "queue_batch_state.go", "queue_config.go", "request.go", diff --git a/submitqueue/entity/batch.go b/submitqueue/entity/batch.go index f9b280cb8..25f9d4af8 100644 --- a/submitqueue/entity/batch.go +++ b/submitqueue/entity/batch.go @@ -31,15 +31,15 @@ const ( BatchStateCreated BatchState = "created" // BatchStateSpeculating is the state of a batch that is undergoing speculative execution. BatchStateSpeculating BatchState = "speculating" - // BatchStateMerging is the state of a batch that is being merged after speculative execution. - BatchStateMerging BatchState = "merging" + // BatchStateLanding is the state of a batch that is being landed after speculative execution. + BatchStateLanding BatchState = "landing" // BatchStateSucceeded is the terminal state of a batch that has been successfully landed. BatchStateSucceeded BatchState = "succeeded" // BatchStateFailed is the terminal state of a batch that has failed. BatchStateFailed BatchState = "failed" // BatchStateCancelling is the non-terminal intent state set when a cancel has been requested but the // batch has not yet been transitioned to BatchStateCancelled. A batch in this state may still reach - // BatchStateSucceeded or BatchStateFailed if a concurrent merge wins the race (e.g. the push had + // BatchStateSucceeded or BatchStateFailed if a concurrent land wins the race (e.g. the push had // already completed before the cancel CAS observed the batch); those terminal states prevail. // Forward-progress controllers must treat this state as halted (no new work). The speculate // controller owns the transition to the terminal BatchStateCancelled and the downstream fan-out @@ -67,7 +67,7 @@ func (s BatchState) IsTerminal() bool { var nonCancellableBatchStates = map[BatchState]bool{ BatchStateUnknown: true, BatchStateCreating: true, - BatchStateMerging: true, + BatchStateLanding: true, BatchStateSucceeded: true, BatchStateFailed: true, BatchStateCancelled: true, @@ -80,7 +80,7 @@ func (s BatchState) IsCancellable() bool { } // IsBatchStateHalted returns true if the batch is either terminal or in the process of being cancelled. -// Forward-progress controllers (build, buildsignal, speculate, merge) use this to short-circuit +// Forward-progress controllers (build, buildsignal, speculate, land) use this to short-circuit // work for batches that the user has asked to cancel — even though Cancelling is non-terminal, no // further pipeline work should start (cancel will write the terminal state and fan out). func IsBatchStateHalted(s BatchState) bool { @@ -94,7 +94,7 @@ func AllBatchStates() []BatchState { BatchStateCreating, BatchStateCreated, BatchStateSpeculating, - BatchStateMerging, + BatchStateLanding, BatchStateSucceeded, BatchStateFailed, BatchStateCancelling, @@ -108,7 +108,7 @@ func ActiveBatchStates() []BatchState { return []BatchState{ BatchStateCreated, BatchStateSpeculating, - BatchStateMerging, + BatchStateLanding, BatchStateCancelling, } } @@ -127,11 +127,11 @@ func DependencyBatchStates() []BatchState { return []BatchState{ BatchStateCreated, BatchStateSpeculating, - BatchStateMerging, + BatchStateLanding, } } -// Batch represents a group of requests to land (merge into target branch of the source control repository). +// Batch represents a group of requests to land on the source control repository's target branch. type Batch struct { // ID is the globally unique identifier for the batch. Format: "/batch/". ID string diff --git a/submitqueue/entity/batch_test.go b/submitqueue/entity/batch_test.go index 558d95141..0d1496d14 100644 --- a/submitqueue/entity/batch_test.go +++ b/submitqueue/entity/batch_test.go @@ -31,7 +31,7 @@ func TestBatchState_IsTerminal(t *testing.T) { {name: "creating", state: BatchStateCreating, terminal: false}, {name: "created", state: BatchStateCreated, terminal: false}, {name: "speculating", state: BatchStateSpeculating, terminal: false}, - {name: "merging", state: BatchStateMerging, terminal: false}, + {name: "landing", state: BatchStateLanding, terminal: false}, {name: "succeeded", state: BatchStateSucceeded, terminal: true}, {name: "failed", state: BatchStateFailed, terminal: true}, {name: "cancelled", state: BatchStateCancelled, terminal: true}, @@ -52,7 +52,7 @@ func TestIsCancellable(t *testing.T) { assert.True(t, BatchState("future").IsCancellable()) assert.False(t, BatchStateUnknown.IsCancellable()) assert.False(t, BatchStateCreating.IsCancellable()) - assert.False(t, BatchStateMerging.IsCancellable()) + assert.False(t, BatchStateLanding.IsCancellable()) assert.False(t, BatchStateSucceeded.IsCancellable()) assert.False(t, BatchStateFailed.IsCancellable()) assert.False(t, BatchStateCancelled.IsCancellable()) diff --git a/submitqueue/entity/merge_result.go b/submitqueue/entity/merge_result.go deleted file mode 100644 index ad5078201..000000000 --- a/submitqueue/entity/merge_result.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package entity - -// MergeResult holds the outcome of a mergeability check. -type MergeResult struct { - // Mergeable is true if the request's changes are expected to merge cleanly. - Mergeable bool - // Reason is a human-readable explanation when Mergeable is false. - // Empty when Mergeable is true. - Reason string -} diff --git a/submitqueue/entity/push_result.go b/submitqueue/entity/push_result.go deleted file mode 100644 index 0b1352c0e..000000000 --- a/submitqueue/entity/push_result.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package entity - -import "github.com/uber/submitqueue/platform/base/change" - -// OutcomeStatus describes what happened to a single Change during a push. -type OutcomeStatus string - -const ( - // OutcomeStatusUnknown is the unreachable zero value, set by default - // when the structure is initialized. It should never be seen in the system. - OutcomeStatusUnknown OutcomeStatus = "" - // OutcomeStatusCommitted means the change produced one or more commits - // on the target branch. CommitSHAs lists those commits in apply order. - OutcomeStatusCommitted OutcomeStatus = "committed" - // OutcomeStatusAlreadyExisted means the change produced no commits - // because every part of it is already present in the target branch - // (e.g. it previously landed via another path, or a prior change in - // the same push subsumed it). CommitSHAs is empty for this status. - // In git terms this is what a `cherry-pick` surfaces as "rebased out". - OutcomeStatusAlreadyExisted OutcomeStatus = "already_existed" -) - -// ChangeOutcome describes what happened to a single Change inside a push. -type ChangeOutcome struct { - // Change is the input change this outcome corresponds to. - Change change.Change - // Status describes whether the change produced commits or was already - // present on the target branch. - Status OutcomeStatus - // CommitSHAs lists the commits this change produced on the target - // branch, in apply order. A single Change may produce multiple commits - // (e.g. a stack of PRs). Empty when Status is OutcomeStatusAlreadyExisted. - CommitSHAs []string -} - -// BatchOutcome groups the per-change outcomes for a single pushed batch, so a -// merge-train push (several batches in one call) stays correlatable back to the -// batch each change belonged to. There is no per-batch status: a push is -// all-or-nothing across the whole call, so a per-batch pass/fail would be -// uniformly redundant. -type BatchOutcome struct { - // BatchID is the input batch this outcome corresponds to. - BatchID string - // Outcomes is one entry per change in the batch, in apply order. - Outcomes []ChangeOutcome -} - -// PushResult is the outcome of a successful push. -type PushResult struct { - // Batches is one entry per pushed batch, in the same order as the batches - // passed to the push. The slice length equals the input length. - Batches []BatchOutcome -} diff --git a/submitqueue/entity/request.go b/submitqueue/entity/request.go index 83bd30a5d..5d970d25d 100644 --- a/submitqueue/entity/request.go +++ b/submitqueue/entity/request.go @@ -29,7 +29,7 @@ const ( RequestStateUnknown RequestState = "" // RequestStateStarted is the initial state of a land request. It is confirmed by the system but the processing is not started yet. RequestStateStarted RequestState = "started" - // RequestStateValidated indicates that the request has been validated (duplicate check, merge check etc.) successfully. + // RequestStateValidated indicates that the request has been validated (duplicate check, landability check, etc.) successfully. RequestStateValidated RequestState = "validated" // RequestStateBatched indicates that the request is enrolled in a batch whose dependencies have been // resolved. The CAS-write of this state is the serialization point against cancellation: it lands with @@ -45,7 +45,7 @@ const ( RequestStateError RequestState = "error" // RequestStateCancelling is the non-terminal intent state set when the user has requested cancellation but the // request has not yet been transitioned to RequestStateCancelled. A request in this state may still reach - // RequestStateLanded or RequestStateError if a concurrent merge or failure wins the race; those terminal + // RequestStateLanded or RequestStateError if a concurrent land or failure wins the race; those terminal // states prevail. Forward-progress controllers must treat this state the same as terminal (i.e. do not start // any new work on the request). RequestStateCancelling RequestState = "cancelling" @@ -68,7 +68,7 @@ func IsRequestStateHalted(s RequestState) bool { return IsRequestStateTerminal(s) || s == RequestStateCancelling } -// Request defines a request to land (merge into target branch of the source control repository) a set of code changes. +// Request defines a request to land a set of code changes on the source control repository's target branch. // The object is immutable after creation. type Request struct { // **************** diff --git a/submitqueue/entity/request_log.go b/submitqueue/entity/request_log.go index 72bad9ef6..a1be20fc3 100644 --- a/submitqueue/entity/request_log.go +++ b/submitqueue/entity/request_log.go @@ -45,10 +45,10 @@ const ( // RequestStatusStarted is the initial status of a request. It corresponds to the RequestStateStarted state and typically set by the orchestrator service when the request is received and persisted to the operating database. RequestStatusStarted RequestStatus = "started" - // RequestStatusValidating indicates that the request is currently being validated (e.g., duplicate check, merge check, etc.). + // RequestStatusValidating indicates that the request is currently being validated (e.g., duplicate check, landability check, etc.). RequestStatusValidating RequestStatus = "validating" - // RequestStatusValidated indicates that the request has been validated (duplicate check, merge check etc.) successfully. It corresponds to the RequestStateValidated state. + // RequestStatusValidated indicates that the request has been validated (duplicate check, landability check, etc.) successfully. It corresponds to the RequestStateValidated state. RequestStatusValidated RequestStatus = "validated" // RequestStatusBatching indicates that a batch has been created for the request and is resolving what it must serialize behind. @@ -75,7 +75,7 @@ const ( RequestStatusError RequestStatus = "error" // RequestStatusCancelling indicates that the user has requested cancellation but the request has not yet transitioned - // to the RequestStateCancelled state. Cancellation is best-effort: a request that has already been merged or that + // to the RequestStateCancelled state. Cancellation is best-effort: a request that has already landed or that // races to completion before the cancel propagates through the pipeline may still land. Observers should treat this // as intent only and rely on RequestStatusCancelled (or RequestStatusLanded) for the terminal outcome. Emitted by // the gateway when the Cancel RPC is received. diff --git a/submitqueue/entity/speculation.go b/submitqueue/entity/speculation.go index 1d199c902..734dfe1f6 100644 --- a/submitqueue/entity/speculation.go +++ b/submitqueue/entity/speculation.go @@ -185,7 +185,7 @@ type SpeculationPathSet struct { } // PathAction is an action proposed on a speculation path. The set is limited to -// build and cancel; there is no merge or fail action, because a batch's verdict +// build and cancel; there is no land or fail action, because a batch's verdict // is a controller-owned fact, not a proposed action. type PathAction string diff --git a/submitqueue/extension/mergechecker/BUILD.bazel b/submitqueue/extension/mergechecker/BUILD.bazel deleted file mode 100644 index 23ecace72..000000000 --- a/submitqueue/extension/mergechecker/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library") - -go_library( - name = "go_default_library", - srcs = ["mergechecker.go"], - importpath = "github.com/uber/submitqueue/submitqueue/extension/mergechecker", - visibility = ["//visibility:public"], - deps = ["//submitqueue/entity:go_default_library"], -) diff --git a/submitqueue/extension/mergechecker/fake/BUILD.bazel b/submitqueue/extension/mergechecker/fake/BUILD.bazel deleted file mode 100644 index d1c85e0ce..000000000 --- a/submitqueue/extension/mergechecker/fake/BUILD.bazel +++ /dev/null @@ -1,26 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = ["fake.go"], - importpath = "github.com/uber/submitqueue/submitqueue/extension/mergechecker/fake", - visibility = ["//visibility:public"], - deps = [ - "//platform/fakemarker:go_default_library", - "//submitqueue/entity:go_default_library", - "//submitqueue/extension/mergechecker:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = ["fake_test.go"], - embed = [":go_default_library"], - deps = [ - "//platform/base/change:go_default_library", - "//submitqueue/entity:go_default_library", - "//submitqueue/extension/mergechecker:go_default_library", - "@com_github_stretchr_testify//assert:go_default_library", - "@com_github_stretchr_testify//require:go_default_library", - ], -) diff --git a/submitqueue/extension/mergechecker/fake/fake.go b/submitqueue/extension/mergechecker/fake/fake.go deleted file mode 100644 index effffc5dd..000000000 --- a/submitqueue/extension/mergechecker/fake/fake.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package fake provides a mergechecker.MergeChecker whose outcome is driven by -// the input change. With no marker it reports every change as mergeable, -// behaving as a best-case stub for wiring and baselines. A failure can be -// injected end-to-end (e.g. from an e2e land request) by embedding a marker -// token in a change URI of the form "sq-fake=": -// -// sq-fake=unmergeable -> Result{Mergeable: false} -// sq-fake=mergecheck-error -> non-nil error -// -// This lets a single running stack exercise negative paths purely by varying -// request payloads. It is intended for examples and tests only, never -// production. -package fake - -import ( - "context" - "fmt" - - "github.com/uber/submitqueue/platform/fakemarker" - "github.com/uber/submitqueue/submitqueue/entity" - "github.com/uber/submitqueue/submitqueue/extension/mergechecker" -) - -// Recognized marker tokens. See the package doc for the convention. -const ( - tokenUnmergeable = "unmergeable" - tokenError = "mergecheck-error" -) - -// checker is a mergechecker.MergeChecker that reports changes as mergeable -// unless a marker token in a change URI requests otherwise. -type checker struct { - // cfg is the per-queue identity this checker was built for. - cfg mergechecker.Config -} - -// New returns a mergechecker.MergeChecker bound to the queue named in cfg that -// defaults to mergeable and honors marker tokens embedded in change URIs. -func New(cfg mergechecker.Config) mergechecker.MergeChecker { - return checker{cfg: cfg} -} - -// Check reports the change as mergeable unless a recognized marker token is -// present in one of the request's change URIs. -func (checker) Check(_ context.Context, request entity.Request) (entity.MergeResult, error) { - switch fakemarker.Token(request.Change.URIs) { - case tokenUnmergeable: - return entity.MergeResult{Mergeable: false, Reason: "fake: marked unmergeable"}, nil - case tokenError: - return entity.MergeResult{}, fmt.Errorf("fake: marked merge-check error") - default: - return entity.MergeResult{Mergeable: true}, nil - } -} diff --git a/submitqueue/extension/mergechecker/fake/fake_test.go b/submitqueue/extension/mergechecker/fake/fake_test.go deleted file mode 100644 index 5fb833d41..000000000 --- a/submitqueue/extension/mergechecker/fake/fake_test.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fake - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/uber/submitqueue/platform/base/change" - "github.com/uber/submitqueue/submitqueue/entity" - "github.com/uber/submitqueue/submitqueue/extension/mergechecker" -) - -// testCfg is the per-queue identity used by every case in this file. -var testCfg = mergechecker.Config{QueueName: "test-queue"} - -func TestNew_ImplementsInterface(t *testing.T) { - var _ mergechecker.MergeChecker = New(testCfg) -} - -func TestChecker_Check(t *testing.T) { - tests := []struct { - name string - uris []string - wantMergeable bool - wantErr bool - }{ - { - name: "no marker is mergeable", - uris: []string{"github://github.example.com/owner/repo/pull/1/abc"}, - wantMergeable: true, - }, - { - name: "no URIs is mergeable", - uris: nil, - wantMergeable: true, - }, - { - name: "unmergeable marker", - uris: []string{"github://github.example.com/owner/repo/pull/1/abc?sq-fake=unmergeable"}, - }, - { - name: "error marker", - uris: []string{"github://github.example.com/owner/repo/pull/1/abc?sq-fake=mergecheck-error"}, - wantErr: true, - }, - { - name: "marker on second uri", - uris: []string{ - "github://github.example.com/owner/repo/pull/1/abc", - "github://github.example.com/owner/repo/pull/2/def?sq-fake=unmergeable", - }, - }, - } - - c := New(testCfg) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - res, err := c.Check(context.Background(), entity.Request{Change: change.Change{URIs: tt.uris}}) - if tt.wantErr { - require.Error(t, err) - return - } - require.NoError(t, err) - assert.Equal(t, tt.wantMergeable, res.Mergeable) - }) - } -} diff --git a/submitqueue/extension/mergechecker/github/BUILD.bazel b/submitqueue/extension/mergechecker/github/BUILD.bazel deleted file mode 100644 index faf87e605..000000000 --- a/submitqueue/extension/mergechecker/github/BUILD.bazel +++ /dev/null @@ -1,41 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "checker.go", - "graphql.go", - "validate.go", - ], - importpath = "github.com/uber/submitqueue/submitqueue/extension/mergechecker/github", - visibility = ["//visibility:public"], - deps = [ - "//platform/base/change/github:go_default_library", - "//platform/metrics:go_default_library", - "//submitqueue/entity:go_default_library", - "//submitqueue/extension/mergechecker:go_default_library", - "@com_github_uber_go_tally//:go_default_library", - "@org_uber_go_zap//:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = [ - "checker_test.go", - "graphql_test.go", - "validate_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//platform/base/change:go_default_library", - "//platform/base/change/github:go_default_library", - "//platform/http:go_default_library", - "//submitqueue/entity:go_default_library", - "//submitqueue/extension/mergechecker:go_default_library", - "@com_github_stretchr_testify//assert:go_default_library", - "@com_github_stretchr_testify//require:go_default_library", - "@com_github_uber_go_tally//:go_default_library", - "@org_uber_go_zap//zaptest:go_default_library", - ], -) diff --git a/submitqueue/extension/mergechecker/github/checker.go b/submitqueue/extension/mergechecker/github/checker.go deleted file mode 100644 index 7d01c1063..000000000 --- a/submitqueue/extension/mergechecker/github/checker.go +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package github - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - - "github.com/uber-go/tally" - entitygithub "github.com/uber/submitqueue/platform/base/change/github" - "github.com/uber/submitqueue/platform/metrics" - "github.com/uber/submitqueue/submitqueue/entity" - "github.com/uber/submitqueue/submitqueue/extension/mergechecker" - "go.uber.org/zap" -) - -// Params holds the dependencies for the GitHub MergeChecker. -type Params struct { - // Config is the per-queue identity handed to the Factory that built this - // merge checker. - Config mergechecker.Config - // HTTPClient is a pre-configured HTTP client. The caller is responsible for - // configuring the base URL (via BaseURLTransport) and auth (via a transport layer). - HTTPClient *http.Client - // Logger is the structured logger. - Logger *zap.SugaredLogger - // MetricsScope is the metrics scope for instrumentation. - MetricsScope tally.Scope -} - -// mergeChecker implements the mergechecker.MergeChecker interface using the GitHub GraphQL API. -type mergeChecker struct { - // cfg is the per-queue identity this checker was built for. - cfg mergechecker.Config - httpClient *http.Client - logger *zap.SugaredLogger - metricsScope tally.Scope -} - -// Verify mergeChecker implements mergechecker.MergeChecker at compile time. -var _ mergechecker.MergeChecker = (*mergeChecker)(nil) - -// NewMergeChecker creates a new GitHub MergeChecker bound to the queue named in -// params.Config. -func NewMergeChecker(params Params) mergechecker.MergeChecker { - return &mergeChecker{ - cfg: params.Config, - httpClient: params.HTTPClient, - logger: params.Logger.Named("github_mergechecker"), - metricsScope: params.MetricsScope.SubScope("github_mergechecker"), - } -} - -// Check assesses whether a request's change can merge cleanly using the GitHub GraphQL API. -func (c *mergeChecker) Check(ctx context.Context, request entity.Request) (result entity.MergeResult, retErr error) { - const opName = "check" - - op := metrics.Begin(c.metricsScope, opName, metrics.LongLatencyBuckets) - defer func() { op.Complete(retErr) }() - - change := request.Change - - // Parse all change IDs - // TODO: classify parse errors as user errors (non-retryable) vs system errors. - changeIDs := make([]entitygithub.ChangeID, 0, len(change.URIs)) - for _, rawID := range change.URIs { - cid, err := entitygithub.ParseChangeID(rawID) - if err != nil { - metrics.NamedCounter(c.metricsScope, opName, "parse_errors", 1) - return result, fmt.Errorf("failed to parse change ID %q: %w", rawID, err) - } - changeIDs = append(changeIDs, cid) - } - - // Fetch PR info from GitHub GraphQL API - prInfoMap, err := c.fetchPRInfo(ctx, changeIDs) - if err != nil { - metrics.NamedCounter(c.metricsScope, opName, "graphql_errors", 1) - return result, fmt.Errorf("failed to fetch PR info: %w", err) - } - - // Validate PR mergeability - mergeable, reason, err := validatePRs(changeIDs, prInfoMap) - if err != nil { - metrics.NamedCounter(c.metricsScope, opName, "validation_errors", 1) - return result, err - } - - if !mergeable { - metrics.NamedCounter(c.metricsScope, opName, "not_mergeable", 1) - c.logger.Infow("change not mergeable", - "reason", reason, - "change_uris", change.URIs, - ) - } else { - metrics.NamedCounter(c.metricsScope, opName, "mergeable", 1) - } - - result.Mergeable = mergeable - result.Reason = reason - return result, nil -} - -// fetchPRInfo executes a batched GraphQL query to fetch PR info for all change IDs. -func (c *mergeChecker) fetchPRInfo(ctx context.Context, changeIDs []entitygithub.ChangeID) (map[int]PRInfo, error) { - query := buildGraphQLQuery(changeIDs) - - reqBody, err := json.Marshal(graphQLRequest{Query: query}) - if err != nil { - return nil, fmt.Errorf("failed to marshal graphql request: %w", err) - } - - httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, "/graphql", bytes.NewReader(reqBody)) - if err != nil { - return nil, fmt.Errorf("failed to create http request: %w", err) - } - httpReq.Header.Set("Content-Type", "application/json") - - resp, err := c.httpClient.Do(httpReq) - if err != nil { - return nil, fmt.Errorf("graphql request failed: %w", err) - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read graphql response: %w", err) - } - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("graphql request returned status %d: %s", resp.StatusCode, string(body)) - } - - return parseGraphQLResponse(body, changeIDs) -} diff --git a/submitqueue/extension/mergechecker/github/checker_test.go b/submitqueue/extension/mergechecker/github/checker_test.go deleted file mode 100644 index e38f1b389..000000000 --- a/submitqueue/extension/mergechecker/github/checker_test.go +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package github - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/uber-go/tally" - "github.com/uber/submitqueue/platform/base/change" - phttp "github.com/uber/submitqueue/platform/http" - "github.com/uber/submitqueue/submitqueue/entity" - "github.com/uber/submitqueue/submitqueue/extension/mergechecker" - "go.uber.org/zap/zaptest" -) - -func newTestMergeChecker(t *testing.T, serverURL string) mergechecker.MergeChecker { - t.Helper() - client, err := phttp.NewClient(serverURL) - require.NoError(t, err) - return NewMergeChecker(Params{ - HTTPClient: client, - Logger: zaptest.NewLogger(t).Sugar(), - MetricsScope: tally.NoopScope, - }) -} - -// Sample 40-char lowercase hex SHAs used across the test cases. -const ( - sha1Full = "1111111111111111111111111111111111111111" - sha2Full = "2222222222222222222222222222222222222222" - shaAFull = "abcdef0123456789abcdef0123456789abcdef01" - shaOldFull = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" - shaNewFull = "feedfacefeedfacefeedfacefeedfacefeedface" -) - -func graphQLHandler(t *testing.T, prInfos []PRInfo) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - t.Helper() - - data := make(map[string]json.RawMessage, len(prInfos)) - for i, pr := range prInfos { - alias := fmt.Sprintf("pr%d", i) - prJSON, err := json.Marshal(map[string]any{ - "pullRequest": map[string]any{ - "number": pr.Number, - "mergeable": string(pr.Mergeable), - "baseRefName": pr.BaseRefName, - "headRefName": pr.HeadRefName, - "headRefOid": pr.HeadRefOid, - "state": string(pr.State), - }, - }) - require.NoError(t, err) - data[alias] = json.RawMessage(prJSON) - } - - resp := graphQLResponse{Data: data} - w.Header().Set("Content-Type", "application/json") - err := json.NewEncoder(w).Encode(resp) - require.NoError(t, err) - } -} - -func TestMergeChecker_Check(t *testing.T) { - tests := []struct { - name string - handler http.HandlerFunc - change change.Change - wantMergeable bool - wantReason string - wantErr bool - }{ - { - name: "single PR mergeable", - handler: graphQLHandler(t, []PRInfo{ - {Number: 1, Mergeable: PRMergeableStateMergeable, BaseRefName: "main", HeadRefName: "feature-1", HeadRefOid: shaAFull, State: PRStateOpen}, - }), - change: change.Change{URIs: []string{"github://github.example.com/uber/repo/pull/1/" + shaAFull}}, - wantMergeable: true, - }, - { - name: "single PR conflicting", - handler: graphQLHandler(t, []PRInfo{ - {Number: 1, Mergeable: PRMergeableStateConflicting, BaseRefName: "main", HeadRefName: "feature-1", HeadRefOid: shaAFull, State: PRStateOpen}, - }), - change: change.Change{URIs: []string{"github://github.example.com/uber/repo/pull/1/" + shaAFull}}, - wantMergeable: false, - wantReason: "PR #1 has merge conflicts", - }, - { - name: "stack of two PRs mergeable", - handler: graphQLHandler(t, []PRInfo{ - {Number: 1, Mergeable: PRMergeableStateMergeable, BaseRefName: "main", HeadRefName: "feature-1", HeadRefOid: sha1Full, State: PRStateOpen}, - {Number: 2, Mergeable: PRMergeableStateMergeable, BaseRefName: "feature-1", HeadRefName: "feature-2", HeadRefOid: sha2Full, State: PRStateOpen}, - }), - change: change.Change{URIs: []string{"github://github.example.com/uber/repo/pull/1/" + sha1Full, "github://github.example.com/uber/repo/pull/2/" + sha2Full}}, - wantMergeable: true, - }, - { - name: "unknown mergeability returns error", - handler: graphQLHandler(t, []PRInfo{ - {Number: 1, Mergeable: PRMergeableStateUnknown, BaseRefName: "main", HeadRefName: "feature-1", HeadRefOid: shaAFull, State: PRStateOpen}, - }), - change: change.Change{URIs: []string{"github://github.example.com/uber/repo/pull/1/" + shaAFull}}, - wantErr: true, - }, - { - name: "stale SHA not mergeable", - handler: graphQLHandler(t, []PRInfo{ - {Number: 1, Mergeable: PRMergeableStateMergeable, BaseRefName: "main", HeadRefName: "feature-1", HeadRefOid: shaNewFull, State: PRStateOpen}, - }), - change: change.Change{URIs: []string{"github://github.example.com/uber/repo/pull/1/" + shaOldFull}}, - wantMergeable: false, - wantReason: fmt.Sprintf("PR #1 head SHA changed: expected %s, got %s", shaOldFull, shaNewFull), - }, - { - name: "invalid change ID", - handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - t.Fatal("should not reach server") - }), - change: change.Change{URIs: []string{"invalid-change-id"}}, - wantErr: true, - }, - { - name: "server error", - handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte("internal server error")) - }), - change: change.Change{URIs: []string{"github://github.example.com/uber/repo/pull/1/" + shaAFull}}, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - server := httptest.NewServer(tt.handler) - defer server.Close() - - mc := newTestMergeChecker(t, server.URL) - result, err := mc.Check(context.Background(), entity.Request{Change: tt.change}) - if tt.wantErr { - require.Error(t, err) - return - } - require.NoError(t, err) - assert.Equal(t, tt.wantMergeable, result.Mergeable) - assert.Equal(t, tt.wantReason, result.Reason) - }) - } -} diff --git a/submitqueue/extension/mergechecker/github/graphql.go b/submitqueue/extension/mergechecker/github/graphql.go deleted file mode 100644 index 1c33c8065..000000000 --- a/submitqueue/extension/mergechecker/github/graphql.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package github - -import ( - "encoding/json" - "fmt" - "strings" - - entitygithub "github.com/uber/submitqueue/platform/base/change/github" -) - -// graphQLRequest is the request body for the GitHub GraphQL API. -type graphQLRequest struct { - // Query is the GraphQL query string. - Query string `json:"query"` -} - -// graphQLResponse is the top-level response from the GitHub GraphQL API. -type graphQLResponse struct { - // Data contains the query results keyed by alias. - Data map[string]json.RawMessage `json:"data"` - // Errors contains any GraphQL errors. - Errors []graphQLError `json:"errors"` -} - -// graphQLError represents a single GraphQL error. -type graphQLError struct { - // Message is the error message. - Message string `json:"message"` -} - -// repositoryResponse represents a repository query result. -type repositoryResponse struct { - // PullRequest contains the PR data. - PullRequest prResponse `json:"pullRequest"` -} - -// prResponse represents the fields fetched for a single pull request. -type prResponse struct { - // Number is the PR number. - Number int `json:"number"` - // Mergeable is the mergeability state. - Mergeable string `json:"mergeable"` - // BaseRefName is the base branch name. - BaseRefName string `json:"baseRefName"` - // HeadRefName is the head branch name. - HeadRefName string `json:"headRefName"` - // HeadRefOid is the head commit SHA. - HeadRefOid string `json:"headRefOid"` - // State is the PR state (OPEN, CLOSED, MERGED). - State string `json:"state"` -} - -// buildGraphQLQuery builds a batched GraphQL query for multiple PRs. -// Each PR gets an alias like "pr0", "pr1", etc. -func buildGraphQLQuery(changeIDs []entitygithub.ChangeID) string { - var sb strings.Builder - sb.WriteString("query {") - - for i, cid := range changeIDs { - fmt.Fprintf(&sb, ` - pr%d: repository(owner: %q, name: %q) { - pullRequest(number: %d) { - number - mergeable - baseRefName - headRefName - headRefOid - state - } - }`, i, cid.Org, cid.Repo, cid.PRNumber) - } - - sb.WriteString("\n}") - return sb.String() -} - -// parseGraphQLResponse parses the GraphQL response body and returns a map of PR number to PRInfo. -func parseGraphQLResponse(body []byte, changeIDs []entitygithub.ChangeID) (map[int]PRInfo, error) { - var resp graphQLResponse - if err := json.Unmarshal(body, &resp); err != nil { - return nil, fmt.Errorf("failed to parse GraphQL response: %w", err) - } - - if len(resp.Errors) > 0 { - messages := make([]string, len(resp.Errors)) - for i, e := range resp.Errors { - messages[i] = e.Message - } - return nil, fmt.Errorf("GraphQL errors: %s", strings.Join(messages, "; ")) - } - - result := make(map[int]PRInfo, len(changeIDs)) - for i := range changeIDs { - alias := fmt.Sprintf("pr%d", i) - raw, ok := resp.Data[alias] - if !ok { - return nil, fmt.Errorf("missing alias %q in GraphQL response", alias) - } - - var repoResp repositoryResponse - if err := json.Unmarshal(raw, &repoResp); err != nil { - return nil, fmt.Errorf("failed to parse alias %q: %w", alias, err) - } - - pr := repoResp.PullRequest - result[pr.Number] = PRInfo{ - Number: pr.Number, - Mergeable: PRMergeableState(pr.Mergeable), - BaseRefName: pr.BaseRefName, - HeadRefName: pr.HeadRefName, - HeadRefOid: pr.HeadRefOid, - State: PRState(pr.State), - } - } - - return result, nil -} diff --git a/submitqueue/extension/mergechecker/github/graphql_test.go b/submitqueue/extension/mergechecker/github/graphql_test.go deleted file mode 100644 index 909114fc9..000000000 --- a/submitqueue/extension/mergechecker/github/graphql_test.go +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package github - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - entitygithub "github.com/uber/submitqueue/platform/base/change/github" -) - -func TestBuildGraphQLQuery(t *testing.T) { - tests := []struct { - name string - changeIDs []entitygithub.ChangeID - wantParts []string - }{ - { - name: "single PR", - changeIDs: []entitygithub.ChangeID{ - {Scheme: "github", Org: "uber", Repo: "submitqueue", PRNumber: 42, HeadCommitSHA: "abc123"}, - }, - wantParts: []string{ - "query {", - `pr0: repository(owner: "uber", name: "submitqueue")`, - "pullRequest(number: 42)", - "number", "mergeable", "baseRefName", "headRefName", "headRefOid", "state", - }, - }, - { - name: "multiple PRs across repos", - changeIDs: []entitygithub.ChangeID{ - {Scheme: "github", Org: "uber", Repo: "repo", PRNumber: 1, HeadCommitSHA: "sha1"}, - {Scheme: "github", Org: "uber", Repo: "repo", PRNumber: 2, HeadCommitSHA: "sha2"}, - {Scheme: "github", Org: "corp", Repo: "app", PRNumber: 99, HeadCommitSHA: "sha99"}, - }, - wantParts: []string{ - `pr0: repository(owner: "uber", name: "repo")`, - "pullRequest(number: 1)", - `pr1: repository(owner: "uber", name: "repo")`, - "pullRequest(number: 2)", - `pr2: repository(owner: "corp", name: "app")`, - "pullRequest(number: 99)", - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - query := buildGraphQLQuery(tt.changeIDs) - for _, part := range tt.wantParts { - assert.Contains(t, query, part) - } - }) - } -} - -func TestParseGraphQLResponse(t *testing.T) { - tests := []struct { - name string - body string - changeIDs []entitygithub.ChangeID - want map[int]PRInfo - wantErr bool - }{ - { - name: "success with two PRs", - changeIDs: []entitygithub.ChangeID{ - {Scheme: "github", Org: "uber", Repo: "repo", PRNumber: 1, HeadCommitSHA: "sha1"}, - {Scheme: "github", Org: "uber", Repo: "repo", PRNumber: 2, HeadCommitSHA: "sha2"}, - }, - body: mustMarshalGraphQLResponse(t, map[string]json.RawMessage{ - "pr0": json.RawMessage(`{"pullRequest":{"number":1,"mergeable":"MERGEABLE","baseRefName":"main","headRefName":"feature-1","headRefOid":"sha1","state":"OPEN"}}`), - "pr1": json.RawMessage(`{"pullRequest":{"number":2,"mergeable":"CONFLICTING","baseRefName":"feature-1","headRefName":"feature-2","headRefOid":"sha2","state":"OPEN"}}`), - }), - want: map[int]PRInfo{ - 1: {Number: 1, Mergeable: PRMergeableStateMergeable, BaseRefName: "main", HeadRefName: "feature-1", HeadRefOid: "sha1", State: PRStateOpen}, - 2: {Number: 2, Mergeable: PRMergeableStateConflicting, BaseRefName: "feature-1", HeadRefName: "feature-2", HeadRefOid: "sha2", State: PRStateOpen}, - }, - }, - { - name: "GraphQL errors", - changeIDs: []entitygithub.ChangeID{ - {Scheme: "github", Org: "uber", Repo: "repo", PRNumber: 1, HeadCommitSHA: "sha1"}, - }, - body: `{"data":null,"errors":[{"message":"Not Found"},{"message":"Forbidden"}]}`, - wantErr: true, - }, - { - name: "invalid JSON", - changeIDs: []entitygithub.ChangeID{ - {Scheme: "github", Org: "uber", Repo: "repo", PRNumber: 1, HeadCommitSHA: "sha1"}, - }, - body: `invalid`, - wantErr: true, - }, - { - name: "missing alias", - changeIDs: []entitygithub.ChangeID{ - {Scheme: "github", Org: "uber", Repo: "repo", PRNumber: 1, HeadCommitSHA: "sha1"}, - }, - body: `{"data":{}}`, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result, err := parseGraphQLResponse([]byte(tt.body), tt.changeIDs) - if tt.wantErr { - require.Error(t, err) - return - } - require.NoError(t, err) - assert.Equal(t, tt.want, result) - }) - } -} - -// mustMarshalGraphQLResponse is a test helper to build a GraphQL response body. -func mustMarshalGraphQLResponse(t *testing.T, data map[string]json.RawMessage) string { - t.Helper() - resp := graphQLResponse{Data: data} - body, err := json.Marshal(resp) - require.NoError(t, err) - return string(body) -} diff --git a/submitqueue/extension/mergechecker/github/validate.go b/submitqueue/extension/mergechecker/github/validate.go deleted file mode 100644 index a5163b76b..000000000 --- a/submitqueue/extension/mergechecker/github/validate.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package github - -import ( - "fmt" - - entitygithub "github.com/uber/submitqueue/platform/base/change/github" -) - -// PRMergeableState represents the mergeability state of a pull request. -type PRMergeableState string - -const ( - // PRMergeableStateMergeable indicates the PR can be merged cleanly. - PRMergeableStateMergeable PRMergeableState = "MERGEABLE" - // PRMergeableStateConflicting indicates the PR has merge conflicts. - PRMergeableStateConflicting PRMergeableState = "CONFLICTING" - // PRMergeableStateUnknown indicates GitHub hasn't computed mergeability yet. - // GitHub computes mergeability asynchronously after pushes. The GraphQL API - // returns UNKNOWN when the computation hasn't finished, even though the API - // call itself is synchronous. Callers should retry after a short delay. - PRMergeableStateUnknown PRMergeableState = "UNKNOWN" -) - -// PRState represents the state of a pull request. -type PRState string - -const ( - // PRStateOpen indicates the PR is open. - PRStateOpen PRState = "OPEN" - // PRStateClosed indicates the PR is closed. - PRStateClosed PRState = "CLOSED" - // PRStateMerged indicates the PR has been merged. - PRStateMerged PRState = "MERGED" -) - -// PRInfo holds the relevant pull request information fetched from GitHub. -type PRInfo struct { - // Number is the pull request number. - Number int - // Mergeable is the mergeability state of the PR. - Mergeable PRMergeableState - // BaseRefName is the base branch the PR targets. - BaseRefName string - // HeadRefName is the head branch of the PR. - HeadRefName string - // HeadRefOid is the current head commit SHA of the PR. - HeadRefOid string - // State is the current state of the PR (OPEN, CLOSED, MERGED). - State PRState -} - -// validatePRs validates that all PRs are open, individually mergeable, and not stale. -// Returns (true, "", nil) if all PRs pass validation. -// Returns (false, reason, nil) if definitively not mergeable (conflicts, closed, stale SHA). -// Returns (false, "", error) if mergeability is UNKNOWN (retryable — GitHub hasn't computed yet). -func validatePRs(changeIDs []entitygithub.ChangeID, prInfoMap map[int]PRInfo) (bool, string, error) { - for _, cid := range changeIDs { - pr, ok := prInfoMap[cid.PRNumber] - if !ok { - return false, "", fmt.Errorf("PR #%d not found in API response", cid.PRNumber) - } - - // Check PR is open - if pr.State != PRStateOpen { - return false, fmt.Sprintf("PR #%d is %s", cid.PRNumber, pr.State), nil - } - - // Check mergeability - switch pr.Mergeable { - case PRMergeableStateConflicting: - return false, fmt.Sprintf("PR #%d has merge conflicts", cid.PRNumber), nil - case PRMergeableStateUnknown: - return false, "", fmt.Errorf("mergeability unknown for PR #%d, retry later", cid.PRNumber) - case PRMergeableStateMergeable: - // OK, continue - default: - return false, "", fmt.Errorf("unexpected mergeable state %q for PR #%d", pr.Mergeable, cid.PRNumber) - } - - // Check head commit SHA matches (staleness check) - if pr.HeadRefOid != cid.HeadCommitSHA { - return false, fmt.Sprintf("PR #%d head SHA changed: expected %s, got %s", cid.PRNumber, cid.HeadCommitSHA, pr.HeadRefOid), nil - } - } - - return true, "", nil -} diff --git a/submitqueue/extension/mergechecker/github/validate_test.go b/submitqueue/extension/mergechecker/github/validate_test.go deleted file mode 100644 index efe622558..000000000 --- a/submitqueue/extension/mergechecker/github/validate_test.go +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package github - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - entitygithub "github.com/uber/submitqueue/platform/base/change/github" -) - -func TestValidatePRs(t *testing.T) { - tests := []struct { - name string - changeIDs []entitygithub.ChangeID - prInfoMap map[int]PRInfo - wantOK bool - wantReason string - wantErr bool - }{ - { - name: "single PR mergeable", - changeIDs: []entitygithub.ChangeID{ - {Scheme: "github", Org: "uber", Repo: "repo", PRNumber: 1, HeadCommitSHA: "abc123"}, - }, - prInfoMap: map[int]PRInfo{ - 1: {Number: 1, Mergeable: PRMergeableStateMergeable, BaseRefName: "main", HeadRefName: "feature-1", HeadRefOid: "abc123", State: PRStateOpen}, - }, - wantOK: true, - }, - { - name: "stack of three PRs all mergeable", - changeIDs: []entitygithub.ChangeID{ - {Scheme: "github", Org: "uber", Repo: "repo", PRNumber: 1, HeadCommitSHA: "sha1"}, - {Scheme: "github", Org: "uber", Repo: "repo", PRNumber: 2, HeadCommitSHA: "sha2"}, - {Scheme: "github", Org: "uber", Repo: "repo", PRNumber: 3, HeadCommitSHA: "sha3"}, - }, - prInfoMap: map[int]PRInfo{ - 1: {Number: 1, Mergeable: PRMergeableStateMergeable, BaseRefName: "main", HeadRefName: "feature-1", HeadRefOid: "sha1", State: PRStateOpen}, - 2: {Number: 2, Mergeable: PRMergeableStateMergeable, BaseRefName: "feature-1", HeadRefName: "feature-2", HeadRefOid: "sha2", State: PRStateOpen}, - 3: {Number: 3, Mergeable: PRMergeableStateMergeable, BaseRefName: "feature-2", HeadRefName: "feature-3", HeadRefOid: "sha3", State: PRStateOpen}, - }, - wantOK: true, - }, - { - name: "PR closed", - changeIDs: []entitygithub.ChangeID{ - {Scheme: "github", Org: "uber", Repo: "repo", PRNumber: 1, HeadCommitSHA: "sha1"}, - }, - prInfoMap: map[int]PRInfo{ - 1: {Number: 1, Mergeable: PRMergeableStateMergeable, BaseRefName: "main", HeadRefName: "feature-1", HeadRefOid: "sha1", State: PRStateClosed}, - }, - wantOK: false, - wantReason: "PR #1 is CLOSED", - }, - { - name: "PR already merged", - changeIDs: []entitygithub.ChangeID{ - {Scheme: "github", Org: "uber", Repo: "repo", PRNumber: 1, HeadCommitSHA: "sha1"}, - }, - prInfoMap: map[int]PRInfo{ - 1: {Number: 1, Mergeable: PRMergeableStateMergeable, BaseRefName: "main", HeadRefName: "feature-1", HeadRefOid: "sha1", State: PRStateMerged}, - }, - wantOK: false, - wantReason: "PR #1 is MERGED", - }, - { - name: "PR has conflicts", - changeIDs: []entitygithub.ChangeID{ - {Scheme: "github", Org: "uber", Repo: "repo", PRNumber: 1, HeadCommitSHA: "sha1"}, - }, - prInfoMap: map[int]PRInfo{ - 1: {Number: 1, Mergeable: PRMergeableStateConflicting, BaseRefName: "main", HeadRefName: "feature-1", HeadRefOid: "sha1", State: PRStateOpen}, - }, - wantOK: false, - wantReason: "PR #1 has merge conflicts", - }, - { - name: "unknown mergeability returns error", - changeIDs: []entitygithub.ChangeID{ - {Scheme: "github", Org: "uber", Repo: "repo", PRNumber: 1, HeadCommitSHA: "sha1"}, - }, - prInfoMap: map[int]PRInfo{ - 1: {Number: 1, Mergeable: PRMergeableStateUnknown, BaseRefName: "main", HeadRefName: "feature-1", HeadRefOid: "sha1", State: PRStateOpen}, - }, - wantOK: false, - wantErr: true, - }, - { - name: "stale SHA", - changeIDs: []entitygithub.ChangeID{ - {Scheme: "github", Org: "uber", Repo: "repo", PRNumber: 1, HeadCommitSHA: "old_sha"}, - }, - prInfoMap: map[int]PRInfo{ - 1: {Number: 1, Mergeable: PRMergeableStateMergeable, BaseRefName: "main", HeadRefName: "feature-1", HeadRefOid: "new_sha", State: PRStateOpen}, - }, - wantOK: false, - wantReason: "PR #1 head SHA changed: expected old_sha, got new_sha", - }, - { - name: "PR not found in map", - changeIDs: []entitygithub.ChangeID{ - {Scheme: "github", Org: "uber", Repo: "repo", PRNumber: 999, HeadCommitSHA: "sha1"}, - }, - prInfoMap: map[int]PRInfo{}, - wantOK: false, - wantErr: true, - }, - { - name: "second PR in stack conflicting", - changeIDs: []entitygithub.ChangeID{ - {Scheme: "github", Org: "uber", Repo: "repo", PRNumber: 1, HeadCommitSHA: "sha1"}, - {Scheme: "github", Org: "uber", Repo: "repo", PRNumber: 2, HeadCommitSHA: "sha2"}, - }, - prInfoMap: map[int]PRInfo{ - 1: {Number: 1, Mergeable: PRMergeableStateMergeable, BaseRefName: "main", HeadRefName: "feature-1", HeadRefOid: "sha1", State: PRStateOpen}, - 2: {Number: 2, Mergeable: PRMergeableStateConflicting, BaseRefName: "feature-1", HeadRefName: "feature-2", HeadRefOid: "sha2", State: PRStateOpen}, - }, - wantOK: false, - wantReason: "PR #2 has merge conflicts", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ok, reason, err := validatePRs(tt.changeIDs, tt.prInfoMap) - if tt.wantErr { - require.Error(t, err) - } else { - require.NoError(t, err) - } - assert.Equal(t, tt.wantOK, ok) - assert.Equal(t, tt.wantReason, reason) - }) - } -} diff --git a/submitqueue/extension/mergechecker/mergechecker.go b/submitqueue/extension/mergechecker/mergechecker.go deleted file mode 100644 index 7ad8c41c4..000000000 --- a/submitqueue/extension/mergechecker/mergechecker.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mergechecker - -//go:generate mockgen -source=mergechecker.go -destination=mock/mergechecker_mock.go -package=mock - -import ( - "context" - - "github.com/uber/submitqueue/submitqueue/entity" -) - -// MergeChecker predicts whether a request's changes can merge cleanly. -type MergeChecker interface { - // Check is a fail-fast mergeability check that optimistically assesses - // whether the request's changes can be merged. It is handed the request - // identity and reads request.Change itself. A positive result does not - // guarantee that the changes will apply cleanly at merge time. - Check(ctx context.Context, request entity.Request) (entity.MergeResult, error) -} - -// Config carries the per-queue identity handed to a Factory. The system knows -// only the queue name; everything an implementation needs is injected at -// construction by the integrator. -type Config struct { - // QueueName identifies the queue this MergeChecker serves. - QueueName string -} - -// Factory builds the MergeChecker for a queue. Implementations are provided by -// integrators (and tests) and inject whatever they need at construction. -type Factory interface { - // For returns the MergeChecker for the given queue. - For(cfg Config) (MergeChecker, error) -} diff --git a/submitqueue/extension/mergechecker/mock/BUILD.bazel b/submitqueue/extension/mergechecker/mock/BUILD.bazel deleted file mode 100644 index 1b958a953..000000000 --- a/submitqueue/extension/mergechecker/mock/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library") - -go_library( - name = "go_default_library", - srcs = ["mergechecker_mock.go"], - importpath = "github.com/uber/submitqueue/submitqueue/extension/mergechecker/mock", - visibility = ["//visibility:public"], - deps = [ - "//submitqueue/entity:go_default_library", - "//submitqueue/extension/mergechecker:go_default_library", - "@org_uber_go_mock//gomock:go_default_library", - ], -) diff --git a/submitqueue/extension/mergechecker/mock/mergechecker_mock.go b/submitqueue/extension/mergechecker/mock/mergechecker_mock.go deleted file mode 100644 index 48b1237c8..000000000 --- a/submitqueue/extension/mergechecker/mock/mergechecker_mock.go +++ /dev/null @@ -1,97 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: mergechecker.go -// -// Generated by this command: -// -// mockgen -source=mergechecker.go -destination=mock/mergechecker_mock.go -package=mock -// - -// Package mock is a generated GoMock package. -package mock - -import ( - context "context" - reflect "reflect" - - entity "github.com/uber/submitqueue/submitqueue/entity" - mergechecker "github.com/uber/submitqueue/submitqueue/extension/mergechecker" - gomock "go.uber.org/mock/gomock" -) - -// MockMergeChecker is a mock of MergeChecker interface. -type MockMergeChecker struct { - ctrl *gomock.Controller - recorder *MockMergeCheckerMockRecorder - isgomock struct{} -} - -// MockMergeCheckerMockRecorder is the mock recorder for MockMergeChecker. -type MockMergeCheckerMockRecorder struct { - mock *MockMergeChecker -} - -// NewMockMergeChecker creates a new mock instance. -func NewMockMergeChecker(ctrl *gomock.Controller) *MockMergeChecker { - mock := &MockMergeChecker{ctrl: ctrl} - mock.recorder = &MockMergeCheckerMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockMergeChecker) EXPECT() *MockMergeCheckerMockRecorder { - return m.recorder -} - -// Check mocks base method. -func (m *MockMergeChecker) Check(ctx context.Context, request entity.Request) (entity.MergeResult, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Check", ctx, request) - ret0, _ := ret[0].(entity.MergeResult) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Check indicates an expected call of Check. -func (mr *MockMergeCheckerMockRecorder) Check(ctx, request any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Check", reflect.TypeOf((*MockMergeChecker)(nil).Check), ctx, request) -} - -// MockFactory is a mock of Factory interface. -type MockFactory struct { - ctrl *gomock.Controller - recorder *MockFactoryMockRecorder - isgomock struct{} -} - -// MockFactoryMockRecorder is the mock recorder for MockFactory. -type MockFactoryMockRecorder struct { - mock *MockFactory -} - -// NewMockFactory creates a new mock instance. -func NewMockFactory(ctrl *gomock.Controller) *MockFactory { - mock := &MockFactory{ctrl: ctrl} - mock.recorder = &MockFactoryMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockFactory) EXPECT() *MockFactoryMockRecorder { - return m.recorder -} - -// For mocks base method. -func (m *MockFactory) For(cfg mergechecker.Config) (mergechecker.MergeChecker, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "For", cfg) - ret0, _ := ret[0].(mergechecker.MergeChecker) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// For indicates an expected call of For. -func (mr *MockFactoryMockRecorder) For(cfg any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "For", reflect.TypeOf((*MockFactory)(nil).For), cfg) -} diff --git a/submitqueue/extension/speculation/allocator/sticky/sticky.go b/submitqueue/extension/speculation/allocator/sticky/sticky.go index fac42c657..d5135b6f3 100644 --- a/submitqueue/extension/speculation/allocator/sticky/sticky.go +++ b/submitqueue/extension/speculation/allocator/sticky/sticky.go @@ -81,7 +81,7 @@ func (a alloc) Allocate(ctx context.Context, pathSets []entity.SpeculationPathSe // keeps its slot until it actually stops. // // Only the path's status matters. No batch state enters this - // decision — "merging" and the rest are states of a batch, never of + // decision — "landing" and the rest are states of a batch, never of // a path — so the rule is simply that CI is still busy with it. funded[entry.ID] = true } diff --git a/submitqueue/extension/speculation/generator/bestfirst/README.md b/submitqueue/extension/speculation/generator/bestfirst/README.md index bd34769dd..b661b2e8e 100644 --- a/submitqueue/extension/speculation/generator/bestfirst/README.md +++ b/submitqueue/extension/speculation/generator/bestfirst/README.md @@ -10,7 +10,7 @@ The [best-first speculation path generation RFC](../../../../../doc/rfc/submitqu - `Next` removes the highest-ranked candidate, advances only that head's stream, constructs that candidate's complete path, and returns it. Pulling long enough returns every path exactly once in non-increasing score order. - Ranking scores are sums of log probabilities, avoiding underflow while preserving probability order. They are meaningful only within the run that produced them. - Exact ties prefer fewer flips; head ID then decides between heads (the cross-head heap holds one candidate per head), and taken flip indexes decide within a head. -- A dependency counts as resolved only once it is terminal. Merging and cancelling are both still in progress and either can end the other way, so both stay open questions here. Whether a path betting against a merging dependency is worth funding is a matter of price, and price is the scorer's to say. +- A dependency counts as resolved only once it is terminal. Landing and cancelling are both still in progress and either can end the other way, so both stay open questions here. Whether a path betting against a landing dependency is worth funding is a matter of price, and price is the scorer's to say. - A dependency that cannot be priced — the scorer call failed, the score was not a probability, or the snapshot never carried the batch — is treated as very likely to succeed rather than ending the run. One unusable number costs its own estimate, never the queue's whole set of candidates. A batch missing from the snapshot is never handed to the scorer at all: it would resolve to a zero batch belonging to no queue. - The snapshot must contain every batch a head's direct dependencies reference, carry unique non-empty batch IDs, and give no head an empty, duplicate, or self dependency. That is the caller's precondition, not something checked here: a malformed snapshot yields undefined candidates rather than an error. diff --git a/submitqueue/extension/speculation/generator/bestfirst/bestfirst_test.go b/submitqueue/extension/speculation/generator/bestfirst/bestfirst_test.go index 7dfb9f352..5f0a5d560 100644 --- a/submitqueue/extension/speculation/generator/bestfirst/bestfirst_test.go +++ b/submitqueue/extension/speculation/generator/bestfirst/bestfirst_test.go @@ -337,7 +337,7 @@ func TestBestFirst_OnlySpeculatingHeadsProduceCandidates(t *testing.T) { {state: entity.BatchStateUnknown}, {state: entity.BatchStateCreated}, {state: entity.BatchStateSpeculating, want: true}, - {state: entity.BatchStateMerging}, + {state: entity.BatchStateLanding}, {state: entity.BatchStateSucceeded}, {state: entity.BatchStateFailed}, {state: entity.BatchStateCancelling}, @@ -417,13 +417,13 @@ func TestBestFirst_NeverScoresAnAbsentDependency(t *testing.T) { assert.InDelta(t, math.Log(defaultProbability), cands[0].RankingScore, 1e-9) } -// A merging dependency is still in progress — the merge can fail — so it stays +// A landing dependency is still in progress — the land can fail — so it stays // an open question here like any other. Whether a path betting against it is // worth funding is a matter of price, which is the scorer's to say, not a // state the search hard-codes. -func TestBestFirst_MergingDependencyStaysOpen(t *testing.T) { +func TestBestFirst_LandingDependencyStaysOpen(t *testing.T) { batches := []entity.Batch{ - {ID: "q/landing", State: entity.BatchStateMerging}, + {ID: "q/landing", State: entity.BatchStateLanding}, {ID: "q/H", State: entity.BatchStateSpeculating, Dependencies: []string{"q/landing"}}, } sc := newCountingScorer(map[string]float64{"q/landing": 0.9}) @@ -432,8 +432,8 @@ func TestBestFirst_MergingDependencyStaysOpen(t *testing.T) { require.NoError(t, err) cands := drainAll(t, iter) - assert.Equal(t, 1, sc.calls["q/landing"], "a merging dependency is priced like any other") - require.Len(t, cands, 2, "both sides of a merge that has not landed yet") + assert.Equal(t, 1, sc.calls["q/landing"], "a landing dependency is priced like any other") + require.Len(t, cands, 2, "both sides of a land that has not completed yet") assert.Equal(t, entity.DependencyAssumptionSucceeds, assumptionFor(cands[0].Path, "q/landing")) assert.Equal(t, entity.DependencyAssumptionFails, assumptionFor(cands[1].Path, "q/landing")) } diff --git a/submitqueue/extension/speculation/scorer/scorer.go b/submitqueue/extension/speculation/scorer/scorer.go index a0d75401c..c654cd821 100644 --- a/submitqueue/extension/speculation/scorer/scorer.go +++ b/submitqueue/extension/speculation/scorer/scorer.go @@ -29,7 +29,7 @@ type Scorer interface { // ultimately succeeds — reaches its terminal Succeeded state with its // changes landed, rather than Failed or Cancelled. A passing build is // necessary but not sufficient: a batch whose build already passed can - // still fail to merge, so this is the probability of the final outcome, + // still fail to land, so this is the probability of the final outcome, // not of the build alone. It is handed the batch identity and resolves the // batch's changes itself through an injected changeset.Resolver. // diff --git a/submitqueue/extension/speculation/speculator/README.md b/submitqueue/extension/speculation/speculator/README.md index 62343796a..3b842a7c7 100644 --- a/submitqueue/extension/speculation/speculator/README.md +++ b/submitqueue/extension/speculation/speculator/README.md @@ -1,6 +1,6 @@ # speculator -The `speculator` package defines the one speculation extension the speculate controller calls. A `Speculator` decides **which speculation paths to build and which running ones to cancel**, within the queue's build budget — and nothing else. It can never express a verdict: whether a batch merges or fails is fixed by the facts and computed by the controller, so swapping in a different `Speculator` changes which paths run, never a batch's outcome. +The `speculator` package defines the one speculation extension the speculate controller calls. A `Speculator` decides **which speculation paths to build and which running ones to cancel**, within the queue's build budget — and nothing else. It can never express a verdict: whether a batch lands or fails is fixed by the facts and computed by the controller, so swapping in a different `Speculator` changes which paths run, never a batch's outcome. `Speculate` is handed the queue's in-flight batches plus any finalized batches still referenced as dependencies (each with its dependency list and state) and every path set for them — live and recently finished, so a `Speculator` will not re-propose a path that already passed or failed. It returns a list of build and cancel actions; a path it wants left as-is has no entry in the result. The controller validates the output (dropping builds it shouldn't propose and rejecting cancels of passed paths), so an implementation may read extra injected data without affecting correctness. diff --git a/submitqueue/extension/speculation/speculator/speculator.go b/submitqueue/extension/speculation/speculator/speculator.go index acf7237f8..296d1c19e 100644 --- a/submitqueue/extension/speculation/speculator/speculator.go +++ b/submitqueue/extension/speculation/speculator/speculator.go @@ -25,7 +25,7 @@ import ( // Speculator decides which speculation paths to build and which running ones to // cancel, within the queue's build budget. It is the only speculation extension the // speculate controller calls. It can never express a verdict: whether a batch -// merges or fails is fixed by the facts and computed by the controller, so a +// lands or fails is fixed by the facts and computed by the controller, so a // swapped-in Speculator changes which paths run, never a batch's outcome. type Speculator interface { // Speculate proposes this run's path actions. diff --git a/submitqueue/extension/speculation/speculator/standard/README.md b/submitqueue/extension/speculation/speculator/standard/README.md index c5d869e65..c96f1f8f6 100644 --- a/submitqueue/extension/speculation/speculator/standard/README.md +++ b/submitqueue/extension/speculation/speculator/standard/README.md @@ -4,7 +4,7 @@ The `standard` `Speculator` funds the queue's most promising speculation paths f Each run it considers candidate paths in descending order of their probability of being the future that actually happens, and proposes builds down that ranking. Paths already pending or building keep the slot they hold rather than restarting; paths whose builds already finished are skipped for as long as their records remain in the supplied path sets, so a finished path can be proposed again — for a retry, say — once retention drops it; new builds fill whatever budget remains. -When the budget runs out, everything below the cut waits for a later run. That is safe because the propose-side cannot invent a batch verdict: the speculate controller still decides merge from the persisted paths, including complete coverage of unsettled dependencies. +When the budget runs out, everything below the cut waits for a later run. That is safe because the propose-side cannot invent a batch verdict: the speculate controller still decides land from the persisted paths, including complete coverage of unsettled dependencies. Both halves are swappable. The ranking is the `Generator`'s: the default `bestfirst` scores each path by the probability that all its assumptions hold. The budget policy is the `Allocator`'s: the default `sticky` fills only free slots and never preempts, where a preempting allocator would cancel a low-value in-flight path to fund a better one. diff --git a/submitqueue/extension/storage/mysql/batch_store_test.go b/submitqueue/extension/storage/mysql/batch_store_test.go index 7cd69aeff..145fd7620 100644 --- a/submitqueue/extension/storage/mysql/batch_store_test.go +++ b/submitqueue/extension/storage/mysql/batch_store_test.go @@ -205,7 +205,7 @@ func TestBatchStore_Update(t *testing.T) { Queue: "monorepo", Contains: []string{"monorepo/3", "monorepo/4"}, Dependencies: []string{"monorepo/batch/1", "monorepo/batch/2"}, - State: entity.BatchStateMerging, + State: entity.BatchStateLanding, Version: oldVersion, } containsJSON, err := json.Marshal(batch.Contains) diff --git a/submitqueue/gateway/controller/cancel.go b/submitqueue/gateway/controller/cancel.go index 644ae3448..2cd42459e 100644 --- a/submitqueue/gateway/controller/cancel.go +++ b/submitqueue/gateway/controller/cancel.go @@ -32,7 +32,7 @@ import ( // CancelController handles cancel business logic for the gateway. It validates the request, // records a RequestStatusCancelling log entry (intent-only — cancellation is best-effort -// and may still race a successful merge), publishes a CancelRequest to the cancel topic, +// and may still race a successful land), publishes a CancelRequest to the cancel topic, // and returns a response. The orchestrator-side cancel controller performs the actual // state transitions and emits the terminal RequestStatusCancelled log entry. type CancelController interface { @@ -66,7 +66,7 @@ func NewCancelController(logger *zap.SugaredLogger, scope tally.Scope, stores st // is performed asynchronously by the orchestrator cancel controller. Cancel is idempotent: // the orchestrator treats already-terminal requests as a no-op. // -// Cancellation is best-effort: a request that has already merged or that races to +// Cancellation is best-effort: a request that has already landed or that races to // completion before the cancel propagates may still land. The RequestStatusCancelling // entry written here records the user's intent; the terminal outcome is reflected by a // later RequestStatusCancelled (orchestrator side) or RequestStatusLanded entry. diff --git a/submitqueue/orchestrator/BUILD.bazel b/submitqueue/orchestrator/BUILD.bazel index 35b1d581d..f806c7789 100644 --- a/submitqueue/orchestrator/BUILD.bazel +++ b/submitqueue/orchestrator/BUILD.bazel @@ -28,9 +28,9 @@ go_library( "//submitqueue/orchestrator/controller/conclude:go_default_library", "//submitqueue/orchestrator/controller/dependencyanalysis:go_default_library", "//submitqueue/orchestrator/controller/dlq:go_default_library", - "//submitqueue/orchestrator/controller/merge:go_default_library", - "//submitqueue/orchestrator/controller/mergeconflictsignal:go_default_library", - "//submitqueue/orchestrator/controller/mergesignal:go_default_library", + "//submitqueue/orchestrator/controller/land:go_default_library", + "//submitqueue/orchestrator/controller/landconflictsignal:go_default_library", + "//submitqueue/orchestrator/controller/landsignal:go_default_library", "//submitqueue/orchestrator/controller/speculate:go_default_library", "//submitqueue/orchestrator/controller/start:go_default_library", "//submitqueue/orchestrator/controller/validate:go_default_library", diff --git a/submitqueue/orchestrator/README.md b/submitqueue/orchestrator/README.md index 6dea30d7c..50a837c41 100644 --- a/submitqueue/orchestrator/README.md +++ b/submitqueue/orchestrator/README.md @@ -8,17 +8,17 @@ The pipeline is queue-driven: each stage consumes one topic, advances one entity - **start** — receives `LandRequest` from the gateway, persists the `Request` entity, and emits `Started`. - **cancel** — records cancellation intent and hands affected batches to speculation for best-effort cancellation. -- **validate** — checks for duplicates, resolves change metadata, and publishes a `MergeRequest` to Runway's `merge-conflict-check` topic. -- **merge-conflict-check-signal** — correlates the dry-run result, fails the request on conflict, or forwards it to batching. +- **validate** — checks for duplicates, resolves change metadata, and adapts the land request to Runway's `MergeRequest` on the `merge-conflict-check` topic. +- **landconflictsignal** — consumes `MergeResult` from Runway's `merge-conflict-check-signal`, fails the request on conflict, or forwards it to batching. - **batch** — creates an inert batch attempt and hands it to dependency analysis. - **dependency-analysis** — enrols requests, computes dependencies, and promotes the selected batch attempt. - **speculate** — reconciles queue-wide path state, decides outcomes, and allocates speculative builds. - **build** — triggers a CI build for a speculative path. - **buildsignal** — polls or receives CI state, records the result, wakes `speculate`, and holds non-terminal deliveries until the next poll. -- **merge** — publishes a committing `MergeRequest` to Runway's `runway-merge` topic. -- **merge-signal** — correlates the merge result and fans out to `conclude` and back to `speculate`. +- **land** — adapts a batch to a committing `MergeRequest` on Runway's `runway-merge` topic. +- **landsignal** — consumes `MergeResult` from Runway's `merge-signal`, correlates the land result, and fans out to `conclude` and back to `speculate`. - **conclude** — maps the terminal batch state to the request states. - **submitqueue-hook** — dispatches lifecycle hook events to configured integrations. - **DLQ reconcilers** — one per primary consumed topic, driving stuck requests/batches to a conservative terminal `failed` state. -The orchestrator publishes request-log entries to `log`, but does not consume or persist them; the gateway owns that stage. It also publishes full cross-service requests to Runway's merge-conflict-check and merge topics. +The orchestrator publishes request-log entries to `log`, but does not consume or persist them; the gateway owns that stage. It also adapts SubmitQueue's land terminology to full cross-service requests on Runway's merge-conflict-check and merge topics. diff --git a/submitqueue/orchestrator/controller/README.md b/submitqueue/orchestrator/controller/README.md index 4fc03735a..9f69b9a4b 100644 --- a/submitqueue/orchestrator/controller/README.md +++ b/submitqueue/orchestrator/controller/README.md @@ -50,7 +50,7 @@ Such effects require a provider-supported idempotency key, a stable operation id The speculate topic is a dirty signal for a queue, not a command to transition only the named batch. A run reloads the queue's in-flight batches, dependency outcomes, path sets, and build results; admits any batches still in `Created`; commits outcomes to a fixed point; asks the configured speculator for new proposals; persists changed path sets; and dispatches pending builds. -A batch can advance to merge when one passed path matches all settled dependency outcomes. It can also advance while dependencies remain unsettled when passed paths cover every possible outcome of those dependencies, proving that the head passed regardless of how they finish. +A batch can advance to land when one passed path matches all settled dependency outcomes. It can also advance while dependencies remain unsettled when passed paths cover every possible outcome of those dependencies, proving that the head passed regardless of how they finish. Path-set changes are persisted before build dispatch. Terminal outcomes are committed before later decisions derive from them. Selected request-log announcements are intentionally published before their corresponding state write so replay cannot lose the observation. diff --git a/submitqueue/orchestrator/controller/cancel/cancel.go b/submitqueue/orchestrator/controller/cancel/cancel.go index 243e3af32..07493accf 100644 --- a/submitqueue/orchestrator/controller/cancel/cancel.go +++ b/submitqueue/orchestrator/controller/cancel/cancel.go @@ -26,14 +26,14 @@ // - The request is associated with one or more batch attempts — the controller // records cancellation intent on every cancellable attempt and hands each one // to speculate. Creating attempts are ignored because their dependency set is -// not yet resolved and nothing downstream can see them, while Merging and +// not yet resolved and nothing downstream can see them, while Landing and // terminal attempts retain their existing outcome for conclude to reconcile. // // The split exists so that the terminal write and the work that must precede // it (cancelling builds, respeculating dependents) live in the same controller // — speculate is the single writer of every non-Cancelling batch state and is // already wired with the build/dependent stores. Forward-progress controllers -// (build, buildsignal, merge) observe BatchStateCancelling via +// (build, buildsignal, land) observe BatchStateCancelling via // IsBatchStateHalted and short-circuit while speculate drives the batch to // its terminal state. // @@ -149,7 +149,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er // Step 1: record the cancellation intent on the request itself by transitioning // to RequestStateCancelling. This is non-terminal; forward-progress controllers // (validate, batch) treat it as halted, but conclude may still write a different - // terminal state if a concurrent merge or failure wins the race. + // terminal state if a concurrent land or failure wins the race. request, err = c.markCancelling(ctx, store, request) if err != nil { return err @@ -180,10 +180,10 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er firstErr = err } } - case batch.State == entity.BatchStateMerging: - // Merge owns the outcome once it has started. Conclude will reconcile the request with that outcome. + case batch.State == entity.BatchStateLanding: + // Land owns the outcome once it has started. Conclude will reconcile the request with that outcome. foundApplicableBatch = true - metrics.NamedCounter(c.metricsScope, opName, "batch_merging", 1) + metrics.NamedCounter(c.metricsScope, opName, "batch_landing", 1) case batch.State.IsTerminal(): // The terminal batch outcome wins; conclude may not have reconciled the request yet. foundApplicableBatch = true @@ -311,7 +311,7 @@ func (c *Controller) cancelBatch(ctx context.Context, store storage.Storage, bat if err != nil { metrics.NamedCounter(c.metricsScope, opName, "batch_update_errors", 1) // storage.ErrVersionMismatch here means the batch advanced concurrently - // (e.g. speculate / merge progressed). Returned as-is because the + // (e.g. speculate / land progressed). Returned as-is because the // sentinel is intrinsically retryable; the re-fetch will see the new state // and either short-circuit (already terminal) or attempt the transition // again. diff --git a/submitqueue/orchestrator/controller/cancel/cancel_test.go b/submitqueue/orchestrator/controller/cancel/cancel_test.go index 01274bafa..e5bc743e4 100644 --- a/submitqueue/orchestrator/controller/cancel/cancel_test.go +++ b/submitqueue/orchestrator/controller/cancel/cancel_test.go @@ -517,7 +517,7 @@ func TestProcess_NonCancellableBatchSuppressesRequestCancellation(t *testing.T) name string state entity.BatchState }{ - {name: "merging", state: entity.BatchStateMerging}, + {name: "landing", state: entity.BatchStateLanding}, {name: "succeeded", state: entity.BatchStateSucceeded}, {name: "failed", state: entity.BatchStateFailed}, {name: "cancelled", state: entity.BatchStateCancelled}, diff --git a/submitqueue/orchestrator/controller/conclude/conclude.go b/submitqueue/orchestrator/controller/conclude/conclude.go index 8f7c28d38..86432d831 100644 --- a/submitqueue/orchestrator/controller/conclude/conclude.go +++ b/submitqueue/orchestrator/controller/conclude/conclude.go @@ -106,7 +106,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er ) // Map batch terminal state to request state. - // We expect the batch to be in a terminal state as written by the merge + // We expect the batch to be in a terminal state as written by the land // controller (Succeeded) or the speculate controller (Failed via // failOnDependency, Cancelled via cancelBatch). requestState, err := batchStateToRequestState(batch.State) diff --git a/submitqueue/orchestrator/controller/dependencyanalysis/dependencyanalysis.go b/submitqueue/orchestrator/controller/dependencyanalysis/dependencyanalysis.go index cb36da1d8..66f8ec81d 100644 --- a/submitqueue/orchestrator/controller/dependencyanalysis/dependencyanalysis.go +++ b/submitqueue/orchestrator/controller/dependencyanalysis/dependencyanalysis.go @@ -219,7 +219,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er metrics.NamedCounter(c.metricsScope, opName, "reannounced", 1) default: - // Speculating or merging: the announcement landed and the batch has + // Speculating or landing: the announcement landed and the batch has // already moved past this stage. metrics.NamedCounter(c.metricsScope, opName, "already_admitted", 1) return nil @@ -246,7 +246,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er // with its own hand-off. This is where that is resolved: the topic is // partitioned by queue and consumed in order, so the first hand-off through // here enrols the request and the second finds it and stops. Without the check -// the same change would end up in two live batches, both admitted, both merged. +// the same change would end up in two live batches, both admitted, both landed. func (c *Controller) requestEnrolledInAnotherBatch(ctx context.Context, store storage.Storage, batch entity.Batch) (bool, error) { for _, requestID := range batch.Contains { existing, stale, err := corebatch.FindByRequestID(ctx, store, requestID) diff --git a/submitqueue/orchestrator/controller/dependencyanalysis/dependencyanalysis_test.go b/submitqueue/orchestrator/controller/dependencyanalysis/dependencyanalysis_test.go index 1a04e7032..c04619d17 100644 --- a/submitqueue/orchestrator/controller/dependencyanalysis/dependencyanalysis_test.go +++ b/submitqueue/orchestrator/controller/dependencyanalysis/dependencyanalysis_test.go @@ -410,7 +410,7 @@ func TestController_Process_HaltedBatchAcksWithoutPublishing(t *testing.T) { // A batch already admitted got its announcement; re-sending one would only buy // a redundant re-plan. func TestController_Process_AlreadyAdmittedAcksWithoutPublishing(t *testing.T) { - for _, state := range []entity.BatchState{entity.BatchStateSpeculating, entity.BatchStateMerging} { + for _, state := range []entity.BatchState{entity.BatchStateSpeculating, entity.BatchStateLanding} { t.Run(string(state), func(t *testing.T) { ctrl := gomock.NewController(t) diff --git a/submitqueue/orchestrator/controller/dlq/BUILD.bazel b/submitqueue/orchestrator/controller/dlq/BUILD.bazel index ffcbd2eb1..a78013a4e 100644 --- a/submitqueue/orchestrator/controller/dlq/BUILD.bazel +++ b/submitqueue/orchestrator/controller/dlq/BUILD.bazel @@ -6,9 +6,9 @@ go_library( "batch.go", "buildsignal.go", "dlq.go", + "landconflictsignal.go", + "landsignal.go", "log.go", - "mergeconflictsignal.go", - "mergesignal.go", "request.go", "speculate.go", ], @@ -35,9 +35,9 @@ go_test( "batch_test.go", "buildsignal_test.go", "dlq_test.go", + "landconflictsignal_test.go", + "landsignal_test.go", "log_test.go", - "mergeconflictsignal_test.go", - "mergesignal_test.go", "publisher_test.go", "request_test.go", "speculate_test.go", diff --git a/submitqueue/orchestrator/controller/dlq/README.md b/submitqueue/orchestrator/controller/dlq/README.md index befa1bb9f..a4dfe6ccf 100644 --- a/submitqueue/orchestrator/controller/dlq/README.md +++ b/submitqueue/orchestrator/controller/dlq/README.md @@ -31,7 +31,7 @@ Two controller shapes cover the eleven primary pipeline topics: | Controller | Topics | Decoded ID | Terminal state | |---|---|---|---| | `NewDLQRequestController` | `start`, `validate`, `batch`, `cancel`, `log` | `RequestID` | `RequestStateError` | -| `NewDLQBatchController` | `speculate`, `build`, `merge`, `conclude` | `BatchID` | `BatchStateFailed` + fan-out to member requests as `RequestStateError` | +| `NewDLQBatchController` | `speculate`, `build`, `land`, `conclude` | `BatchID` | `BatchStateFailed` + fan-out to member requests as `RequestStateError` | `buildsignal` carries a `Build` payload and has its own small dedicated controller. The split exists because the DLQ message payload shape mirrors the primary topic's payload shape (the queue framework preserves bytes verbatim under the `_dlq` topic name), so the decoder is what changes per topic — not the reconciliation step. The package-level `RequestIDDecoder` interface plus `DecodeLandRequestID` / `DecodeCancelRequestID` / `DecodeRequestID` cover the three payload shapes used by request-scoped topics. diff --git a/submitqueue/orchestrator/controller/dlq/batch.go b/submitqueue/orchestrator/controller/dlq/batch.go index 5111926bf..9bc25fb65 100644 --- a/submitqueue/orchestrator/controller/dlq/batch.go +++ b/submitqueue/orchestrator/controller/dlq/batch.go @@ -27,7 +27,7 @@ import ( ) // batchController is the DLQ reconciler for batch-scoped pipeline stages -// (build, merge, conclude). All three topics carry a BatchID payload, so this +// (build, land, conclude). All three topics carry a BatchID payload, so this // controller is registered three times — one per topic, each with the matching // DLQ topic key and consumer group. // @@ -39,7 +39,7 @@ import ( // // Blaming the batch on the message is right for these stages because their // work is that batch: whatever failed, it failed doing this batch's build, -// merge, or conclusion. The speculate stage is not like that — it re-plans a +// land, or conclusion. The speculate stage is not like that — it re-plans a // whole queue from a message that names one batch — so it has its own // reconciler; see speculate.go. type batchController struct { diff --git a/submitqueue/orchestrator/controller/dlq/batch_test.go b/submitqueue/orchestrator/controller/dlq/batch_test.go index a125271df..a75fea26b 100644 --- a/submitqueue/orchestrator/controller/dlq/batch_test.go +++ b/submitqueue/orchestrator/controller/dlq/batch_test.go @@ -33,11 +33,11 @@ func TestDLQBatchController_InterfaceAndAccessors(t *testing.T) { store := storagemock.NewMockStorage(ctrl) store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() - c := NewDLQBatchController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, TopicKey(topickey.TopicKeyMerge), "orchestrator-merge-dlq") + c := NewDLQBatchController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, TopicKey(topickey.TopicKeyLand), "orchestrator-land-dlq") - assert.Equal(t, "submitqueue-merge_dlq", c.Name()) - assert.Equal(t, consumer.TopicKey("submitqueue-merge_dlq"), c.TopicKey()) - assert.Equal(t, "orchestrator-merge-dlq", c.ConsumerGroup()) + assert.Equal(t, "submitqueue-land_dlq", c.Name()) + assert.Equal(t, consumer.TopicKey("submitqueue-land_dlq"), c.TopicKey()) + assert.Equal(t, "orchestrator-land-dlq", c.ConsumerGroup()) } func TestDLQBatchController_Process_FailsAndFansOut(t *testing.T) { @@ -46,7 +46,7 @@ func TestDLQBatchController_Process_FailsAndFansOut(t *testing.T) { batchStore := storagemock.NewMockBatchStore(ctrl) batch := entity.Batch{ ID: "q/batch/9", Queue: "q", Contains: []string{"q/1"}, - State: entity.BatchStateMerging, Version: 2, + State: entity.BatchStateLanding, Version: 2, } batchStore.EXPECT().Get(gomock.Any(), "q/batch/9").Return(batch, nil) batchStore.EXPECT().Update(gomock.Any(), batchWithState(batch, entity.BatchStateFailed), int32(2), int32(3)).Return(nil) @@ -67,7 +67,7 @@ func TestDLQBatchController_Process_FailsAndFansOut(t *testing.T) { store.EXPECT().GetBatchStore().Return(batchStore).AnyTimes() store.EXPECT().GetRequestStore().Return(requestStore).AnyTimes() - c := NewDLQBatchController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, registry, TopicKey(topickey.TopicKeyMerge), "orchestrator-merge-dlq") + c := NewDLQBatchController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, registry, TopicKey(topickey.TopicKeyLand), "orchestrator-land-dlq") payload, err := entity.BatchID{ID: "q/batch/9"}.ToBytes() require.NoError(t, err) @@ -81,7 +81,7 @@ func TestDLQBatchController_Process_MalformedPayloadFails(t *testing.T) { store := storagemock.NewMockStorage(ctrl) store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() - c := NewDLQBatchController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, TopicKey(topickey.TopicKeyMerge), "orchestrator-merge-dlq") + c := NewDLQBatchController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, TopicKey(topickey.TopicKeyLand), "orchestrator-land-dlq") delivery := newMockDelivery(ctrl, []byte("garbage")) err := c.Process(context.Background(), delivery) @@ -93,7 +93,7 @@ func TestDLQBatchController_Process_EmptyIDFails(t *testing.T) { store := storagemock.NewMockStorage(ctrl) store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() - c := NewDLQBatchController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, TopicKey(topickey.TopicKeyMerge), "orchestrator-merge-dlq") + c := NewDLQBatchController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, TopicKey(topickey.TopicKeyLand), "orchestrator-land-dlq") payload, err := entity.BatchID{ID: ""}.ToBytes() require.NoError(t, err) diff --git a/submitqueue/orchestrator/controller/dlq/dlq.go b/submitqueue/orchestrator/controller/dlq/dlq.go index ee9c6b387..3d99e4d1c 100644 --- a/submitqueue/orchestrator/controller/dlq/dlq.go +++ b/submitqueue/orchestrator/controller/dlq/dlq.go @@ -28,7 +28,7 @@ // new `{topic}_dlq` name). The DLQ controllers decode that payload to recover // the affected request or batch, then transition it to a terminal failed // state — Error for requests, Failed for batches — with an idempotent -// optimistic-locking write so concurrent activity (a late merge, a cancel +// optimistic-locking write so concurrent activity (a late land, a cancel // race) wins cleanly. Batch failures also fan out to the member requests so // the gateway no longer reports them as in-progress. package dlq diff --git a/submitqueue/orchestrator/controller/dlq/dlq_test.go b/submitqueue/orchestrator/controller/dlq/dlq_test.go index 8d7f63b98..d44bf3443 100644 --- a/submitqueue/orchestrator/controller/dlq/dlq_test.go +++ b/submitqueue/orchestrator/controller/dlq/dlq_test.go @@ -221,7 +221,7 @@ func TestFailBatch_TransitionsAndFansOut(t *testing.T) { batchStore := storagemock.NewMockBatchStore(ctrl) batch := entity.Batch{ ID: "q/batch/1", Queue: "q", Contains: []string{"q/1", "q/2"}, - State: entity.BatchStateMerging, Version: 4, + State: entity.BatchStateLanding, Version: 4, } batchStore.EXPECT().Get(gomock.Any(), "q/batch/1").Return(batch, nil) batchStore.EXPECT().Update(gomock.Any(), batchWithState(batch, entity.BatchStateFailed), int32(4), int32(5)).Return(nil) diff --git a/submitqueue/orchestrator/controller/dlq/mergeconflictsignal.go b/submitqueue/orchestrator/controller/dlq/landconflictsignal.go similarity index 75% rename from submitqueue/orchestrator/controller/dlq/mergeconflictsignal.go rename to submitqueue/orchestrator/controller/dlq/landconflictsignal.go index c51561288..471b30a0d 100644 --- a/submitqueue/orchestrator/controller/dlq/mergeconflictsignal.go +++ b/submitqueue/orchestrator/controller/dlq/landconflictsignal.go @@ -26,11 +26,11 @@ import ( "go.uber.org/zap" ) -// mergeConflictSignalController is the DLQ reconciler for the -// mergeconflictsignal topic. Its payload carries a runway +// landConflictSignalController is the DLQ reconciler for the +// landconflictsignal topic. Its payload carries a runway // MergeResult whose id is the request id echoed back, so // reconciliation fails that request directly via failRequest. -type mergeConflictSignalController struct { +type landConflictSignalController struct { logger *zap.SugaredLogger metricsScope tally.Scope stores storage.Factory @@ -39,12 +39,12 @@ type mergeConflictSignalController struct { consumerGroup string } -// Verify mergeConflictSignalController implements consumer.Controller at compile time. -var _ consumer.Controller = (*mergeConflictSignalController)(nil) +// Verify landConflictSignalController implements consumer.Controller at compile time. +var _ consumer.Controller = (*landConflictSignalController)(nil) -// NewDLQMergeConflictSignalController builds a DLQ controller for the -// mergeconflictsignal topic. -func NewDLQMergeConflictSignalController( +// NewDLQLandConflictSignalController builds a DLQ controller for the +// landconflictsignal topic. +func NewDLQLandConflictSignalController( logger *zap.SugaredLogger, scope tally.Scope, stores storage.Factory, @@ -53,7 +53,7 @@ func NewDLQMergeConflictSignalController( consumerGroup string, ) consumer.Controller { name := string(topicKey) + "_controller" - return &mergeConflictSignalController{ + return &landConflictSignalController{ logger: logger.Named(name), metricsScope: scope.SubScope(name), stores: stores, @@ -63,8 +63,8 @@ func NewDLQMergeConflictSignalController( } } -// Process reconciles a single DLQ delivery for the mergeconflictsignal topic. -func (c *mergeConflictSignalController) Process(ctx context.Context, delivery consumer.Delivery) error { +// Process reconciles a single DLQ delivery for the landconflictsignal topic. +func (c *landConflictSignalController) Process(ctx context.Context, delivery consumer.Delivery) error { const opName = "process" msg := delivery.Message() @@ -72,7 +72,7 @@ func (c *mergeConflictSignalController) Process(ctx context.Context, delivery co result := &runwaymq.MergeResult{} if err := runwaymq.Unmarshal(msg.Payload, result); err != nil { metrics.NamedCounter(c.metricsScope, opName, "deserialize_errors", 1) - return fmt.Errorf("failed to decode merge conflict check result from dlq payload: %w", err) + return fmt.Errorf("failed to decode land conflict check result from dlq payload: %w", err) } store, err := c.stores.For(storage.Config{QueueName: result.GetQueueName()}) @@ -101,16 +101,16 @@ func (c *mergeConflictSignalController) Process(ctx context.Context, delivery co } // Name returns the controller name for logging and metrics. -func (c *mergeConflictSignalController) Name() string { +func (c *landConflictSignalController) Name() string { return string(c.topicKey) } // TopicKey returns the topic key this controller subscribes to. -func (c *mergeConflictSignalController) TopicKey() consumer.TopicKey { +func (c *landConflictSignalController) TopicKey() consumer.TopicKey { return c.topicKey } // ConsumerGroup returns the consumer group for offset tracking. -func (c *mergeConflictSignalController) ConsumerGroup() string { +func (c *landConflictSignalController) ConsumerGroup() string { return c.consumerGroup } diff --git a/submitqueue/orchestrator/controller/dlq/mergeconflictsignal_test.go b/submitqueue/orchestrator/controller/dlq/landconflictsignal_test.go similarity index 71% rename from submitqueue/orchestrator/controller/dlq/mergeconflictsignal_test.go rename to submitqueue/orchestrator/controller/dlq/landconflictsignal_test.go index bf23a1d6a..74acba067 100644 --- a/submitqueue/orchestrator/controller/dlq/mergeconflictsignal_test.go +++ b/submitqueue/orchestrator/controller/dlq/landconflictsignal_test.go @@ -29,19 +29,19 @@ import ( "go.uber.org/zap/zaptest" ) -func TestDLQMergeConflictSignalController_InterfaceAndAccessors(t *testing.T) { +func TestDLQLandConflictSignalController_InterfaceAndAccessors(t *testing.T) { ctrl := gomock.NewController(t) store := storagemock.NewMockStorage(ctrl) store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() - c := NewDLQMergeConflictSignalController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, TopicKey(runwaymq.TopicKeyMergeConflictCheckSignal), "orchestrator-mergeconflictsignal-dlq") + c := NewDLQLandConflictSignalController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, TopicKey(runwaymq.TopicKeyMergeConflictCheckSignal), "orchestrator-landconflictsignal-dlq") assert.Equal(t, "merge-conflict-check-signal_dlq", c.Name()) assert.Equal(t, consumer.TopicKey("merge-conflict-check-signal_dlq"), c.TopicKey()) - assert.Equal(t, "orchestrator-mergeconflictsignal-dlq", c.ConsumerGroup()) + assert.Equal(t, "orchestrator-landconflictsignal-dlq", c.ConsumerGroup()) } -func TestDLQMergeConflictSignalController_Process_ReconcilesRequest(t *testing.T) { +func TestDLQLandConflictSignalController_Process_ReconcilesRequest(t *testing.T) { ctrl := gomock.NewController(t) requestStore := storagemock.NewMockRequestStore(ctrl) @@ -59,7 +59,7 @@ func TestDLQMergeConflictSignalController_Process_ReconcilesRequest(t *testing.T store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() store.EXPECT().GetRequestStore().Return(requestStore).AnyTimes() - c := NewDLQMergeConflictSignalController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, registry, TopicKey(runwaymq.TopicKeyMergeConflictCheckSignal), "orchestrator-mergeconflictsignal-dlq") + c := NewDLQLandConflictSignalController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, registry, TopicKey(runwaymq.TopicKeyMergeConflictCheckSignal), "orchestrator-landconflictsignal-dlq") payload, err := runwaymq.Marshal(&runwaymq.MergeResult{Id: "q/1", Outcome: runwaypb.Outcome_FAILED, Reason: "boom"}) require.NoError(t, err) @@ -68,12 +68,12 @@ func TestDLQMergeConflictSignalController_Process_ReconcilesRequest(t *testing.T require.NoError(t, c.Process(context.Background(), delivery)) } -func TestDLQMergeConflictSignalController_Process_MalformedPayloadFails(t *testing.T) { +func TestDLQLandConflictSignalController_Process_MalformedPayloadFails(t *testing.T) { ctrl := gomock.NewController(t) store := storagemock.NewMockStorage(ctrl) store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() - c := NewDLQMergeConflictSignalController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, TopicKey(runwaymq.TopicKeyMergeConflictCheckSignal), "orchestrator-mergeconflictsignal-dlq") + c := NewDLQLandConflictSignalController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, TopicKey(runwaymq.TopicKeyMergeConflictCheckSignal), "orchestrator-landconflictsignal-dlq") delivery := newMockDelivery(ctrl, []byte("garbage")) require.Error(t, c.Process(context.Background(), delivery)) diff --git a/submitqueue/orchestrator/controller/dlq/mergesignal.go b/submitqueue/orchestrator/controller/dlq/landsignal.go similarity index 77% rename from submitqueue/orchestrator/controller/dlq/mergesignal.go rename to submitqueue/orchestrator/controller/dlq/landsignal.go index 8eb3d9e4f..412f51bb3 100644 --- a/submitqueue/orchestrator/controller/dlq/mergesignal.go +++ b/submitqueue/orchestrator/controller/dlq/landsignal.go @@ -26,11 +26,11 @@ import ( "go.uber.org/zap" ) -// mergeSignalController is the DLQ reconciler for the mergesignal topic. Its -// payload carries a runway MergeResult whose id is the batch id echoed back, so +// landSignalController is the DLQ reconciler for the landsignal topic. Its +// payload carries a Runway MergeResult whose id is the batch id echoed back, so // reconciliation fails that batch directly via failBatch (which also fans out // to the member requests). -type mergeSignalController struct { +type landSignalController struct { logger *zap.SugaredLogger metricsScope tally.Scope stores storage.Factory @@ -39,11 +39,11 @@ type mergeSignalController struct { consumerGroup string } -// Verify mergeSignalController implements consumer.Controller at compile time. -var _ consumer.Controller = (*mergeSignalController)(nil) +// Verify landSignalController implements consumer.Controller at compile time. +var _ consumer.Controller = (*landSignalController)(nil) -// NewDLQMergeSignalController builds a DLQ controller for the mergesignal topic. -func NewDLQMergeSignalController( +// NewDLQLandSignalController builds a DLQ controller for the landsignal topic. +func NewDLQLandSignalController( logger *zap.SugaredLogger, scope tally.Scope, stores storage.Factory, @@ -52,7 +52,7 @@ func NewDLQMergeSignalController( consumerGroup string, ) consumer.Controller { name := string(topicKey) + "_controller" - return &mergeSignalController{ + return &landSignalController{ logger: logger.Named(name), metricsScope: scope.SubScope(name), stores: stores, @@ -62,8 +62,8 @@ func NewDLQMergeSignalController( } } -// Process reconciles a single DLQ delivery for the mergesignal topic. -func (c *mergeSignalController) Process(ctx context.Context, delivery consumer.Delivery) error { +// Process reconciles a single DLQ delivery for the landsignal topic. +func (c *landSignalController) Process(ctx context.Context, delivery consumer.Delivery) error { const opName = "process" msg := delivery.Message() @@ -71,7 +71,7 @@ func (c *mergeSignalController) Process(ctx context.Context, delivery consumer.D result := &runwaymq.MergeResult{} if err := runwaymq.Unmarshal(msg.Payload, result); err != nil { metrics.NamedCounter(c.metricsScope, opName, "deserialize_errors", 1) - return fmt.Errorf("failed to decode merge result from dlq payload: %w", err) + return fmt.Errorf("failed to decode land result from dlq payload: %w", err) } store, err := c.stores.For(storage.Config{QueueName: result.GetQueueName()}) @@ -100,16 +100,16 @@ func (c *mergeSignalController) Process(ctx context.Context, delivery consumer.D } // Name returns the controller name for logging and metrics. -func (c *mergeSignalController) Name() string { +func (c *landSignalController) Name() string { return string(c.topicKey) } // TopicKey returns the topic key this controller subscribes to. -func (c *mergeSignalController) TopicKey() consumer.TopicKey { +func (c *landSignalController) TopicKey() consumer.TopicKey { return c.topicKey } // ConsumerGroup returns the consumer group for offset tracking. -func (c *mergeSignalController) ConsumerGroup() string { +func (c *landSignalController) ConsumerGroup() string { return c.consumerGroup } diff --git a/submitqueue/orchestrator/controller/dlq/mergesignal_test.go b/submitqueue/orchestrator/controller/dlq/landsignal_test.go similarity index 76% rename from submitqueue/orchestrator/controller/dlq/mergesignal_test.go rename to submitqueue/orchestrator/controller/dlq/landsignal_test.go index 69a122ae4..ef3e5efe5 100644 --- a/submitqueue/orchestrator/controller/dlq/mergesignal_test.go +++ b/submitqueue/orchestrator/controller/dlq/landsignal_test.go @@ -29,27 +29,27 @@ import ( "go.uber.org/zap/zaptest" ) -func TestDLQMergeSignalController_InterfaceAndAccessors(t *testing.T) { +func TestDLQLandSignalController_InterfaceAndAccessors(t *testing.T) { ctrl := gomock.NewController(t) store := storagemock.NewMockStorage(ctrl) store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() - c := NewDLQMergeSignalController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, TopicKey(runwaymq.TopicKeyMergeSignal), "orchestrator-mergesignal-dlq") + c := NewDLQLandSignalController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, TopicKey(runwaymq.TopicKeyMergeSignal), "orchestrator-landsignal-dlq") assert.Equal(t, "merge-signal_dlq", c.Name()) assert.Equal(t, consumer.TopicKey("merge-signal_dlq"), c.TopicKey()) - assert.Equal(t, "orchestrator-mergesignal-dlq", c.ConsumerGroup()) + assert.Equal(t, "orchestrator-landsignal-dlq", c.ConsumerGroup()) } // The payload id is the batch id echoed back, so reconciliation fails the batch // and fans out to its member requests via failBatch. -func TestDLQMergeSignalController_Process_ReconcilesBatch(t *testing.T) { +func TestDLQLandSignalController_Process_ReconcilesBatch(t *testing.T) { ctrl := gomock.NewController(t) batchStore := storagemock.NewMockBatchStore(ctrl) batch := entity.Batch{ ID: "q/batch/1", Queue: "q", Contains: []string{"q/1"}, - State: entity.BatchStateMerging, Version: 2, + State: entity.BatchStateLanding, Version: 2, } batchStore.EXPECT().Get(gomock.Any(), "q/batch/1").Return(batch, nil) batchStore.EXPECT().Update(gomock.Any(), batchWithState(batch, entity.BatchStateFailed), int32(2), int32(3)).Return(nil) @@ -70,7 +70,7 @@ func TestDLQMergeSignalController_Process_ReconcilesBatch(t *testing.T) { store.EXPECT().GetBatchStore().Return(batchStore).AnyTimes() store.EXPECT().GetRequestStore().Return(requestStore).AnyTimes() - c := NewDLQMergeSignalController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, registry, TopicKey(runwaymq.TopicKeyMergeSignal), "orchestrator-mergesignal-dlq") + c := NewDLQLandSignalController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, registry, TopicKey(runwaymq.TopicKeyMergeSignal), "orchestrator-landsignal-dlq") payload, err := runwaymq.Marshal(&runwaymq.MergeResult{Id: "q/batch/1", Outcome: runwaypb.Outcome_FAILED, Reason: "boom"}) require.NoError(t, err) @@ -79,12 +79,12 @@ func TestDLQMergeSignalController_Process_ReconcilesBatch(t *testing.T) { require.NoError(t, c.Process(context.Background(), delivery)) } -func TestDLQMergeSignalController_Process_MalformedPayloadFails(t *testing.T) { +func TestDLQLandSignalController_Process_MalformedPayloadFails(t *testing.T) { ctrl := gomock.NewController(t) store := storagemock.NewMockStorage(ctrl) store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() - c := NewDLQMergeSignalController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, TopicKey(runwaymq.TopicKeyMergeSignal), "orchestrator-mergesignal-dlq") + c := NewDLQLandSignalController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, TopicKey(runwaymq.TopicKeyMergeSignal), "orchestrator-landsignal-dlq") delivery := newMockDelivery(ctrl, []byte("garbage")) require.Error(t, c.Process(context.Background(), delivery)) diff --git a/submitqueue/orchestrator/controller/dlq/request_test.go b/submitqueue/orchestrator/controller/dlq/request_test.go index adc34d064..f25269d21 100644 --- a/submitqueue/orchestrator/controller/dlq/request_test.go +++ b/submitqueue/orchestrator/controller/dlq/request_test.go @@ -213,7 +213,7 @@ func TestDLQRequestController_Process_SkipsRequestOwnedByLiveBatch(t *testing.T) for _, state := range []entity.BatchState{ entity.BatchStateCreated, entity.BatchStateSpeculating, - entity.BatchStateMerging, + entity.BatchStateLanding, entity.BatchStateCancelling, } { t.Run(string(state), func(t *testing.T) { diff --git a/submitqueue/orchestrator/controller/dlq/speculate.go b/submitqueue/orchestrator/controller/dlq/speculate.go index 7e6fe5876..9d6403a32 100644 --- a/submitqueue/orchestrator/controller/dlq/speculate.go +++ b/submitqueue/orchestrator/controller/dlq/speculate.go @@ -42,7 +42,7 @@ import ( // So this reconciler reads the failure's subjects and acts on those. It also // republishes to speculate afterwards, because a dead letter here consumes an // edge the queue needed. Speculation is driven only by messages; a batch -// admitted to Speculating produces no build to signal and no merge to conclude, +// admitted to Speculating produces no build to signal and no land to conclude, // so once the message that would have funded it is gone, nothing is left to // look at it again. Without the republish the failure of one batch silently // strands every other batch in the queue. diff --git a/submitqueue/orchestrator/controller/merge/BUILD.bazel b/submitqueue/orchestrator/controller/land/BUILD.bazel similarity index 96% rename from submitqueue/orchestrator/controller/merge/BUILD.bazel rename to submitqueue/orchestrator/controller/land/BUILD.bazel index a0eb8359a..806647872 100644 --- a/submitqueue/orchestrator/controller/merge/BUILD.bazel +++ b/submitqueue/orchestrator/controller/land/BUILD.bazel @@ -2,8 +2,8 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", - srcs = ["merge.go"], - importpath = "github.com/uber/submitqueue/submitqueue/orchestrator/controller/merge", + srcs = ["land.go"], + importpath = "github.com/uber/submitqueue/submitqueue/orchestrator/controller/land", visibility = ["//visibility:public"], deps = [ "//api/base/change/protopb:go_default_library", @@ -23,7 +23,7 @@ go_library( go_test( name = "go_default_test", - srcs = ["merge_test.go"], + srcs = ["land_test.go"], embed = [":go_default_library"], deps = [ "//api/base/mergestrategy/protopb:go_default_library", diff --git a/submitqueue/orchestrator/controller/merge/merge.go b/submitqueue/orchestrator/controller/land/land.go similarity index 75% rename from submitqueue/orchestrator/controller/merge/merge.go rename to submitqueue/orchestrator/controller/land/land.go index a9657457b..5c3dfaa99 100644 --- a/submitqueue/orchestrator/controller/merge/merge.go +++ b/submitqueue/orchestrator/controller/land/land.go @@ -12,14 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package merge implements the trigger stage for the asynchronous merge. It -// consumes a batch ready to land, builds the full merge request from the +// Package land implements the trigger stage for the asynchronous land. It +// consumes a batch ready to land, builds the full land request from the // batch's member requests (one step per request, in Contains order), and -// publishes it to runway's merge queue using the batch id as the client-owned -// correlation id. Runway performs the merge out of process and publishes the -// result to the merge-signal queue, which the mergesignal stage consumes and +// publishes it to Runway's merge queue using the batch id as the client-owned +// correlation id. Runway executes the request as a merge and publishes the +// result to the merge-signal queue, which the landsignal stage consumes and // correlates back to the batch by that id. -package merge +package land import ( "context" @@ -29,7 +29,7 @@ import ( "go.uber.org/zap" changepb "github.com/uber/submitqueue/api/base/change/protopb" - strategypb "github.com/uber/submitqueue/api/base/mergestrategy/protopb" + mergestrategypb "github.com/uber/submitqueue/api/base/mergestrategy/protopb" runwaymq "github.com/uber/submitqueue/api/runway/messagequeue" "github.com/uber/submitqueue/platform/base/mergestrategy" "github.com/uber/submitqueue/platform/consumer" @@ -40,13 +40,13 @@ import ( "github.com/uber/submitqueue/submitqueue/extension/storage" ) -// Controller handles merge queue messages. Implements consumer.Controller. +// Controller handles land queue messages. Implements consumer.Controller. // -// It loads the batch and its member requests, assembles the full merge request +// It loads the batch and its member requests, assembles the full land request // (one step per member request, in Contains order, each carrying that request's -// change and land strategy), and publishes it to runway's merge queue. Runway -// performs the merge out of process and returns the result on the merge-signal -// queue; the mergesignal stage consumes it and transitions the batch. This +// change and land strategy), and publishes it to Runway's merge queue. Runway +// executes the request as a merge and returns the result on the merge-signal +// queue; the landsignal stage consumes it and transitions the batch. This // controller therefore performs no state transition itself. type Controller struct { logger *zap.SugaredLogger @@ -61,8 +61,8 @@ type Controller struct { // Verify Controller implements consumer.Controller interface at compile time. var _ consumer.Controller = (*Controller)(nil) -// NewController creates a new merge controller for the orchestrator. -// runwayTopicKey is the runway-owned topic this controller publishes merge +// NewController creates a new land controller for the orchestrator. +// runwayTopicKey is the runway-owned topic this controller publishes land // requests to (TopicKeyMerge). func NewController( logger *zap.SugaredLogger, @@ -74,8 +74,8 @@ func NewController( consumerGroup string, ) *Controller { return &Controller{ - logger: logger.Named("merge_controller"), - metricsScope: scope.SubScope("merge_controller"), + logger: logger.Named("land_controller"), + metricsScope: scope.SubScope("land_controller"), stores: stores, registry: registry, runwayTopicKey: runwayTopicKey, @@ -84,12 +84,12 @@ func NewController( } } -// Process publishes the full merge request to runway. Returns nil to ack +// Process publishes the full Runway merge request for this land. Returns nil to ack // (success), or error to nack/reject. // // Error classification: deserialize and storage failures are non-retryable // (reject to DLQ). The publish to runway is retryable — it is the hand-off that -// keeps the merge alive, so a transient enqueue blip should replay rather than +// keeps the land alive, so a transient enqueue blip should replay rather than // strand the batch. func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) error { const opName = "process" @@ -122,7 +122,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er return fmt.Errorf("payload queue %q does not match queue %q of batch %s", bid.Queue, batch.Queue, batch.ID) } - c.logger.Infow("received merge event", + c.logger.Infow("received land event", "batch_id", batch.ID, "queue", batch.Queue, "state", string(batch.State), @@ -131,27 +131,27 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er "partition_key", msg.PartitionKey, ) - // Short-circuit halted batches (terminal or cancelling): no merge should be + // Short-circuit halted batches (terminal or cancelling): no land should be // kicked off for a batch that will not proceed. Unlike the old synchronous - // merge there is no terminal re-fan-out here — the mergesignal stage owns the + // land there is no terminal re-fan-out here — the landsignal stage owns the // state transition and fan-out once runway's result returns, so a redelivery // at this stage simply acks. if entity.IsBatchStateHalted(batch.State) { metrics.NamedCounter(c.metricsScope, opName, "skipped_halted", 1) - c.logger.Infow("skipping merge for halted batch", + c.logger.Infow("skipping land for halted batch", "batch_id", batch.ID, "state", string(batch.State), ) return nil } - // Build the full payload runway needs to perform the merge. The batch id is + // Build the full merge payload Runway needs to execute the land. The batch id is // the client-owned correlation id, so a redelivery republishes the same id // and runway dedupes on it; the result is matched straight back to the batch. - req, err := c.buildMergeRequest(ctx, store, batch) + req, err := c.buildLandRequest(ctx, store, batch) if err != nil { metrics.NamedCounter(c.metricsScope, opName, "storage_errors", 1) - return fmt.Errorf("failed to build merge request for batch %s: %w", batch.ID, err) + return fmt.Errorf("failed to build land request for batch %s: %w", batch.ID, err) } // Report that the members are landing before the request goes out, so a @@ -169,7 +169,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er return fmt.Errorf("failed to publish to runway merge: %w", err) } - c.logger.Infow("published merge to runway", + c.logger.Infow("published merge request to Runway", "batch_id", batch.ID, "steps", len(req.Steps), "topic_key", c.runwayTopicKey, @@ -178,10 +178,10 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er return nil // Success - message will be acked } -// buildMergeRequest loads the batch's member requests and assembles the runway -// merge request: one MergeStep per request, in Contains order, attributed by +// buildLandRequest loads the batch's member requests and assembles the runway +// MergeRequest: one MergeStep per request, in Contains order, attributed by // request id and carrying that request's change and land strategy. -func (c *Controller) buildMergeRequest(ctx context.Context, store storage.Storage, batch entity.Batch) (*runwaymq.MergeRequest, error) { +func (c *Controller) buildLandRequest(ctx context.Context, store storage.Storage, batch entity.Batch) (*runwaymq.MergeRequest, error) { steps := make([]*runwaymq.MergeStep, 0, len(batch.Contains)) for _, requestID := range batch.Contains { request, err := store.GetRequestStore().Get(ctx, requestID) @@ -204,29 +204,31 @@ func (c *Controller) buildMergeRequest(ctx context.Context, store storage.Storag // toProtoStrategy maps the shared mergestrategy.MergeStrategy entity to the // proto Strategy enum carried on the wire. An unknown strategy maps to DEFAULT, // letting runway apply the queue's configured default. -func toProtoStrategy(s mergestrategy.MergeStrategy) strategypb.Strategy { +func toProtoStrategy(s mergestrategy.MergeStrategy) mergestrategypb.Strategy { switch s { case mergestrategy.MergeStrategyRebase: - return strategypb.Strategy_REBASE + return mergestrategypb.Strategy_REBASE case mergestrategy.MergeStrategySquashRebase: - return strategypb.Strategy_SQUASH_REBASE + return mergestrategypb.Strategy_SQUASH_REBASE case mergestrategy.MergeStrategyMerge: - return strategypb.Strategy_MERGE + return mergestrategypb.Strategy_MERGE + case mergestrategy.MergeStrategyPromote: + return mergestrategypb.Strategy_PROMOTE default: - return strategypb.Strategy_DEFAULT + return mergestrategypb.Strategy_DEFAULT } } -// publish serializes the runway merge request and publishes it to the given +// publish serializes the Runway merge request and publishes it to the given // topic key, partitioned by queue. // -// The correlation ID is the message ID with no cause: a batch is asked to merge +// The correlation ID is the message ID with no cause: a batch is asked to land // once, so a redelivery that re-asks is meant to dedup rather than have Runway // merge the same batch twice. func (c *Controller) publish(ctx context.Context, key consumer.TopicKey, req *runwaymq.MergeRequest, partitionKey string) error { payload, err := runwaymq.Marshal(req) if err != nil { - return fmt.Errorf("failed to serialize merge request: %w", err) + return fmt.Errorf("failed to serialize land request: %w", err) } if err := publish.Message(ctx, c.registry, key, publish.IntentID(req.GetId()), payload, partitionKey); err != nil { @@ -238,7 +240,7 @@ func (c *Controller) publish(ctx context.Context, key consumer.TopicKey, req *ru // Name returns the controller name for logging and metrics. func (c *Controller) Name() string { - return "merge" + return "land" } // TopicKey returns the topic key this controller subscribes to. diff --git a/submitqueue/orchestrator/controller/merge/merge_test.go b/submitqueue/orchestrator/controller/land/land_test.go similarity index 90% rename from submitqueue/orchestrator/controller/merge/merge_test.go rename to submitqueue/orchestrator/controller/land/land_test.go index 90c63cd6c..557c1721a 100644 --- a/submitqueue/orchestrator/controller/merge/merge_test.go +++ b/submitqueue/orchestrator/controller/land/land_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package merge +package land import ( "context" @@ -53,6 +53,26 @@ func batchIDPayload(t *testing.T, id string) []byte { return payload } +func TestToProtoStrategy(t *testing.T) { + tests := []struct { + name string + in mergestrategy.MergeStrategy + want strategypb.Strategy + }{ + {name: "default", in: mergestrategy.MergeStrategyUnknown, want: strategypb.Strategy_DEFAULT}, + {name: "rebase", in: mergestrategy.MergeStrategyRebase, want: strategypb.Strategy_REBASE}, + {name: "squash rebase", in: mergestrategy.MergeStrategySquashRebase, want: strategypb.Strategy_SQUASH_REBASE}, + {name: "merge", in: mergestrategy.MergeStrategyMerge, want: strategypb.Strategy_MERGE}, + {name: "promote", in: mergestrategy.MergeStrategyPromote, want: strategypb.Strategy_PROMOTE}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, toProtoStrategy(tt.in)) + }) + } +} + func newDelivery(t *testing.T, ctrl *gomock.Controller, batchID, partitionKey string) *consumermock.MockDelivery { msg := entityqueue.NewMessage(batchID, batchIDPayload(t, batchID), partitionKey, nil) delivery := consumermock.NewMockDelivery(ctrl) @@ -68,14 +88,14 @@ func newController(t *testing.T, store *storagemock.MockStorage, registry consum staticStorageFactory{store: store}, registry, runwaymq.TopicKeyMerge, - topickey.TopicKeyMerge, - "orchestrator-merge", + topickey.TopicKeyLand, + "orchestrator-land", ) } // publishes records what a controller run published, in order and by topic. The // controller writes to two topics now — the request-log fan-out and the runway -// merge request — so tests need to tell them apart and to see which came first. +// land request — so tests need to tell them apart and to see which came first. type publishes struct { inOrder []string byTopic map[string][]entityqueue.Message @@ -122,9 +142,9 @@ func TestNewController(t *testing.T) { c := newController(t, store, registry) require.NotNil(t, c) - assert.Equal(t, topickey.TopicKeyMerge, c.TopicKey()) - assert.Equal(t, "orchestrator-merge", c.ConsumerGroup()) - assert.Equal(t, "merge", c.Name()) + assert.Equal(t, topickey.TopicKeyLand, c.TopicKey()) + assert.Equal(t, "orchestrator-land", c.ConsumerGroup()) + assert.Equal(t, "land", c.Name()) var _ consumer.Controller = c } @@ -148,7 +168,7 @@ func TestProcess_PublishesFullPayloadToRunway(t *testing.T) { ID: batchID, Queue: "test-queue", Contains: []string{req1.ID, req2.ID}, - State: entity.BatchStateMerging, + State: entity.BatchStateLanding, Version: 4, } @@ -221,7 +241,7 @@ func TestProcess_HaltedBatchSkips(t *testing.T) { batchStore.EXPECT().Get(gomock.Any(), batchID).Return(batch, nil) // No request-store reads and no publish for a halted batch: the - // members are told nothing and runway is not asked to merge. + // members are told nothing and runway is not asked to land. store := storagemock.NewMockStorage(ctrl) store.EXPECT().GetBatchStore().Return(batchStore).AnyTimes() @@ -236,7 +256,7 @@ func TestProcess_HaltedBatchSkips(t *testing.T) { // TestProcess_ReportsLandingBeforeDispatch covers the request-log half of this // stage: every member of the batch is told it is landing, and it is told before -// the merge request goes out. The ordering is what makes a lost log entry +// the land request goes out. The ordering is what makes a lost log entry // recoverable — a failure here nacks with nothing announced to runway, and the // redelivery re-publishes under the same occurrence, which the queue dedupes. func TestProcess_ReportsLandingBeforeDispatch(t *testing.T) { @@ -248,7 +268,7 @@ func TestProcess_ReportsLandingBeforeDispatch(t *testing.T) { batch := entity.Batch{ ID: batchID, Queue: "test-queue", Contains: []string{req1.ID, req2.ID}, - State: entity.BatchStateMerging, Version: 2, + State: entity.BatchStateLanding, Version: 2, } batchStore := storagemock.NewMockBatchStore(ctrl) @@ -298,7 +318,7 @@ func TestProcess_PublishFailureReturnsError(t *testing.T) { const batchID = "test-queue/batch/2" req := entity.Request{ID: "test-queue/1", Queue: "test-queue", LandStrategy: mergestrategy.MergeStrategyRebase} - batch := entity.Batch{ID: batchID, Queue: "test-queue", Contains: []string{req.ID}, State: entity.BatchStateMerging, Version: 1} + batch := entity.Batch{ID: batchID, Queue: "test-queue", Contains: []string{req.ID}, State: entity.BatchStateLanding, Version: 1} batchStore := storagemock.NewMockBatchStore(ctrl) batchStore.EXPECT().Get(gomock.Any(), batchID).Return(batch, nil) @@ -315,7 +335,7 @@ func TestProcess_PublishFailureReturnsError(t *testing.T) { require.Error(t, c.Process(context.Background(), newDelivery(t, ctrl, batchID, batch.Queue))) // A failed log publish must stop the run before runway hears about - // the merge, so the redelivery can repair the log entry. + // the land, so the redelivery can repair the log entry. if tt.failTopic == "log" { assert.Empty(t, rec.byTopic["runway-merge"]) } diff --git a/submitqueue/orchestrator/controller/mergeconflictsignal/BUILD.bazel b/submitqueue/orchestrator/controller/landconflictsignal/BUILD.bazel similarity index 93% rename from submitqueue/orchestrator/controller/mergeconflictsignal/BUILD.bazel rename to submitqueue/orchestrator/controller/landconflictsignal/BUILD.bazel index 94f0c2827..2a7edbab5 100644 --- a/submitqueue/orchestrator/controller/mergeconflictsignal/BUILD.bazel +++ b/submitqueue/orchestrator/controller/landconflictsignal/BUILD.bazel @@ -2,8 +2,8 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", - srcs = ["mergeconflictsignal.go"], - importpath = "github.com/uber/submitqueue/submitqueue/orchestrator/controller/mergeconflictsignal", + srcs = ["landconflictsignal.go"], + importpath = "github.com/uber/submitqueue/submitqueue/orchestrator/controller/landconflictsignal", visibility = ["//visibility:public"], deps = [ "//api/runway/messagequeue:go_default_library", @@ -22,7 +22,7 @@ go_library( go_test( name = "go_default_test", - srcs = ["mergeconflictsignal_test.go"], + srcs = ["landconflictsignal_test.go"], embed = [":go_default_library"], deps = [ "//api/runway/messagequeue:go_default_library", diff --git a/submitqueue/orchestrator/controller/mergeconflictsignal/mergeconflictsignal.go b/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal.go similarity index 86% rename from submitqueue/orchestrator/controller/mergeconflictsignal/mergeconflictsignal.go rename to submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal.go index dbfb69b5c..ab3e6fe12 100644 --- a/submitqueue/orchestrator/controller/mergeconflictsignal/mergeconflictsignal.go +++ b/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal.go @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package mergeconflictsignal consumes merge-conflict check results from runway's -// signal queue, correlates them to the request by the echoed id, and either -// advances the request to the batch stage (mergeable) or fails it (conflicted). -// Unlike buildsignal it is purely result-driven — runway pushes the result, so +// Package landconflictsignal consumes merge-conflict results from Runway's +// signal queue, correlates them to the land request by the echoed id, and either +// advances the request to the batch stage (landable) or fails it (conflicted). +// Unlike buildsignal it is purely result-driven — Runway pushes the result, so // there is no poll loop or self-reschedule. -package mergeconflictsignal +package landconflictsignal import ( "context" @@ -36,7 +36,7 @@ import ( "go.uber.org/zap" ) -// Controller handles mergeconflictsignal queue messages. Implements consumer.Controller. +// Controller handles landconflictsignal queue messages. Implements consumer.Controller. type Controller struct { logger *zap.SugaredLogger metricsScope tally.Scope @@ -49,7 +49,7 @@ type Controller struct { // Verify Controller implements consumer.Controller interface at compile time. var _ consumer.Controller = (*Controller)(nil) -// NewController creates a new mergeconflictsignal controller for the orchestrator. +// NewController creates a new landconflictsignal controller for the orchestrator. func NewController( logger *zap.SugaredLogger, scope tally.Scope, @@ -59,8 +59,8 @@ func NewController( consumerGroup string, ) *Controller { return &Controller{ - logger: logger.Named("mergeconflictsignal_controller"), - metricsScope: scope.SubScope("mergeconflictsignal_controller"), + logger: logger.Named("landconflictsignal_controller"), + metricsScope: scope.SubScope("landconflictsignal_controller"), stores: stores, registry: registry, topicKey: topicKey, @@ -71,7 +71,7 @@ func NewController( // Process consumes a runway check result and advances or fails the request. // Returns nil to ack, or error to nack/reject. // -// A not-mergeable verdict is an expected outcome of the check, not a failure: +// A not-landable verdict is an expected outcome of the check, not a failure: // the request is driven to terminal Error inline and the message is acked. Only // infrastructure faults — deserialize, storage, the terminal transition, and the // batch publish — return an error and reject to the DLQ, where the request is @@ -86,7 +86,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er result := &runwaymq.MergeResult{} if err := runwaymq.Unmarshal(msg.Payload, result); err != nil { metrics.NamedCounter(c.metricsScope, opName, "deserialize_errors", 1) - return fmt.Errorf("failed to deserialize merge conflict check result: %w", err) + return fmt.Errorf("failed to deserialize land conflict check result: %w", err) } store, err := c.stores.For(storage.Config{QueueName: result.GetQueueName()}) @@ -102,9 +102,9 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er return fmt.Errorf("failed to get request %s: %w", result.Id, err) } - c.logger.Infow("received mergeconflict signal", + c.logger.Infow("received landconflict signal", "request_id", request.ID, - "mergeable", result.Outcome == runwaypb.Outcome_SUCCEEDED, + "landable", result.Outcome == runwaypb.Outcome_SUCCEEDED, "attempt", delivery.Attempt(), "partition_key", msg.PartitionKey, ) @@ -112,7 +112,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er // Short-circuit halted requests: the cancel path owns driving them terminal. if entity.IsRequestStateHalted(request.State) { metrics.NamedCounter(c.metricsScope, opName, "skipped_halted", 1) - c.logger.Infow("skipping mergeconflict signal for halted request", + c.logger.Infow("skipping landconflict signal for halted request", "request_id", request.ID, "state", string(request.State), ) @@ -120,8 +120,8 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er } if result.Outcome != runwaypb.Outcome_SUCCEEDED { - metrics.NamedCounter(c.metricsScope, opName, "not_mergeable", 1) - c.logger.Infow("request not mergeable", + metrics.NamedCounter(c.metricsScope, opName, "not_landable", 1) + c.logger.Infow("request not landable", "request_id", request.ID, "reason", result.Reason, ) @@ -132,7 +132,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er return nil } - // Advance the request to Validated now that the merge-conflict check passed. + // Advance the request to Validated now that the land-conflict check passed. newVersion := request.Version + 1 request.State = entity.RequestStateValidated if err := store.GetRequestStore().Update(ctx, request, request.Version, newVersion); err != nil { @@ -161,7 +161,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er } // failRequest drives the request to terminal RequestStateError and records the -// conflict reason on the request log. A not-mergeable verdict is an expected +// conflict reason on the request log. A not-landable verdict is an expected // terminal outcome of the check, so the request is concluded here directly. // // Idempotent under at-least-once delivery: a redelivery whose request is already @@ -216,7 +216,7 @@ func (c *Controller) publishRequestID(ctx context.Context, key consumer.TopicKey // Name returns the controller name for logging and metrics. func (c *Controller) Name() string { - return "mergeconflictsignal" + return "landconflictsignal" } // TopicKey returns the topic key this controller subscribes to. diff --git a/submitqueue/orchestrator/controller/mergeconflictsignal/mergeconflictsignal_test.go b/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal_test.go similarity index 93% rename from submitqueue/orchestrator/controller/mergeconflictsignal/mergeconflictsignal_test.go rename to submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal_test.go index c71dd9fb7..853a21deb 100644 --- a/submitqueue/orchestrator/controller/mergeconflictsignal/mergeconflictsignal_test.go +++ b/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package mergeconflictsignal +package landconflictsignal import ( "context" @@ -65,7 +65,7 @@ const ( testQueue = "test-queue" ) -func TestProcess_MergeablePublishesToBatch(t *testing.T) { +func TestProcess_LandablePublishesToBatch(t *testing.T) { ctrl := gomock.NewController(t) reqStore := storagemock.NewMockRequestStore(ctrl) @@ -97,7 +97,7 @@ func TestProcess_MergeablePublishesToBatch(t *testing.T) { require.NoError(t, err) controller := NewController(zaptest.NewLogger(t).Sugar(), tally.NoopScope, staticStorageFactory{store: store}, registry, - runwaymq.TopicKeyMergeConflictCheckSignal, "orchestrator-mergeconflictsignal") + runwaymq.TopicKeyMergeConflictCheckSignal, "orchestrator-landconflictsignal") res := runwaymq.MergeResult{Id: testRequestID, Outcome: runwaypb.Outcome_SUCCEEDED} msg := entityqueue.NewMessage(testRequestID, resultPayload(t, res), testQueue, nil) @@ -119,7 +119,7 @@ func TestProcess_MergeablePublishesToBatch(t *testing.T) { assert.Equal(t, testRequestID, rid.ID) } -func TestProcess_NotMergeableMarksRequestError(t *testing.T) { +func TestProcess_NotLandableMarksRequestError(t *testing.T) { ctrl := gomock.NewController(t) reqStore := storagemock.NewMockRequestStore(ctrl) @@ -154,11 +154,11 @@ func TestProcess_NotMergeableMarksRequestError(t *testing.T) { require.NoError(t, err) controller := NewController(zaptest.NewLogger(t).Sugar(), tally.NoopScope, staticStorageFactory{store: store}, registry, - runwaymq.TopicKeyMergeConflictCheckSignal, "orchestrator-mergeconflictsignal") + runwaymq.TopicKeyMergeConflictCheckSignal, "orchestrator-landconflictsignal") res := runwaymq.MergeResult{Id: testRequestID, Outcome: runwaypb.Outcome_FAILED, Reason: "conflict in foo.go"} msg := entityqueue.NewMessage(testRequestID, resultPayload(t, res), testQueue, nil) - // Not-mergeable is an expected terminal outcome, so Process acks (no error). + // Not-landable is an expected terminal outcome, so Process acks (no error). require.NoError(t, controller.Process(context.Background(), newDelivery(ctrl, msg))) // The single publish is the terminal log entry carrying the conflict reason. @@ -181,7 +181,7 @@ func TestFailRequest_UpdateFailureLeavesRequestUnchanged(t *testing.T) { store.EXPECT().GetRequestStore().Return(reqStore) controller := NewController(zaptest.NewLogger(t).Sugar(), tally.NoopScope, staticStorageFactory{store: store}, consumer.TopicRegistry{}, - runwaymq.TopicKeyMergeConflictCheckSignal, "orchestrator-mergeconflictsignal") + runwaymq.TopicKeyMergeConflictCheckSignal, "orchestrator-landconflictsignal") err := controller.failRequest(context.Background(), store, request, "conflict") require.Error(t, err) @@ -209,7 +209,7 @@ func TestProcess_HaltedRequestSkips(t *testing.T) { require.NoError(t, err) controller := NewController(zaptest.NewLogger(t).Sugar(), tally.NoopScope, staticStorageFactory{store: store}, registry, - runwaymq.TopicKeyMergeConflictCheckSignal, "orchestrator-mergeconflictsignal") + runwaymq.TopicKeyMergeConflictCheckSignal, "orchestrator-landconflictsignal") res := runwaymq.MergeResult{Id: testRequestID, Outcome: runwaypb.Outcome_SUCCEEDED} msg := entityqueue.NewMessage(testRequestID, resultPayload(t, res), testQueue, nil) diff --git a/submitqueue/orchestrator/controller/mergesignal/BUILD.bazel b/submitqueue/orchestrator/controller/landsignal/BUILD.bazel similarity index 94% rename from submitqueue/orchestrator/controller/mergesignal/BUILD.bazel rename to submitqueue/orchestrator/controller/landsignal/BUILD.bazel index 9163af3db..74e0e7f04 100644 --- a/submitqueue/orchestrator/controller/mergesignal/BUILD.bazel +++ b/submitqueue/orchestrator/controller/landsignal/BUILD.bazel @@ -2,8 +2,8 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", - srcs = ["mergesignal.go"], - importpath = "github.com/uber/submitqueue/submitqueue/orchestrator/controller/mergesignal", + srcs = ["landsignal.go"], + importpath = "github.com/uber/submitqueue/submitqueue/orchestrator/controller/landsignal", visibility = ["//visibility:public"], deps = [ "//api/runway/messagequeue:go_default_library", @@ -22,7 +22,7 @@ go_library( go_test( name = "go_default_test", - srcs = ["mergesignal_test.go"], + srcs = ["landsignal_test.go"], embed = [":go_default_library"], deps = [ "//api/runway/messagequeue:go_default_library", diff --git a/submitqueue/orchestrator/controller/mergesignal/mergesignal.go b/submitqueue/orchestrator/controller/landsignal/landsignal.go similarity index 82% rename from submitqueue/orchestrator/controller/mergesignal/mergesignal.go rename to submitqueue/orchestrator/controller/landsignal/landsignal.go index 7611dd837..350dd1160 100644 --- a/submitqueue/orchestrator/controller/mergesignal/mergesignal.go +++ b/submitqueue/orchestrator/controller/landsignal/landsignal.go @@ -12,14 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package mergesignal consumes merge results from runway's merge-signal queue, +// Package landsignal consumes merge results from Runway's merge-signal queue, // correlates them to the batch by the echoed id, and transitions the batch to a -// terminal state — Succeeded when runway merged the batch, Failed when it could -// not — then fans the batch out to conclude (so member requests pick up the -// outcome) and speculate (so dependents can re-plan). Like mergeconflictsignal -// it is purely result-driven — runway pushes the result, so there is no poll +// terminal state — Succeeded when Runway's merge completed the land, Failed +// when it could not — then fans the batch out to conclude (so member requests pick up the +// outcome) and speculate (so dependents can re-plan). Like landconflictsignal +// it is purely result-driven — Runway pushes the result, so there is no poll // loop or self-reschedule. -package mergesignal +package landsignal import ( "context" @@ -38,7 +38,7 @@ import ( "go.uber.org/zap" ) -// Controller handles mergesignal queue messages. Implements consumer.Controller. +// Controller handles landsignal queue messages. Implements consumer.Controller. type Controller struct { logger *zap.SugaredLogger metricsScope tally.Scope @@ -51,7 +51,7 @@ type Controller struct { // Verify Controller implements consumer.Controller interface at compile time. var _ consumer.Controller = (*Controller)(nil) -// NewController creates a new mergesignal controller for the orchestrator. +// NewController creates a new landsignal controller for the orchestrator. func NewController( logger *zap.SugaredLogger, scope tally.Scope, @@ -61,8 +61,8 @@ func NewController( consumerGroup string, ) *Controller { return &Controller{ - logger: logger.Named("mergesignal_controller"), - metricsScope: scope.SubScope("mergesignal_controller"), + logger: logger.Named("landsignal_controller"), + metricsScope: scope.SubScope("landsignal_controller"), stores: stores, registry: registry, topicKey: topicKey, @@ -70,10 +70,10 @@ func NewController( } } -// Process consumes a runway merge result and advances or fails the batch. +// Process consumes a runway land result and advances or fails the batch. // Returns nil to ack, or error to nack/reject. // -// A not-merged verdict is an expected outcome of the merge, not a failure: the +// A not-landed verdict is an expected outcome of the land, not a failure: the // batch is driven to terminal Failed inline and the message is acked. Only // infrastructure faults — deserialize, storage, the state transition, and the // fan-out publishes — return an error and reject to the DLQ, where the batch is @@ -88,7 +88,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er result := &runwaymq.MergeResult{} if err := runwaymq.Unmarshal(msg.Payload, result); err != nil { metrics.NamedCounter(c.metricsScope, opName, "deserialize_errors", 1) - return fmt.Errorf("failed to deserialize merge result: %w", err) + return fmt.Errorf("failed to deserialize land result: %w", err) } store, err := c.stores.For(storage.Config{QueueName: result.GetQueueName()}) @@ -104,9 +104,9 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er return fmt.Errorf("failed to get batch %s: %w", result.Id, err) } - c.logger.Infow("received merge signal", + c.logger.Infow("received land signal", "batch_id", batch.ID, - "merged", result.Outcome == runwaypb.Outcome_SUCCEEDED, + "landed", result.Outcome == runwaypb.Outcome_SUCCEEDED, "state", string(batch.State), "version", batch.Version, "attempt", delivery.Attempt(), @@ -115,22 +115,22 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er // Cancelling: the cancel path (via speculate) owns the terminal write and the // downstream fan-out for a batch the user asked to cancel. Silently ack — do - // not transition (a racing terminal merge result must not override the + // not transition (a racing terminal land result must not override the // cancel) and do not fan out. if batch.State == entity.BatchStateCancelling { metrics.NamedCounter(c.metricsScope, opName, "skipped_cancelling", 1) return nil } - // A merge failure's reason travels to conclude on the fan-out message, not on + // A land failure's reason travels to conclude on the fan-out message, not on // the batch, so it reaches the request's terminal log without becoming durable - // batch state. Empty on the merged path. Computed before the idempotency check + // batch state. Empty on the landed path. Computed before the idempotency check // so a redelivered failed batch re-fans-out with its reason intact. var failureReason string if result.Outcome != runwaypb.Outcome_SUCCEEDED { failureReason = result.Reason if failureReason == "" { - failureReason = "merge failed" + failureReason = "land failed" } } @@ -150,14 +150,14 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er var newState entity.BatchState if result.Outcome == runwaypb.Outcome_SUCCEEDED { newState = entity.BatchStateSucceeded - c.logger.Infow("merged batch", + c.logger.Infow("landed batch", "batch_id", batch.ID, "steps", result.Steps, ) } else { - metrics.NamedCounter(c.metricsScope, opName, "not_merged", 1) + metrics.NamedCounter(c.metricsScope, opName, "not_landed", 1) newState = entity.BatchStateFailed - c.logger.Warnw("batch merge failed", + c.logger.Warnw("batch land failed", "batch_id", batch.ID, "reason", result.Reason, ) @@ -175,7 +175,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er // fanout publishes the batch ID to conclude (so requests are updated) and to // speculate (so dependents can re-evaluate now that this batch is done). // -// Both messages name the merge as their cause. Without it the speculate +// Both messages name the land as their cause. Without it the speculate // publish would reuse the bare batch ID, which the batch controller already // published at creation, and the queue would drop this one as a duplicate for // as long as that row survives — leaving dependents unwoken. Conclude is @@ -187,11 +187,11 @@ func (c *Controller) fanout(ctx context.Context, batchID, queue, failureReason s if failureReason != "" { concludeMeta = map[string]string{topickey.MetadataKeyFailureReason: failureReason} } - if err := c.publish(ctx, topickey.TopicKeyConclude, publish.IntentID(batchID, "conclude", "merged"), batchID, queue, concludeMeta); err != nil { + if err := c.publish(ctx, topickey.TopicKeyConclude, publish.IntentID(batchID, "conclude", "landed"), batchID, queue, concludeMeta); err != nil { metrics.NamedCounter(c.metricsScope, "process", "publish_conclude_errors", 1) return fmt.Errorf("failed to publish to conclude: %w", err) } - if err := c.publish(ctx, topickey.TopicKeySpeculate, publish.IntentID(batchID, "merged"), batchID, queue, nil); err != nil { + if err := c.publish(ctx, topickey.TopicKeySpeculate, publish.IntentID(batchID, "landed"), batchID, queue, nil); err != nil { metrics.NamedCounter(c.metricsScope, "process", "publish_speculate_errors", 1) return fmt.Errorf("failed to publish to speculate: %w", err) } @@ -216,7 +216,7 @@ func (c *Controller) publish(ctx context.Context, key consumer.TopicKey, msgID, // Name returns the controller name for logging and metrics. func (c *Controller) Name() string { - return "mergesignal" + return "landsignal" } // TopicKey returns the topic key this controller subscribes to. diff --git a/submitqueue/orchestrator/controller/mergesignal/mergesignal_test.go b/submitqueue/orchestrator/controller/landsignal/landsignal_test.go similarity index 95% rename from submitqueue/orchestrator/controller/mergesignal/mergesignal_test.go rename to submitqueue/orchestrator/controller/landsignal/landsignal_test.go index 73b78c3e3..54626b89e 100644 --- a/submitqueue/orchestrator/controller/mergesignal/mergesignal_test.go +++ b/submitqueue/orchestrator/controller/landsignal/landsignal_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package mergesignal +package landsignal import ( "context" @@ -100,7 +100,7 @@ func newController(t *testing.T, store *storagemock.MockStorage, registry consum staticStorageFactory{store: store}, registry, runwaymq.TopicKeyMergeSignal, - "orchestrator-mergesignal", + "orchestrator-landsignal", ) } @@ -112,12 +112,12 @@ func TestNewController(t *testing.T) { c := newController(t, store, recordingRegistry(t, ctrl, &got)) assert.Equal(t, consumer.TopicKey(runwaymq.TopicKeyMergeSignal), c.TopicKey()) - assert.Equal(t, "orchestrator-mergesignal", c.ConsumerGroup()) - assert.Equal(t, "mergesignal", c.Name()) + assert.Equal(t, "orchestrator-landsignal", c.ConsumerGroup()) + assert.Equal(t, "landsignal", c.Name()) var _ consumer.Controller = c } -func TestProcess_MergedAdvancesBatch(t *testing.T) { +func TestProcess_LandedAdvancesBatch(t *testing.T) { ctrl := gomock.NewController(t) batchStore := storagemock.NewMockBatchStore(ctrl) @@ -126,7 +126,7 @@ func TestProcess_MergedAdvancesBatch(t *testing.T) { Queue: testQueue, Contains: []string{"test-queue/1"}, Dependencies: []string{"test-queue/batch/0"}, - State: entity.BatchStateMerging, + State: entity.BatchStateLanding, Version: 1, } batchStore.EXPECT().Get(gomock.Any(), testBatchID).Return(batch, nil) @@ -152,7 +152,7 @@ func TestProcess_MergedAdvancesBatch(t *testing.T) { assert.ElementsMatch(t, []string{"conclude", "speculate"}, got) } -// The fan-out after a merge must not reuse the bare batch ID. +// The fan-out after a land must not reuse the bare batch ID. // // The batch's own announcement to speculate uses exactly that ID, and the // queue deduplicates on (topic, partition key, message ID) against every row it @@ -169,7 +169,7 @@ func TestProcess_FanoutDoesNotCollideWithTheBatchAnnouncement(t *testing.T) { Queue: testQueue, Contains: []string{"test-queue/1"}, Dependencies: []string{"test-queue/batch/0"}, - State: entity.BatchStateMerging, + State: entity.BatchStateLanding, Version: 1, } batchStore.EXPECT().Get(gomock.Any(), testBatchID).Return(batch, nil) @@ -205,7 +205,7 @@ func TestProcess_FanoutDoesNotCollideWithTheBatchAnnouncement(t *testing.T) { assert.NotEqual(t, byTopic["speculate"], byTopic["conclude"]) } -func TestProcess_NotMergedMarksBatchFailed(t *testing.T) { +func TestProcess_NotLandedMarksBatchFailed(t *testing.T) { ctrl := gomock.NewController(t) batchStore := storagemock.NewMockBatchStore(ctrl) @@ -214,7 +214,7 @@ func TestProcess_NotMergedMarksBatchFailed(t *testing.T) { Queue: testQueue, Contains: []string{"test-queue/1"}, Dependencies: []string{"test-queue/batch/0"}, - State: entity.BatchStateMerging, + State: entity.BatchStateLanding, Version: 3, } batchStore.EXPECT().Get(gomock.Any(), testBatchID).Return(batch, nil) @@ -244,12 +244,12 @@ func TestProcess_NotMergedMarksBatchFailed(t *testing.T) { res := runwaymq.MergeResult{Id: testBatchID, Outcome: runwaypb.Outcome_FAILED, Reason: "conflict in foo.go"} msg := entityqueue.NewMessage(testBatchID, resultPayload(t, res), testQueue, nil) - // Not-merged is an expected terminal outcome, so Process acks (no error). + // Not-landed is an expected terminal outcome, so Process acks (no error). require.NoError(t, newController(t, store, registry).Process(context.Background(), newDelivery(ctrl, msg))) require.Contains(t, byTopic, "conclude") require.Contains(t, byTopic, "speculate") - // The merge reason rides the conclude message so conclude can stamp it on + // The land failure reason rides the conclude message so conclude can stamp it on // the request's terminal log; the speculate wake-up carries none. assert.Equal(t, "conflict in foo.go", byTopic["conclude"].Metadata[topickey.MetadataKeyFailureReason]) assert.Empty(t, byTopic["speculate"].Metadata[topickey.MetadataKeyFailureReason]) diff --git a/submitqueue/orchestrator/controller/speculate/check.go b/submitqueue/orchestrator/controller/speculate/check.go index 14528c913..3e0607ba4 100644 --- a/submitqueue/orchestrator/controller/speculate/check.go +++ b/submitqueue/orchestrator/controller/speculate/check.go @@ -47,10 +47,10 @@ const ( // each drop. // // The Speculator is an extension, so its output is untrusted input: it decides -// which paths run, never whether a batch merges or fails. Every rule here +// which paths run, never whether a batch lands or fails. Every rule here // protects an invariant the extension could otherwise break — acting on a batch // that is finalizing, resurrecting a path a resolved dependency has ruled out, -// or discarding a passed build the queue is about to merge on. A proposal that +// or discarding a passed build the queue is about to use for landing. A proposal that // trips one of these is a bug in the Speculator, not a normal outcome, which is // why the caller counts them. func filterProposals(proposals []entity.Speculation, snap snapshot) ([]entity.Speculation, []rejection) { @@ -130,9 +130,9 @@ func rejectionReason(proposal entity.Speculation, snap snapshot) (rejection, boo // combination its assumptions name. With the length check and position-wise // equality, a missing, extra, or duplicate dependency is also impossible. // -// A malformed path is not merely suboptimal, it is unmergeable — the merge -// preconditions are read off the path's assumptions (see mergeablePath), so a -// path missing a dependency would let its head merge without waiting for it. +// A malformed path is not merely suboptimal, it is unlandable — the land +// preconditions are read off the path's assumptions (see landablePath), so a +// path missing a dependency would let its head land without waiting for it. func isWellFormed(path entity.SpeculationPath, head entity.Batch) bool { if path.Head != head.ID { return false diff --git a/submitqueue/orchestrator/controller/speculate/check_test.go b/submitqueue/orchestrator/controller/speculate/check_test.go index ee2e25f5f..3700ff56a 100644 --- a/submitqueue/orchestrator/controller/speculate/check_test.go +++ b/submitqueue/orchestrator/controller/speculate/check_test.go @@ -82,9 +82,9 @@ func TestFilterProposals_Rejects(t *testing.T) { want: rejectUnknownHead, }, { - name: "head already merging", + name: "head already landing", proposal: entity.Speculation{Path: valid, Action: entity.PathActionBuild}, - snap: checkSnapshot(entity.BatchStateMerging), + snap: checkSnapshot(entity.BatchStateLanding), want: rejectHeadNotSpeculating, }, { diff --git a/submitqueue/orchestrator/controller/speculate/dispatch.go b/submitqueue/orchestrator/controller/speculate/dispatch.go index c489ace90..5e3883f87 100644 --- a/submitqueue/orchestrator/controller/speculate/dispatch.go +++ b/submitqueue/orchestrator/controller/speculate/dispatch.go @@ -34,7 +34,7 @@ import ( // // It walks every in-flight batch, not only the speculating ones. Proposals // apply to speculating heads alone and are simply absent for the rest, but -// observations are not: a merging or cancelling head's paths keep holding CI +// observations are not: a landing or cancelling head's paths keep holding CI // slots until their builds stop, and this is the only writer that can record // that they have. Batches already finalized arrive here clean — // commitOutcome persisted their set with their outcome — so only their diff --git a/submitqueue/orchestrator/controller/speculate/doc.go b/submitqueue/orchestrator/controller/speculate/doc.go index a8927cdbb..79ec04ee7 100644 --- a/submitqueue/orchestrator/controller/speculate/doc.go +++ b/submitqueue/orchestrator/controller/speculate/doc.go @@ -23,8 +23,8 @@ // Batches in a queue depend on the batches ahead of them, so without // speculation everything is serial: C waits for B, B waits for A. Speculation // builds a batch against a guess about how its dependencies turn out. When -// the guess holds, the batch merges the moment the guessed-on dependencies -// land. If passed paths cover every possible outcome, the batch can merge +// the guess holds, the batch lands the moment the guessed-on dependencies +// land. If passed paths cover every possible outcome, the batch can land // before those dependencies settle. // // # Paths @@ -48,11 +48,11 @@ // Fund both and every future is covered: // // - While A is still unresolved, both P1 and P2 passing lets B bypass A and -// merge immediately: either possible future has already been validated. -// - A succeeds and P1 passed: B merges the moment A lands. P2's guess +// land immediately: either possible future has already been validated. +// - A succeeds and P1 passed: B lands the moment A lands. P2's guess // ("A fails") is broken — it can no longer come true — so its build is // cancelled to free the slot. -// - A fails and P2 passed: B merges without A, again with no new build. +// - A fails and P2 passed: B lands without A, again with no new build. // P1's guess is broken. // - A resolved either way, and every unbroken path failed: no future // remains in which B passes, so B fails. @@ -79,7 +79,7 @@ // until its build stops. A path is broken once a dependency's actual result // proves one of its assumptions wrong: its guess can no longer come true, so // its build is cancelled to free the slot. A path is superseded when its head -// becomes mergeable, so any still-running siblings are cancelled too. +// becomes landable, so any still-running siblings are cancelled too. // // Cancelling is intent, not fact: the build keeps its slot until CI actually // stops it, and only an observation of that stop (or proof nothing was ever @@ -91,7 +91,7 @@ // // # The life of a batch, as seen from here // -// Created ──admit──► Speculating ──┬── merge or bypass ──► Merging +// Created ──admit──► Speculating ──┬── land or bypass ──► Landing // └── fail ─────────────► Failed // user cancel (cancel stage): // ... ──► Cancelling ── every path stopped ──► Cancelled diff --git a/submitqueue/orchestrator/controller/speculate/finalize.go b/submitqueue/orchestrator/controller/speculate/finalize.go index d823c93e5..0b08f8fa3 100644 --- a/submitqueue/orchestrator/controller/speculate/finalize.go +++ b/submitqueue/orchestrator/controller/speculate/finalize.go @@ -33,11 +33,11 @@ import ( // snap.speculating holding the heads still open to new work. // // Everything here is a fact, not a choice: a path a resolved dependency ruled -// out is dead, a head whose passed builds establish a merge verdict merges, +// out is dead, a head whose passed builds establish a land verdict lands, // and a batch the user cancelled is finished once its last build stops. // Finalizing before the Speculator is asked is what keeps its work from // being wasted — asked first, it would propose builds for a head that is -// already merging. +// already landing. // // Outcomes cascade, and the head loop runs to a fixed point to collapse a // whole cascade into this one run: @@ -55,7 +55,7 @@ import ( // Deciding the whole cascade up front and writing afterwards would enact // dependents of an outcome whose own write then lost its compare-and-swap — // and the loser of that race is not always benign: a cancellation loses -// precisely to a merge that got there first, which leaves the batch +// precisely to a land that got there first, which leaves the batch // *succeeded*, after its dependents were already failed on the assumption it // was cancelled. Committing per generation costs no extra reads — the // snapshot is read once, and the writes are ones this run makes anyway. @@ -95,11 +95,11 @@ func (c *Controller) finalize(ctx context.Context, snap *snapshot) error { } bypassed := false - if decision == outcomeMerge { + if decision == outcomeLand { // The winning path carries the head out of the queue; its // siblings cannot help it any more and are still holding CI // slots the rest of the queue could use. - winner, ok := mergeablePath(set, *snap) + winner, ok := landablePath(set, *snap) if !ok { winner, bypassed = bypassablePath(batch, set, *snap) } @@ -159,8 +159,8 @@ func (c *Controller) reportSpeculation( ) error { after, hasPassed := livePassedPath(set, snap) - // A merge is decided on the same live passed path, so an ungated report - // would claim a wait on every head that merges straight through. + // A land is decided on the same live passed path, so an ungated report + // would claim a wait on every head that lands straight through. event, path := entity.RequestEventWaiting, after switch { case hasPassed && decision == outcomeWait: @@ -195,7 +195,7 @@ func (c *Controller) reportSpeculation( // moment it is asked to be — its builds hold their CI slots until they // actually stop — so the run marks the paths, the poll loop asks the runner // to stop them, and whichever later run sees them stopped finishes the -// job. That is why cancellation is best effort, and why a merge that wins the +// job. That is why cancellation is best effort, and why a land that wins the // race still prevails. func (c *Controller) finalizeCancellations(ctx context.Context, snap *snapshot, nowMs int64) error { // TODO(respeculate-collateral): re-enqueue Land for every request in batch.Contains @@ -283,9 +283,9 @@ func (c *Controller) commitOutcome(ctx context.Context, snap *snapshot, batch en // durable — see commitOutcome — because everything concluded about the // batches stacked on this one is derived from it. // -// Only a terminal outcome is recorded. Merging is not terminal — a head +// Only a terminal outcome is recorded. Landing is not terminal — a head // stacked on this one assumed it would *succeed*, and it has not yet — so a -// merge outcome resolves nothing for anybody else. +// land outcome resolves nothing for anybody else. func (c *Controller) recordOutcome(snap *snapshot, batchID string, decision outcome) { state, terminal := decision.terminalState() if !terminal { @@ -307,8 +307,8 @@ func (c *Controller) applyOutcome(ctx context.Context, store storage.Storage, ba var state entity.BatchState switch decision { - case outcomeMerge: - state = entity.BatchStateMerging + case outcomeLand: + state = entity.BatchStateLanding case outcomeFail, outcomeCancel: state, _ = decision.terminalState() @@ -347,8 +347,8 @@ func (c *Controller) applyOutcome(ctx context.Context, store storage.Storage, ba ) switch decision { - case outcomeMerge: - if err := c.dispatchMerge(ctx, batch); err != nil { + case outcomeLand: + if err := c.dispatchLand(ctx, batch); err != nil { return true, err } @@ -362,7 +362,7 @@ func (c *Controller) applyOutcome(ctx context.Context, store storage.Storage, ba } // Named for the run that decided it, so a redelivery re-deriving the // same outcome does not conclude the batch twice, and so it stays - // distinct from the conclude mergesignal sends for a merged batch. + // distinct from the conclude landsignal sends for a landed batch. // A conclude that goes missing is recovered by fanout, which is // deliberately un-deduplicated. if err := c.publishBatchIDWithMetadata(ctx, topickey.TopicKeyConclude, publish.IntentID(batch.ID, "conclude", "speculate"), batch.ID, batch.Queue, batch.Queue, concludeMeta); err != nil { @@ -373,11 +373,11 @@ func (c *Controller) applyOutcome(ctx context.Context, store storage.Storage, ba return true, nil } -// dispatchMerge reports speculation finished and hands the batch to the merge -// stage. The stable ID means a redelivery or the Merging self-heal dedupes -// against the request already sent instead of merging twice; the status goes +// dispatchLand reports speculation finished and hands the batch to the land +// stage. The stable ID means a redelivery or the Landing self-heal dedupes +// against the request already sent instead of landing twice; the status goes // first so it cannot be timestamped after the landing the dispatch triggers. -func (c *Controller) dispatchMerge(ctx context.Context, batch entity.Batch) error { +func (c *Controller) dispatchLand(ctx context.Context, batch entity.Batch) error { if err := corerequest.PublishBatchLogs(ctx, c.registry, batch.Queue, batch.Contains, entity.RequestStatusSpeculated, batch.ID, map[string]string{"batch_id": batch.ID}, ); err != nil { @@ -385,9 +385,9 @@ func (c *Controller) dispatchMerge(ctx context.Context, batch entity.Batch) erro return fmt.Errorf("failed to publish request logs for batch %s: %w", batch.ID, err) } - if err := c.publishBatchID(ctx, topickey.TopicKeyMerge, publish.IntentID(batch.ID, "merge-dispatch"), batch.ID, batch.Queue, batch.Queue); err != nil { + if err := c.publishBatchID(ctx, topickey.TopicKeyLand, publish.IntentID(batch.ID, "land-dispatch"), batch.ID, batch.Queue, batch.Queue); err != nil { metrics.NamedCounter(c.metricsScope, opName, "publish_errors", 1) - return fmt.Errorf("failed to publish batch %s to merge: %w", batch.ID, err) + return fmt.Errorf("failed to publish batch %s to land: %w", batch.ID, err) } return nil } @@ -399,7 +399,7 @@ func (c *Controller) dispatchMerge(ctx context.Context, batch entity.Batch) erro // A batch named by a message is repaired through it: a redelivery re-publishes // from one of Process's self-heal branches, and a persistent failure // dead-letters by name. A cascade-decided batch has neither, and finalize only -// walks heads still speculating — so a merged one would never reach Runway, +// walks heads still speculating — so a batch in Landing would never reach Runway, // and a terminal one would leave its requests unreconciled. // // Distinct per publish: the guarantee being bought is that a message exists at @@ -456,7 +456,7 @@ func cancelBrokenPathsInSet(set *entity.SpeculationPathSet, snap snapshot, nowMs }) } -// supersede stops every path other than the winner once the head can merge. +// supersede stops every path other than the winner once the head can land. // Its live siblings cannot help any more but still hold CI slots the rest of // the queue could use. func supersede(set *entity.SpeculationPathSet, winnerID string, nowMs int64) bool { diff --git a/submitqueue/orchestrator/controller/speculate/outcome.go b/submitqueue/orchestrator/controller/speculate/outcome.go index cd6c0c2c7..c7a64a525 100644 --- a/submitqueue/orchestrator/controller/speculate/outcome.go +++ b/submitqueue/orchestrator/controller/speculate/outcome.go @@ -25,10 +25,10 @@ type outcome string const ( // outcomeWait means the batch's outcome is not decided yet. outcomeWait outcome = "wait" - // outcomeMerge means the head can be handed to the merge stage: either a + // outcomeLand means the head can be handed to the land stage: either a // passed path's assumptions have all come true, or passed paths cover every // possible outcome of its unsettled dependencies. - outcomeMerge outcome = "merge" + outcomeLand outcome = "land" // outcomeFail means no future remains in which the head could pass. outcomeFail outcome = "fail" // outcomeCancel means a batch the user asked to cancel has had every path @@ -37,8 +37,8 @@ const ( ) // terminalState returns the batch state an outcome writes, and whether the -// outcome leaves the batch terminal. Merge is the odd one out: it hands the -// batch to the merge stage, which owns the terminal write that follows. +// outcome leaves the batch terminal. Land is the odd one out: it hands the +// batch to the land stage, which owns the terminal write that follows. func (v outcome) terminalState() (entity.BatchState, bool) { switch v { case outcomeFail: @@ -52,11 +52,11 @@ func (v outcome) terminalState() (entity.BatchState, bool) { // decide returns the run's outcome on one open head, from the snapshot alone. func decide(head entity.Batch, set entity.SpeculationPathSet, snap snapshot) outcome { - if _, ok := mergeablePath(set, snap); ok { - return outcomeMerge + if _, ok := landablePath(set, snap); ok { + return outcomeLand } if _, ok := bypassablePath(head, set, snap); ok { - return outcomeMerge + return outcomeLand } if hasNoViableFuture(head, set, snap) { return outcomeFail @@ -64,23 +64,23 @@ func decide(head entity.Batch, set entity.SpeculationPathSet, snap snapshot) out return outcomeWait } -// mergeablePath returns a passed path whose merge preconditions are met: every +// landablePath returns a passed path whose land preconditions are met: every // guess it made about a dependency has been borne out by that dependency's // actual state. // // This is what makes speculation pay — not by shortening the list the head // waits on, but by having already done the work. The build ran against the // guess while the dependencies were still resolving, so when they land the way -// the path assumed there is nothing left to run and the head merges at once. +// the path assumed there is nothing left to run and the head lands at once. // -// A guess that has not been settled yet is not a licence to merge, whichever +// A guess that has not been settled yet is not a licence to land, whichever // way it points. A path that assumed a dependency would fail was built without // that dependency's changes, so landing it while the dependency is still live // puts a combination on the trunk that no build ever validated — which is the -// one thing the queue exists to prevent. The dependency merging is not enough -// either: a merge can fail, so "on its way in" is still an open question, and +// one thing the queue exists to prevent. The dependency landing is not enough +// either: a land can fail, so "on its way in" is still an open question, and // the head waits for the answer. -func mergeablePath(set entity.SpeculationPathSet, snap snapshot) (entity.SpeculationPathEntry, bool) { +func landablePath(set entity.SpeculationPathSet, snap snapshot) (entity.SpeculationPathEntry, bool) { for _, entry := range set.Paths { if entry.Status != entity.SpeculationPathStatusPassed { continue @@ -115,10 +115,10 @@ func passedEntry(set entity.SpeculationPathSet) (entity.SpeculationPathEntry, bo // consistent with how its dependencies are resolving, whether or not they have // finished resolving. // -// It is mergeablePath without the settled requirement, and the difference +// It is landablePath without the settled requirement, and the difference // between the two is exactly the head's waiting room: work this head had to do // is done, and all that is left is other batches finishing. Reported rather -// than acted on — nothing may merge on a path this loose, and decide is +// than acted on — nothing may land on a path this loose, and decide is // deliberately not built on it. func livePassedPath(set entity.SpeculationPathSet, snap snapshot) (entity.SpeculationPathEntry, bool) { for _, entry := range set.Paths { diff --git a/submitqueue/orchestrator/controller/speculate/outcome_test.go b/submitqueue/orchestrator/controller/speculate/outcome_test.go index f1b6fd591..e14dced6e 100644 --- a/submitqueue/orchestrator/controller/speculate/outcome_test.go +++ b/submitqueue/orchestrator/controller/speculate/outcome_test.go @@ -30,10 +30,10 @@ func setOf(entries ...entity.SpeculationPathEntry) entity.SpeculationPathSet { return entity.SpeculationPathSet{Head: head, Paths: entries} } -// The payoff case: a head merges as soon as the dependencies its passed build +// The payoff case: a head lands as soon as the dependencies its passed build // was stacked on have landed, without waiting for the ones it was built // without or told to ignore. -func TestMergeablePath(t *testing.T) { +func TestLandablePath(t *testing.T) { const ( succeeds = entity.DependencyAssumptionSucceeds fails = entity.DependencyAssumptionFails @@ -49,23 +49,23 @@ func TestMergeablePath(t *testing.T) { // dep2 is settled the way its assumption expects throughout, so dep1 // is the only thing under test. { - name: "waits for an assumed-succeeding dependency to merge", + name: "waits for an assumed-succeeding dependency to land", assumption: [2]entity.DependencyAssumption{succeeds, fails}, dep1State: entity.BatchStateSpeculating, dep2State: entity.BatchStateFailed, want: false, }, { - name: "merges once it has", + name: "lands once it has", assumption: [2]entity.DependencyAssumption{succeeds, fails}, dep1State: entity.BatchStateSucceeded, dep2State: entity.BatchStateFailed, want: true, }, { - name: "an assumed-succeeding dependency waits out its merge", + name: "an assumed-succeeding dependency waits out its land", assumption: [2]entity.DependencyAssumption{succeeds, fails}, - dep1State: entity.BatchStateMerging, + dep1State: entity.BatchStateLanding, dep2State: entity.BatchStateFailed, want: false, }, @@ -77,9 +77,9 @@ func TestMergeablePath(t *testing.T) { want: false, }, { - name: "still waits while that dependency is merging", + name: "still waits while that dependency is landing", assumption: [2]entity.DependencyAssumption{fails, fails}, - dep1State: entity.BatchStateMerging, + dep1State: entity.BatchStateLanding, dep2State: entity.BatchStateFailed, want: false, }, @@ -91,14 +91,14 @@ func TestMergeablePath(t *testing.T) { want: false, }, { - name: "merges once it has failed", + name: "lands once it has failed", assumption: [2]entity.DependencyAssumption{fails, fails}, dep1State: entity.BatchStateFailed, dep2State: entity.BatchStateFailed, want: true, }, { - name: "merges once it has been cancelled", + name: "lands once it has been cancelled", assumption: [2]entity.DependencyAssumption{fails, fails}, dep1State: entity.BatchStateCancelled, dep2State: entity.BatchStateFailed, @@ -120,14 +120,14 @@ func TestMergeablePath(t *testing.T) { dep2 = entity.BatchStateSpeculating } set := setOf(passedPath(tt.assumption[0], tt.assumption[1])) - _, ok := mergeablePath(set, snapWith(tt.dep1State, dep2)) + _, ok := landablePath(set, snapWith(tt.dep1State, dep2)) assert.Equal(t, tt.want, ok) }) } } // A path that has not passed cannot carry the head out of the queue. -func TestMergeablePath_IgnoresUnpassedPaths(t *testing.T) { +func TestLandablePath_IgnoresUnpassedPaths(t *testing.T) { for _, status := range []entity.SpeculationPathStatus{ entity.SpeculationPathStatusPending, entity.SpeculationPathStatusBuilding, @@ -138,19 +138,19 @@ func TestMergeablePath_IgnoresUnpassedPaths(t *testing.T) { t.Run(string(status), func(t *testing.T) { set := setOf(entryFor( pathOver(entity.DependencyAssumptionSucceeds, entity.DependencyAssumptionSucceeds), status)) - _, ok := mergeablePath(set, snapWith(entity.BatchStateSucceeded, entity.BatchStateSucceeded)) + _, ok := landablePath(set, snapWith(entity.BatchStateSucceeded, entity.BatchStateSucceeded)) assert.False(t, ok) }) } } // A passed build whose assumptions reality has since contradicted is not a -// licence to merge — it verified a world that did not happen. -func TestMergeablePath_ExcludesBrokenPassedPath(t *testing.T) { +// licence to land — it verified a world that did not happen. +func TestLandablePath_ExcludesBrokenPassedPath(t *testing.T) { set := setOf(passedPath(entity.DependencyAssumptionFails, entity.DependencyAssumptionFails)) // The path was built without dep1, but dep1 landed after all. - _, ok := mergeablePath(set, snapWith(entity.BatchStateSucceeded, entity.BatchStateSpeculating)) + _, ok := landablePath(set, snapWith(entity.BatchStateSucceeded, entity.BatchStateSpeculating)) assert.False(t, ok) } @@ -180,7 +180,7 @@ func TestBypassablePath(t *testing.T) { head: headBatch, set: setOf(allPaths...), dep1State: entity.BatchStateSpeculating, - dep2State: entity.BatchStateMerging, + dep2State: entity.BatchStateLanding, want: true, }, { @@ -266,7 +266,7 @@ func TestBypassablePath(t *testing.T) { dep2State: entity.BatchStateSpeculating, }, { - name: "leaves fully settled dependencies to strict merge", + name: "leaves fully settled dependencies to strict land", head: headBatch, set: setOf(allPaths...), dep1State: entity.BatchStateSucceeded, @@ -285,10 +285,10 @@ func TestBypassablePath(t *testing.T) { } } -// livePassedPath is mergeablePath without the settled requirement, and the gap +// livePassedPath is landablePath without the settled requirement, and the gap // between the two is the head's waiting room: its own work is done and all that // remains is other batches finishing. That window is reported to the members, -// so it has to be recognised while mergeablePath still says no. +// so it has to be recognised while landablePath still says no. func TestLivePassedPath(t *testing.T) { const ( succeeds = entity.DependencyAssumptionSucceeds @@ -404,12 +404,12 @@ func TestDecide(t *testing.T) { pathOver(entity.DependencyAssumptionSucceeds, entity.DependencyAssumptionFails), entity.SpeculationPathStatusBuilding) - assert.Equal(t, outcomeMerge, decide(headBatch, setOf(passed), allResolved)) + assert.Equal(t, outcomeLand, decide(headBatch, setOf(passed), allResolved)) assert.Equal(t, outcomeFail, decide(headBatch, setOf(failed), allResolved)) assert.Equal(t, outcomeWait, decide(headBatch, setOf(building), allResolved)) // A passed path wins over a failed sibling: one way through is enough. - assert.Equal(t, outcomeMerge, decide(headBatch, setOf(failed, passed), allResolved)) + assert.Equal(t, outcomeLand, decide(headBatch, setOf(failed, passed), allResolved)) allUnresolved := snapWith(entity.BatchStateSpeculating, entity.BatchStateSpeculating) fullCoverage := setOf( @@ -418,7 +418,7 @@ func TestDecide(t *testing.T) { passedPath(entity.DependencyAssumptionFails, entity.DependencyAssumptionSucceeds), passedPath(entity.DependencyAssumptionFails, entity.DependencyAssumptionFails), ) - assert.Equal(t, outcomeMerge, decide(headBatch, fullCoverage, allUnresolved)) + assert.Equal(t, outcomeLand, decide(headBatch, fullCoverage, allUnresolved)) } // Once a path has passed, its siblings cannot help the head but are still diff --git a/submitqueue/orchestrator/controller/speculate/run.go b/submitqueue/orchestrator/controller/speculate/run.go index afc642cce..dba882c63 100644 --- a/submitqueue/orchestrator/controller/speculate/run.go +++ b/submitqueue/orchestrator/controller/speculate/run.go @@ -49,7 +49,7 @@ func (c *Controller) run(ctx context.Context, store storage.Storage, trigger ent if len(snap.speculating) == 0 { // No head is open to new work, so there is nothing to ask the // Speculator. The dispatch step still runs: what the build stages saw about a - // merging or cancelling head's paths has to be persisted so those + // landing or cancelling head's paths has to be persisted so those // paths stop counting against the budget. return c.dispatch(ctx, trigger.Queue, snap, nil) } @@ -292,7 +292,7 @@ func terminalPathStatus(status entity.BuildStatus) entity.SpeculationPathStatus // head's dependencies are the facts its paths are built from, so a dependency // withheld is one the Speculator has to plan around blind. Every in-flight // path set goes over for the same reason: a path holds its CI slot until its -// build actually stops, so a merging head's superseded siblings and a +// build actually stops, so a landing head's superseded siblings and a // cancelling head's live builds spend the budget just like a speculating // head's do. Hiding either would let the allocator count occupied slots as // free and oversubscribe CI. @@ -301,7 +301,7 @@ func terminalPathStatus(status entity.BuildStatus) entity.SpeculationPathStatus // head, and check rejects any proposal aimed at a head that is not // speculating. A head this run has just decided still reads as Speculating // here, but it can only rebuild the path that already passed — see -// mergeablePath — which the allocator skips as finished. +// landablePath — which the allocator skips as finished. func (c *Controller) ask(ctx context.Context, queue string, snap snapshot) ([]entity.Speculation, error) { spec, err := c.speculators.For(speculator.Config{QueueName: queue}) if err != nil { diff --git a/submitqueue/orchestrator/controller/speculate/run_test.go b/submitqueue/orchestrator/controller/speculate/run_test.go index 2c96b74e8..29afa7745 100644 --- a/submitqueue/orchestrator/controller/speculate/run_test.go +++ b/submitqueue/orchestrator/controller/speculate/run_test.go @@ -187,7 +187,7 @@ func newRunHarness(t *testing.T, ctrl *gomock.Controller, spec *scriptedSpeculat registry, err := consumer.NewTopicRegistry([]consumer.TopicConfig{ {Key: topickey.TopicKeyBuild, Name: "build", Queue: q}, - {Key: topickey.TopicKeyMerge, Name: "submitqueue-merge", Queue: q}, + {Key: topickey.TopicKeyLand, Name: "submitqueue-land", Queue: q}, {Key: topickey.TopicKeyConclude, Name: "conclude", Queue: q}, {Key: topickey.TopicKeySpeculate, Name: "speculate", Queue: q}, {Key: topickey.TopicKeyLog, Name: "log", Queue: q}, @@ -279,20 +279,20 @@ func TestRun_PassesSnapshotToSpeculator(t *testing.T) { ctrl := gomock.NewController(t) spec := &scriptedSpeculator{} - merging := entity.Batch{ID: "q/batch/merging", Queue: "q", State: entity.BatchStateMerging, Version: 1} - h := newRunHarness(t, ctrl, spec, []entity.Batch{speculatingHead(), merging}) + landing := entity.Batch{ID: "q/batch/landing", Queue: "q", State: entity.BatchStateLanding, Version: 1} + h := newRunHarness(t, ctrl, spec, []entity.Batch{speculatingHead(), landing}) h.noBuildsDispatched() h.batches.EXPECT().Get(gomock.Any(), dep1).Return(entity.Batch{ID: dep1, State: entity.BatchStateSucceeded}, nil) h.batches.EXPECT().Get(gomock.Any(), dep2).Return(entity.Batch{ID: dep2, State: entity.BatchStateSpeculating}, nil) existing := entity.SpeculationPathSet{Head: head, Version: 3} h.pathSets.EXPECT().Get(gomock.Any(), head).Return(existing, nil) - h.pathSets.EXPECT().Get(gomock.Any(), merging.ID).Return(entity.SpeculationPathSet{}, storage.ErrNotFound) + h.pathSets.EXPECT().Get(gomock.Any(), landing.ID).Return(entity.SpeculationPathSet{}, storage.ErrNotFound) require.NoError(t, h.run(head)) require.Equal(t, 1, spec.calls) - assert.ElementsMatch(t, []string{dep1, dep2, head, merging.ID}, h.speculatedOver(), + assert.ElementsMatch(t, []string{dep1, dep2, head, landing.ID}, h.speculatedOver(), "every batch the run read, in no particular order") require.Len(t, spec.gotSets, 1) assert.Equal(t, int32(3), spec.gotSets[0].Version) @@ -525,7 +525,7 @@ func TestRun_DoesNotReReadFinishedPaths(t *testing.T) { spec := &scriptedSpeculator{} h := newRunHarness(t, ctrl, spec, []entity.Batch{speculatingHead()}) - // dep1 is unresolved, so the passed path cannot merge — this test is about + // dep1 is unresolved, so the passed path cannot land — this test is about // observation being skipped, not about outcomes. h.batches.EXPECT().Get(gomock.Any(), dep1).Return(entity.Batch{ID: dep1, State: entity.BatchStateSpeculating}, nil) h.batches.EXPECT().Get(gomock.Any(), dep2).Return(entity.Batch{ID: dep2, State: entity.BatchStateFailed}, nil) @@ -633,7 +633,7 @@ func TestRun_DecidedHeadIsNotSpeculatedOn(t *testing.T) { h := newRunHarness(t, ctrl, spec, []entity.Batch{speculatingHead()}) h.noBuildsDispatched() - // Both dependencies resolved the way the passed path assumed, so it merges. + // Both dependencies resolved the way the passed path assumed, so it lands. h.batches.EXPECT().Get(gomock.Any(), dep1).Return(entity.Batch{ID: dep1, State: entity.BatchStateSucceeded}, nil) h.batches.EXPECT().Get(gomock.Any(), dep2).Return(entity.Batch{ID: dep2, State: entity.BatchStateFailed}, nil) h.pathSets.EXPECT().Get(gomock.Any(), head).Return(entity.SpeculationPathSet{ @@ -642,26 +642,26 @@ func TestRun_DecidedHeadIsNotSpeculatedOn(t *testing.T) { Version: 1, }, nil) - // The head moves to merging. Its set is untouched: the only path is the + // The head moves to landing. Its set is untouched: the only path is the // winner, and no proposal was applied. h.batches.EXPECT(). - Update(gomock.Any(), updateTo{id: head, state: entity.BatchStateMerging}, int32(1), int32(2)). + Update(gomock.Any(), updateTo{id: head, state: entity.BatchStateLanding}, int32(1), int32(2)). Return(nil) require.NoError(t, h.run(head)) assert.Zero(t, spec.calls, "a decided head leaves nothing to speculate about") - assert.Equal(t, []string{"submitqueue-merge"}, h.published) - assert.Contains(t, h.filed, entity.QueueBatchState{Queue: "q", State: entity.BatchStateMerging, BatchID: head}, + assert.Equal(t, []string{"submitqueue-land"}, h.published) + assert.Contains(t, h.filed, entity.QueueBatchState{Queue: "q", State: entity.BatchStateLanding, BatchID: head}, "the outcome must file the head under its new state") assert.Contains(t, h.unfiled, entity.QueueBatchState{State: entity.BatchStateSpeculating, BatchID: head}, "and drop the record under the state it left") } -// The churn this ordering removes: a mergeable head must not gain a funded path +// The churn this ordering removes: a landable head must not gain a funded path // that the same run immediately cancels — the dispatch would have started CI // for work nothing waits for. -func TestRun_MergeableHeadGainsNoNewPath(t *testing.T) { +func TestRun_LandableHeadGainsNoNewPath(t *testing.T) { ctrl := gomock.NewController(t) passed := pathOver(entity.DependencyAssumptionSucceeds, entity.DependencyAssumptionFails) other := pathOver(entity.DependencyAssumptionFails, entity.DependencyAssumptionFails) @@ -689,13 +689,13 @@ func TestRun_MergeableHeadGainsNoNewPath(t *testing.T) { h.pathSets.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)). DoAndReturn(func(_ context.Context, s entity.SpeculationPathSet, _, _ int32) error { - require.Len(t, s.Paths, 2, "no path was funded for a head that is merging") + require.Len(t, s.Paths, 2, "no path was funded for a head that is landing") assert.Equal(t, entity.SpeculationPathStatusPassed, s.Paths[0].Status) assert.Equal(t, entity.SpeculationPathStatusCancelling, s.Paths[1].Status) return nil }) h.batches.EXPECT(). - Update(gomock.Any(), updateTo{id: head, state: entity.BatchStateMerging}, int32(1), int32(2)). + Update(gomock.Any(), updateTo{id: head, state: entity.BatchStateLanding}, int32(1), int32(2)). Return(nil) require.NoError(t, h.run(head)) @@ -705,7 +705,7 @@ func TestRun_MergeableHeadGainsNoNewPath(t *testing.T) { // The dispatch is what takes a batch out of this stage's hands, so sending it // before the write would let Runway act on an outcome a lost compare-and-swap // refused to record. -func TestRun_MergeableHeadDispatchesAfterTheStateWrite(t *testing.T) { +func TestRun_LandableHeadDispatchesAfterTheStateWrite(t *testing.T) { ctrl := gomock.NewController(t) passed := pathOver(entity.DependencyAssumptionSucceeds, entity.DependencyAssumptionSucceeds) @@ -721,21 +721,21 @@ func TestRun_MergeableHeadDispatchesAfterTheStateWrite(t *testing.T) { var publishedBeforeWrite []string h.batches.EXPECT(). - Update(gomock.Any(), updateTo{id: head, state: entity.BatchStateMerging}, int32(1), int32(2)). + Update(gomock.Any(), updateTo{id: head, state: entity.BatchStateLanding}, int32(1), int32(2)). DoAndReturn(func(context.Context, entity.Batch, int32, int32) error { publishedBeforeWrite = append([]string(nil), h.published...) return nil }) require.NoError(t, h.run(head)) - assert.Equal(t, []string{"submitqueue-merge"}, h.published) + assert.Equal(t, []string{"submitqueue-land"}, h.published) assert.Empty(t, publishedBeforeWrite, - "nothing may reach the merge stage before the state it acts on is written") + "nothing may reach the land stage before the state it acts on is written") } // The other half: a lost write means another writer owns the batch, so the // dispatch it would have justified is never sent. -func TestRun_MergeableHeadDispatchesNothingWhenTheStateCASLoses(t *testing.T) { +func TestRun_LandableHeadDispatchesNothingWhenTheStateCASLoses(t *testing.T) { ctrl := gomock.NewController(t) passed := pathOver(entity.DependencyAssumptionSucceeds, entity.DependencyAssumptionSucceeds) @@ -749,7 +749,7 @@ func TestRun_MergeableHeadDispatchesNothingWhenTheStateCASLoses(t *testing.T) { Version: 1, }, nil).AnyTimes() h.batches.EXPECT(). - Update(gomock.Any(), updateTo{id: head, state: entity.BatchStateMerging}, int32(1), int32(2)). + Update(gomock.Any(), updateTo{id: head, state: entity.BatchStateLanding}, int32(1), int32(2)). Return(storage.ErrVersionMismatch) require.NoError(t, h.run(head)) @@ -999,44 +999,44 @@ func TestRun_CancellingLostPathSetRaceSkipsTheTerminalWrite(t *testing.T) { } // The allocator rations a budget measured in occupied CI slots, and a path -// holds its slot until its build actually stops. A merging head's superseded +// holds its slot until its build actually stops. A landing head's superseded // siblings are still running, so hiding their set would let the allocator count // those slots as free and oversubscribe CI. func TestRun_SpeculatorSeesPathSetsOfNonOpenHeads(t *testing.T) { ctrl := gomock.NewController(t) spec := &scriptedSpeculator{} - merging := entity.Batch{ - ID: "q/batch/merging", Queue: "q", State: entity.BatchStateMerging, Version: 1, + landing := entity.Batch{ + ID: "q/batch/landing", Queue: "q", State: entity.BatchStateLanding, Version: 1, } open := entity.Batch{ID: head, Queue: "q", State: entity.BatchStateSpeculating, Version: 1} - h := newRunHarness(t, ctrl, spec, []entity.Batch{open, merging}) + h := newRunHarness(t, ctrl, spec, []entity.Batch{open, landing}) h.noBuildsDispatched() h.pathSets.EXPECT().Get(gomock.Any(), head). Return(entity.SpeculationPathSet{Head: head, Version: 1}, nil) - h.pathSets.EXPECT().Get(gomock.Any(), merging.ID).Return(entity.SpeculationPathSet{ - Head: merging.ID, + h.pathSets.EXPECT().Get(gomock.Any(), landing.ID).Return(entity.SpeculationPathSet{ + Head: landing.ID, Paths: []entity.SpeculationPathEntry{ {ID: "still-running", Status: entity.SpeculationPathStatusCancelling, Attempt: 1}, }, Version: 1, }, nil) - // The undispatched cancelling path is marked cancelled, so the merging head's set is + // The undispatched cancelling path is marked cancelled, so the landing head's set is // rewritten even though it is closed to new work. h.pathSets.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)). DoAndReturn(func(_ context.Context, s entity.SpeculationPathSet, _, _ int32) error { - assert.Equal(t, merging.ID, s.Head) + assert.Equal(t, landing.ID, s.Head) assert.Equal(t, entity.SpeculationPathStatusCancelled, s.Paths[0].Status) return nil }) require.NoError(t, h.run(head)) - assert.ElementsMatch(t, []entity.Batch{open, merging}, spec.gotBatches, + assert.ElementsMatch(t, []entity.Batch{open, landing}, spec.gotBatches, "a closed head is still a fact the open ones are planned against") require.Len(t, spec.gotSets, 2, "every in-flight path set counts against the budget") - assert.Equal(t, merging.ID, spec.gotSets[1].Head) + assert.Equal(t, landing.ID, spec.gotSets[1].Head) } // A queue whose only in-flight head is closed to new work still has to be @@ -1047,13 +1047,13 @@ func TestRun_PersistsObservationsWithNoOpenHead(t *testing.T) { ctrl := gomock.NewController(t) spec := &scriptedSpeculator{} - merging := entity.Batch{ - ID: "q/batch/merging", Queue: "q", State: entity.BatchStateMerging, Version: 1, + landing := entity.Batch{ + ID: "q/batch/landing", Queue: "q", State: entity.BatchStateLanding, Version: 1, } - h := newRunHarness(t, ctrl, spec, []entity.Batch{merging}) - h.pathSets.EXPECT().Get(gomock.Any(), merging.ID).Return(entity.SpeculationPathSet{ - Head: merging.ID, + h := newRunHarness(t, ctrl, spec, []entity.Batch{landing}) + h.pathSets.EXPECT().Get(gomock.Any(), landing.ID).Return(entity.SpeculationPathSet{ + Head: landing.ID, Paths: []entity.SpeculationPathEntry{ {ID: "p1", Status: entity.SpeculationPathStatusCancelling, Attempt: 1}, }, @@ -1110,9 +1110,9 @@ func TestFanout_MintsADistinctMessageIDPerPublish(t *testing.T) { assert.NotEqual(t, ids[0], ids[1]) } -// The merge dispatch is the mirror image: a batch merges once, so the ID has +// The land dispatch is the mirror image: a batch lands once, so the ID has // to be stable across the redelivery and the self-heal that both re-derive it. -func TestDispatchMerge_ReusesOneMessageIDPerBatch(t *testing.T) { +func TestDispatchLand_ReusesOneMessageIDPerBatch(t *testing.T) { ctrl := gomock.NewController(t) var ids []string @@ -1127,7 +1127,7 @@ func TestDispatchMerge_ReusesOneMessageIDPerBatch(t *testing.T) { q.EXPECT().Publisher().Return(pub).AnyTimes() registry, err := consumer.NewTopicRegistry([]consumer.TopicConfig{ - {Key: topickey.TopicKeyMerge, Name: "submitqueue-merge", Queue: q}, + {Key: topickey.TopicKeyLand, Name: "submitqueue-land", Queue: q}, }) require.NoError(t, err) @@ -1137,8 +1137,8 @@ func TestDispatchMerge_ReusesOneMessageIDPerBatch(t *testing.T) { ) batch := entity.Batch{ID: head, Queue: "q"} - require.NoError(t, c.dispatchMerge(context.Background(), batch)) - require.NoError(t, c.dispatchMerge(context.Background(), batch)) + require.NoError(t, c.dispatchLand(context.Background(), batch)) + require.NoError(t, c.dispatchLand(context.Background(), batch)) require.Len(t, ids, 2) assert.Equal(t, ids[0], ids[1]) @@ -1176,7 +1176,7 @@ func cascadePair(t *testing.T, ctrl *gomock.Controller, prerequisiteState entity // An outcome is only a fact once it commits. When the prerequisite's state write // loses its race, nothing derived from that outcome may be enacted — the winner // may have written something else entirely. A cancellation loses precisely to a -// merge that got there first, which leaves the batch succeeded, and a dependent +// land that got there first, which leaves the batch succeeded, and a dependent // broken by that success must not already have been failed on the assumption // it was cancelled. func TestRun_CascadeStopsWhenThePrerequisiteStateCASLoses(t *testing.T) { @@ -1343,26 +1343,26 @@ func TestRun_TriggerBatchNeedsNoRecoverySignal(t *testing.T) { assert.Equal(t, []string{"conclude"}, h.published) } -// Merging needs the same signal for the same reason: the write drops the batch +// Landing needs the same signal for the same reason: the write drops the batch // out of the speculating set, so nothing else would ever dispatch it. -func TestRun_CascadeDerivedBatchIsGivenARecoverySignalBeforeItMerges(t *testing.T) { +func TestRun_CascadeDerivedBatchIsGivenARecoverySignalBeforeItLands(t *testing.T) { ctrl := gomock.NewController(t) - merging := entity.Batch{ID: "q/batch/derived", Queue: "q", State: entity.BatchStateSpeculating, Version: 1} - h := newRunHarness(t, ctrl, &scriptedSpeculator{}, []entity.Batch{merging}) + landing := entity.Batch{ID: "q/batch/derived", Queue: "q", State: entity.BatchStateSpeculating, Version: 1} + h := newRunHarness(t, ctrl, &scriptedSpeculator{}, []entity.Batch{landing}) h.noBuildsDispatched() // No dependencies, so the passed path has nothing left to settle. - passedPath := entity.SpeculationPath{Head: merging.ID} - h.pathSets.EXPECT().Get(gomock.Any(), merging.ID).Return(entity.SpeculationPathSet{ - Head: merging.ID, + passedPath := entity.SpeculationPath{Head: landing.ID} + h.pathSets.EXPECT().Get(gomock.Any(), landing.ID).Return(entity.SpeculationPathSet{ + Head: landing.ID, Paths: []entity.SpeculationPathEntry{entryFor(passedPath, entity.SpeculationPathStatusPassed)}, Version: 1, }, nil).AnyTimes() var publishedBeforeWrite []string h.batches.EXPECT(). - Update(gomock.Any(), updateTo{id: merging.ID, state: entity.BatchStateMerging}, int32(1), int32(2)). + Update(gomock.Any(), updateTo{id: landing.ID, state: entity.BatchStateLanding}, int32(1), int32(2)). DoAndReturn(func(context.Context, entity.Batch, int32, int32) error { publishedBeforeWrite = append([]string(nil), h.published...) return nil @@ -1372,7 +1372,7 @@ func TestRun_CascadeDerivedBatchIsGivenARecoverySignalBeforeItMerges(t *testing. require.NoError(t, h.run(head)) assert.Equal(t, []string{"speculate"}, publishedBeforeWrite) - assert.Equal(t, []string{"speculate", "submitqueue-merge"}, h.published) + assert.Equal(t, []string{"speculate", "submitqueue-land"}, h.published) } // A cancelling path whose build is still running finishes only when CI actually @@ -1424,7 +1424,7 @@ func TestRun_ReportsPassedPathWhileWaiting(t *testing.T) { h := newRunHarness(t, ctrl, spec, []entity.Batch{memberHead()}) h.noBuildsDispatched() // dep1 has landed the way the path assumed; dep2 has not answered yet, so - // the head cannot merge but has nothing of its own left to run. + // the head cannot land but has nothing of its own left to run. h.batches.EXPECT().Get(gomock.Any(), dep1).Return(entity.Batch{ID: dep1, State: entity.BatchStateSucceeded}, nil) h.batches.EXPECT().Get(gomock.Any(), dep2).Return(entity.Batch{ID: dep2, State: entity.BatchStateSpeculating}, nil) @@ -1436,7 +1436,7 @@ func TestRun_ReportsPassedPathWhileWaiting(t *testing.T) { }, nil).AnyTimes() require.NoError(t, h.run(head)) - assert.Empty(t, h.published, "an unsettled head merges nowhere") + assert.Empty(t, h.published, "an unsettled head lands nowhere") require.Len(t, h.logs, 1) assert.Equal(t, "q/1", h.logs[0].RequestID) @@ -1524,10 +1524,10 @@ func TestRun_ReportsInvalidatedWhenTheDependencyFailsInTheSameRun(t *testing.T) assert.Contains(t, got, entity.RequestEventInvalidated) } -// A merge is decided on the same live passed path a wait would be reported +// A land is decided on the same live passed path a wait would be reported // from, so without the gate every landed request would carry a wait it never // had. -func TestRun_MergingHeadReportsSpeculatedAndNoWait(t *testing.T) { +func TestRun_LandingHeadReportsSpeculatedAndNoWait(t *testing.T) { ctrl := gomock.NewController(t) passed := pathOver(entity.DependencyAssumptionSucceeds, entity.DependencyAssumptionSucceeds) @@ -1541,7 +1541,7 @@ func TestRun_MergingHeadReportsSpeculatedAndNoWait(t *testing.T) { Version: 1, }, nil).AnyTimes() h.batches.EXPECT(). - Update(gomock.Any(), updateTo{id: head, state: entity.BatchStateMerging}, int32(1), int32(2)).Return(nil) + Update(gomock.Any(), updateTo{id: head, state: entity.BatchStateLanding}, int32(1), int32(2)).Return(nil) require.NoError(t, h.run(head)) @@ -1561,7 +1561,7 @@ func TestRun_BypassesUnsettledDependenciesWithFullCoverage(t *testing.T) { h := newRunHarness(t, ctrl, &scriptedSpeculator{}, []entity.Batch{memberHead()}) h.batches.EXPECT().Get(gomock.Any(), dep1).Return(entity.Batch{ID: dep1, State: entity.BatchStateSpeculating}, nil) - h.batches.EXPECT().Get(gomock.Any(), dep2).Return(entity.Batch{ID: dep2, State: entity.BatchStateMerging}, nil) + h.batches.EXPECT().Get(gomock.Any(), dep2).Return(entity.Batch{ID: dep2, State: entity.BatchStateLanding}, nil) h.pathSets.EXPECT().Get(gomock.Any(), head).Return(entity.SpeculationPathSet{ Head: head, Paths: []entity.SpeculationPathEntry{ @@ -1573,11 +1573,11 @@ func TestRun_BypassesUnsettledDependenciesWithFullCoverage(t *testing.T) { Version: 1, }, nil).AnyTimes() h.batches.EXPECT(). - Update(gomock.Any(), updateTo{id: head, state: entity.BatchStateMerging}, int32(1), int32(2)).Return(nil) + Update(gomock.Any(), updateTo{id: head, state: entity.BatchStateLanding}, int32(1), int32(2)).Return(nil) require.NoError(t, h.run(head)) - assert.Equal(t, []string{"submitqueue-merge"}, h.published) + assert.Equal(t, []string{"submitqueue-land"}, h.published) assert.Zero(t, h.spec.calls) require.Len(t, h.logs, 1) assert.Equal(t, entity.RequestStatusSpeculated, h.logs[0].Status) @@ -1587,15 +1587,15 @@ func TestRun_BypassesUnsettledDependenciesWithFullCoverage(t *testing.T) { assert.EqualValues(t, 1, counter.Value()) } -// The merge stage publishes landing as its first act on the dispatch. Both +// The land stage publishes landing as its first act on the dispatch. Both // statuses are non-terminal, so the summary is decided on timestamp alone and // a speculated sent afterwards would beat the landing it precedes. -func TestRun_SpeculatedIsReportedBeforeTheMergeDispatch(t *testing.T) { +func TestRun_SpeculatedIsReportedBeforeTheLandDispatch(t *testing.T) { ctrl := gomock.NewController(t) passed := pathOver(entity.DependencyAssumptionSucceeds, entity.DependencyAssumptionSucceeds) h := newRunHarness(t, ctrl, &scriptedSpeculator{}, []entity.Batch{memberHead()}) - h.failPublishTo("submitqueue-merge") + h.failPublishTo("submitqueue-land") h.noBuildsDispatched() h.batches.EXPECT().Get(gomock.Any(), dep1).Return(entity.Batch{ID: dep1, State: entity.BatchStateSucceeded}, nil) h.batches.EXPECT().Get(gomock.Any(), dep2).Return(entity.Batch{ID: dep2, State: entity.BatchStateSucceeded}, nil) @@ -1605,7 +1605,7 @@ func TestRun_SpeculatedIsReportedBeforeTheMergeDispatch(t *testing.T) { Version: 1, }, nil).AnyTimes() h.batches.EXPECT(). - Update(gomock.Any(), updateTo{id: head, state: entity.BatchStateMerging}, int32(1), int32(2)).Return(nil) + Update(gomock.Any(), updateTo{id: head, state: entity.BatchStateLanding}, int32(1), int32(2)).Return(nil) require.Error(t, h.run(head)) diff --git a/submitqueue/orchestrator/controller/speculate/snapshot.go b/submitqueue/orchestrator/controller/speculate/snapshot.go index 2e43b3892..a1677d12d 100644 --- a/submitqueue/orchestrator/controller/speculate/snapshot.go +++ b/submitqueue/orchestrator/controller/speculate/snapshot.go @@ -33,7 +33,7 @@ type snapshot struct { // of one of them. batches map[string]entity.Batch // inFlight is the queue's in-flight batches in queue order, whatever their - // state. This is what the dispatch step walks: a merging or cancelling head + // state. This is what the dispatch step walks: a landing or cancelling head // is closed to new work, but its paths still hold CI slots and their // observations still need persisting. inFlight []entity.Batch diff --git a/submitqueue/orchestrator/controller/speculate/speculate.go b/submitqueue/orchestrator/controller/speculate/speculate.go index ec7c90425..9385e7ada 100644 --- a/submitqueue/orchestrator/controller/speculate/speculate.go +++ b/submitqueue/orchestrator/controller/speculate/speculate.go @@ -79,12 +79,12 @@ func NewController( // // - Created: the batch is admitted first, which makes it visible to the // Speculator (proposals may only target Speculating heads). Reaching an -// outcome on it in the same run is safe: a merge needs a passed path, and +// outcome on it in the same run is safe: a land needs a passed path, and // a head admitted this instant has no paths at all. // - Already terminal: its conclude publish is repeated in case a previous // one was lost — idempotent on the batch ID — and the run that follows is // how dependents learn of an outcome no run has seen yet (a batch -// finalized by another stage, e.g. the merge signal recording a landed +// finalized by another stage, e.g. the land signal recording a landed // push, was never seen breaking the paths that bet against it). // // Everything else — funding paths, cancelling broken ones, driving a @@ -137,11 +137,11 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er } } - // A Merging batch has left the set finalize walks, so a message naming it + // A Landing batch has left the set finalize walks, so a message naming it // is the only thing that will look at it again. - if batch.State == entity.BatchStateMerging { - metrics.NamedCounter(c.metricsScope, opName, "self_heal_merging", 1) - if err := c.dispatchMerge(ctx, batch); err != nil { + if batch.State == entity.BatchStateLanding { + metrics.NamedCounter(c.metricsScope, opName, "self_heal_landing", 1) + if err := c.dispatchLand(ctx, batch); err != nil { return c.attributed(err, entity.BatchSubject(batch.ID)) } } @@ -209,7 +209,7 @@ func (c *Controller) fanout(ctx context.Context, batchID, queue string) error { // // Callers choose msgID, because this controller publishes for two different // kinds of reason. A hand-off that happens once in a batch's life — dispatching -// it to merge, concluding it — names its cause with publish.IntentID, so a +// it to land, concluding it — names its cause with publish.IntentID, so a // redelivery that re-derives the same decision is deduplicated instead of // enacting it twice. A repeat-until-effective nudge — a dispatch re-sent until // the build stage records it, a fan-out repeated in case an earlier one was diff --git a/submitqueue/orchestrator/controller/speculate/speculate_test.go b/submitqueue/orchestrator/controller/speculate/speculate_test.go index 1fe4ef271..19aaf5b35 100644 --- a/submitqueue/orchestrator/controller/speculate/speculate_test.go +++ b/submitqueue/orchestrator/controller/speculate/speculate_test.go @@ -155,7 +155,7 @@ func newProcHarness(t *testing.T, ctrl *gomock.Controller, publishErr error) *pr registry, err := consumer.NewTopicRegistry([]consumer.TopicConfig{ {Key: topickey.TopicKeyBuild, Name: "build", Queue: q}, - {Key: topickey.TopicKeyMerge, Name: "submitqueue-merge", Queue: q}, + {Key: topickey.TopicKeyLand, Name: "submitqueue-land", Queue: q}, {Key: topickey.TopicKeyConclude, Name: "conclude", Queue: q}, {Key: topickey.TopicKeySpeculate, Name: "speculate", Queue: q}, {Key: topickey.TopicKeyLog, Name: "log", Queue: q}, @@ -204,7 +204,7 @@ func TestProcess_AdmitsCreatedBatch(t *testing.T) { h.listsInFlight() require.NoError(t, h.process(t, ctrl, batch.ID)) - assert.Empty(t, h.published, "a batch cannot merge on the message that admitted it") + assert.Empty(t, h.published, "a batch cannot land on the message that admitted it") // Admission is the first thing a member hears after being batched: without // it the request reads "batched" for the whole of speculation. @@ -276,19 +276,19 @@ func TestProcess_TerminalReplansQueue(t *testing.T) { "the dependent must be re-planned against the terminal outcome, which it can only be weighed against if the terminal batch comes too") } -// A Merging batch has left the speculating set, so a message naming it is the +// A Landing batch has left the speculating set, so a message naming it is the // only thing that will look at it again: it re-sends the dispatch to repair // one lost after the state write. -func TestProcess_MergingSelfHeals(t *testing.T) { +func TestProcess_LandingSelfHeals(t *testing.T) { ctrl := gomock.NewController(t) h := newProcHarness(t, ctrl, nil) - batch := testBatch(entity.BatchStateMerging) + batch := testBatch(entity.BatchStateLanding) h.batches.EXPECT().Get(gomock.Any(), batch.ID).Return(batch, nil) h.listsInFlight() require.NoError(t, h.process(t, ctrl, batch.ID)) - assert.Equal(t, []string{"submitqueue-merge"}, h.published) + assert.Equal(t, []string{"submitqueue-land"}, h.published) } func TestProcess_Errors(t *testing.T) { diff --git a/submitqueue/orchestrator/controller/start/start_test.go b/submitqueue/orchestrator/controller/start/start_test.go index 12b88450d..b485a3a33 100644 --- a/submitqueue/orchestrator/controller/start/start_test.go +++ b/submitqueue/orchestrator/controller/start/start_test.go @@ -176,6 +176,7 @@ func TestController_Process_AllStrategies(t *testing.T) { {"rebase", mergestrategy.MergeStrategyRebase}, {"squash rebase", mergestrategy.MergeStrategySquashRebase}, {"merge", mergestrategy.MergeStrategyMerge}, + {"promote", mergestrategy.MergeStrategyPromote}, } for _, tt := range tests { diff --git a/submitqueue/orchestrator/controller/validate/validate.go b/submitqueue/orchestrator/controller/validate/validate.go index 9eacab3cf..10bcd3575 100644 --- a/submitqueue/orchestrator/controller/validate/validate.go +++ b/submitqueue/orchestrator/controller/validate/validate.go @@ -22,7 +22,7 @@ import ( "github.com/uber-go/tally" changepb "github.com/uber/submitqueue/api/base/change/protopb" - strategypb "github.com/uber/submitqueue/api/base/mergestrategy/protopb" + mergestrategypb "github.com/uber/submitqueue/api/base/mergestrategy/protopb" runwaymq "github.com/uber/submitqueue/api/runway/messagequeue" "github.com/uber/submitqueue/platform/base/mergestrategy" "github.com/uber/submitqueue/platform/consumer" @@ -38,8 +38,8 @@ import ( // Controller handles validate queue messages. // It consumes requests, performs local validation checks (duplicate detection via the change store -// and change metadata fetch), then kicks off the asynchronous merge-conflict check by publishing the -// full check request to runway's merge-conflict-check queue. Validation logic is extensible to +// and change metadata fetch), then kicks off the asynchronous land-conflict check by publishing the +// full check request to Runway's merge-conflict-check queue. Validation logic is extensible to // support additional checks. Implements consumer.Controller. type Controller struct { logger *zap.SugaredLogger @@ -57,7 +57,7 @@ type Controller struct { var _ consumer.Controller = (*Controller)(nil) // NewController creates a new validate controller for the orchestrator. -// runwayTopicKey is the runway-owned topic the merge-conflict check request is +// runwayTopicKey is the runway-owned topic the land-conflict check request is // published to (TopicKeyMergeConflictCheck). // validators is an optional factory for custom validation checks; pass nil to skip. func NewController( @@ -86,7 +86,7 @@ func NewController( // Process processes a validate delivery from the queue. // Runs duplicate detection, change metadata fetch, and change claiming, then kicks off the -// asynchronous merge-conflict check by publishing the full check request to runway. +// asynchronous land-conflict check by publishing the full check request to runway. // Returns nil to ack (success or non-retryable rejection), error to nack (retry). func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) error { msg := delivery.Message() @@ -142,7 +142,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er } // Report that validation has begun. This stage is not instantaneous — the - // merge-conflict check below is an async round trip to runway — so without + // land-conflict check below is an async round trip to runway — so without // this the request reads "started" for the whole of it. No occurrence: a // request is validated once, and a redelivery is a retry of that one event. logEntry := entity.NewRequestStatusLog(request.Queue, request.ID, entity.RequestStatusValidating, 0, "", nil) @@ -209,7 +209,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er } // Claim each URI in the change store with its provider details. The claim is - // created here — after duplicate detection and the merge/provider checks — so a + // created here — after duplicate detection, change-provider lookup, and custom validation — so a // rejected request never leaves a claim, and the record is written once with its // details (immutable thereafter; no separate enrichment update). Create is // idempotent per (queue, uri, request_id), so redelivery is a no-op. @@ -218,8 +218,8 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er return fmt.Errorf("failed to claim change records for request %s: %w", request.ID, err) } - // Kick off the asynchronous merge-conflict check: hand the full check request - // to runway via its merge-conflict-check queue, keyed by the request id (the + // Kick off the asynchronous land-conflict check: hand the full check request + // to Runway via its merge-conflict-check queue, keyed by the request id (the // client-owned correlation id) so a redelivery republishes the same id and the // result correlates straight back. At validate time the check is a single step // (candidate vs target branch). @@ -234,12 +234,12 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er }, }, } - if err := c.publishMergeCheck(ctx, req); err != nil { + if err := c.publishLandConflictCheck(ctx, req); err != nil { coremetrics.NamedCounter(c.metricsScope, "process", "publish_errors", 1) return fmt.Errorf("failed to publish to runway merge-conflict-check: %w", err) } - c.logger.Infow("published merge conflict check to runway", + c.logger.Infow("published merge conflict check to Runway", "request_id", request.ID, "topic_key", c.runwayTopicKey, ) @@ -309,16 +309,16 @@ func (c *Controller) checkDuplicate(ctx context.Context, store storage.Storage, return "", nil } -// publishMergeCheck serializes the runway check request and publishes it to the -// runway merge-conflict-check topic, partitioned by queue. +// publishLandConflictCheck serializes the runway check request and publishes it to the +// Runway merge-conflict-check topic, partitioned by queue. // // The correlation ID is the message ID with no cause: a request is checked once, // so a redelivery that re-asks is meant to dedup rather than have Runway run the // same check twice. -func (c *Controller) publishMergeCheck(ctx context.Context, req *runwaymq.MergeRequest) error { +func (c *Controller) publishLandConflictCheck(ctx context.Context, req *runwaymq.MergeRequest) error { payload, err := runwaymq.Marshal(req) if err != nil { - return fmt.Errorf("failed to serialize merge conflict check request: %w", err) + return fmt.Errorf("failed to serialize land conflict check request: %w", err) } if err := publish.Message(ctx, c.registry, c.runwayTopicKey, publish.IntentID(req.GetId()), payload, req.GetQueueName()); err != nil { @@ -331,16 +331,18 @@ func (c *Controller) publishMergeCheck(ctx context.Context, req *runwaymq.MergeR // toProtoStrategy maps the shared mergestrategy.MergeStrategy entity to the proto // Strategy enum carried on the wire. An unknown strategy maps to DEFAULT, letting // runway apply the queue's configured default. -func toProtoStrategy(s mergestrategy.MergeStrategy) strategypb.Strategy { +func toProtoStrategy(s mergestrategy.MergeStrategy) mergestrategypb.Strategy { switch s { case mergestrategy.MergeStrategyRebase: - return strategypb.Strategy_REBASE + return mergestrategypb.Strategy_REBASE case mergestrategy.MergeStrategySquashRebase: - return strategypb.Strategy_SQUASH_REBASE + return mergestrategypb.Strategy_SQUASH_REBASE case mergestrategy.MergeStrategyMerge: - return strategypb.Strategy_MERGE + return mergestrategypb.Strategy_MERGE + case mergestrategy.MergeStrategyPromote: + return mergestrategypb.Strategy_PROMOTE default: - return strategypb.Strategy_DEFAULT + return mergestrategypb.Strategy_DEFAULT } } diff --git a/submitqueue/orchestrator/controller/validate/validate_test.go b/submitqueue/orchestrator/controller/validate/validate_test.go index 4778d408c..053200da2 100644 --- a/submitqueue/orchestrator/controller/validate/validate_test.go +++ b/submitqueue/orchestrator/controller/validate/validate_test.go @@ -48,6 +48,26 @@ type staticStorageFactory struct{ store storage.Storage } // For returns the fixed store aggregate for any queue. func (f staticStorageFactory) For(storage.Config) (storage.Storage, error) { return f.store, nil } +func TestToProtoStrategy(t *testing.T) { + tests := []struct { + name string + in mergestrategy.MergeStrategy + want strategypb.Strategy + }{ + {name: "default", in: mergestrategy.MergeStrategyUnknown, want: strategypb.Strategy_DEFAULT}, + {name: "rebase", in: mergestrategy.MergeStrategyRebase, want: strategypb.Strategy_REBASE}, + {name: "squash rebase", in: mergestrategy.MergeStrategySquashRebase, want: strategypb.Strategy_SQUASH_REBASE}, + {name: "merge", in: mergestrategy.MergeStrategyMerge, want: strategypb.Strategy_MERGE}, + {name: "promote", in: mergestrategy.MergeStrategyPromote, want: strategypb.Strategy_PROMOTE}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, toProtoStrategy(tt.in)) + }) + } +} + func requestWithState(request entity.Request, state entity.RequestState) entity.Request { request.State = state return request @@ -180,7 +200,7 @@ func TestController_Process_Success(t *testing.T) { require.NoError(t, controller.Process(context.Background(), delivery)) } -// TestController_Process_PublishesCheckToRunway verifies the full merge-conflict +// TestController_Process_PublishesCheckToRunway verifies the full land-conflict // check request is published to runway's merge-conflict-check queue (keyed by // the request id, the client-owned correlation id) on the happy path. func TestController_Process_PublishesCheckToRunway(t *testing.T) { diff --git a/submitqueue/orchestrator/pipeline.go b/submitqueue/orchestrator/pipeline.go index 0e8732799..b85defbaf 100644 --- a/submitqueue/orchestrator/pipeline.go +++ b/submitqueue/orchestrator/pipeline.go @@ -43,9 +43,9 @@ import ( "github.com/uber/submitqueue/submitqueue/orchestrator/controller/conclude" "github.com/uber/submitqueue/submitqueue/orchestrator/controller/dependencyanalysis" "github.com/uber/submitqueue/submitqueue/orchestrator/controller/dlq" - "github.com/uber/submitqueue/submitqueue/orchestrator/controller/merge" - "github.com/uber/submitqueue/submitqueue/orchestrator/controller/mergeconflictsignal" - "github.com/uber/submitqueue/submitqueue/orchestrator/controller/mergesignal" + "github.com/uber/submitqueue/submitqueue/orchestrator/controller/land" + "github.com/uber/submitqueue/submitqueue/orchestrator/controller/landconflictsignal" + "github.com/uber/submitqueue/submitqueue/orchestrator/controller/landsignal" "github.com/uber/submitqueue/submitqueue/orchestrator/controller/speculate" "github.com/uber/submitqueue/submitqueue/orchestrator/controller/start" "github.com/uber/submitqueue/submitqueue/orchestrator/controller/validate" @@ -95,9 +95,9 @@ type Deps struct { // // Pipeline: // -// start → cancel → validate ⇢ (runway) ⇢ mergeconflictsignal → batch → speculate → build → buildsignal ─┐ +// start → cancel → validate ⇢ (runway) ⇢ landconflictsignal → batch → speculate → build → buildsignal ─┐ // ↑ ↘ ↻ poll │ -// │ merge → conclude │ +// │ land → conclude │ // │ │ │ // └─────┴───────────────────────┘ var Stages = []pipeline.Stage[Deps]{ @@ -139,10 +139,10 @@ var Stages = []pipeline.Stage[Deps]{ Name: "merge-conflict-check-signal", ConsumerGroup: "orchestrator", New: func(d Deps, sc pipeline.StageContext) (consumer.Controller, error) { - return mergeconflictsignal.NewController(d.Logger, d.Scope, d.Storage, sc.Registry, sc.TopicKey, sc.ConsumerGroup), nil + return landconflictsignal.NewController(d.Logger, d.Scope, d.Storage, sc.Registry, sc.TopicKey, sc.ConsumerGroup), nil }, DLQ: func(d Deps, sc pipeline.StageContext) (consumer.Controller, error) { - return dlq.NewDLQMergeConflictSignalController(d.Logger, d.Scope, d.Storage, sc.Registry, sc.TopicKey, sc.ConsumerGroup), nil + return dlq.NewDLQLandConflictSignalController(d.Logger, d.Scope, d.Storage, sc.Registry, sc.TopicKey, sc.ConsumerGroup), nil }, }, { @@ -201,11 +201,11 @@ var Stages = []pipeline.Stage[Deps]{ }, }, { - Key: topickey.TopicKeyMerge, - Name: "submitqueue-merge", + Key: topickey.TopicKeyLand, + Name: "submitqueue-land", ConsumerGroup: "orchestrator", New: func(d Deps, sc pipeline.StageContext) (consumer.Controller, error) { - return merge.NewController(d.Logger, d.Scope, d.Storage, sc.Registry, runwaymq.TopicKeyMerge, sc.TopicKey, sc.ConsumerGroup), nil + return land.NewController(d.Logger, d.Scope, d.Storage, sc.Registry, runwaymq.TopicKeyMerge, sc.TopicKey, sc.ConsumerGroup), nil }, DLQ: func(d Deps, sc pipeline.StageContext) (consumer.Controller, error) { return dlq.NewDLQBatchController(d.Logger, d.Scope, d.Storage, sc.Registry, sc.TopicKey, sc.ConsumerGroup), nil @@ -216,10 +216,10 @@ var Stages = []pipeline.Stage[Deps]{ Name: "merge-signal", ConsumerGroup: "orchestrator", New: func(d Deps, sc pipeline.StageContext) (consumer.Controller, error) { - return mergesignal.NewController(d.Logger, d.Scope, d.Storage, sc.Registry, sc.TopicKey, sc.ConsumerGroup), nil + return landsignal.NewController(d.Logger, d.Scope, d.Storage, sc.Registry, sc.TopicKey, sc.ConsumerGroup), nil }, DLQ: func(d Deps, sc pipeline.StageContext) (consumer.Controller, error) { - return dlq.NewDLQMergeSignalController(d.Logger, d.Scope, d.Storage, sc.Registry, sc.TopicKey, sc.ConsumerGroup), nil + return dlq.NewDLQLandSignalController(d.Logger, d.Scope, d.Storage, sc.Registry, sc.TopicKey, sc.ConsumerGroup), nil }, }, { @@ -257,9 +257,9 @@ var Stages = []pipeline.Stage[Deps]{ var PublishOnlyTopics = []pipeline.PublishOnlyTopic{ // Log: the orchestrator emits request-log entries; the gateway consumes them. {Key: topickey.TopicKeyLog, Name: "log"}, - // Merge-conflict check: the orchestrator publishes check requests to runway. + // Land-conflict check: the orchestrator publishes check requests to runway. {Key: runwaymq.TopicKeyMergeConflictCheck, Name: "merge-conflict-check"}, - // Merge: the orchestrator publishes merge requests to runway. + // Land: the orchestrator publishes land requests to runway. {Key: runwaymq.TopicKeyMerge, Name: "runway-merge"}, } diff --git a/test/e2e/submitqueue/git_suite_test.go b/test/e2e/submitqueue/git_suite_test.go index 321051ec0..4109c8f25 100644 --- a/test/e2e/submitqueue/git_suite_test.go +++ b/test/e2e/submitqueue/git_suite_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Hermetic end-to-end coverage of a *real* merge. +// Hermetic end-to-end coverage of a *real* land. // // The tier-1 suite (suite_test.go) runs Runway on the noop merger, so "landed" // there proves the pipeline's choreography and nothing about git. This suite @@ -21,10 +21,10 @@ // many ref updates. // // It needs no credential, no network, and no account anywhere, because none of -// that is what the merge machinery depends on — which is what lets these +// that is what the land machinery depends on — which is what lets these // assertions gate a pull request. What it deliberately cannot cover is the half // that is specific to a change provider: reading change metadata, that -// provider's CI, and a real change being marked merged. Those need a repository +// provider's CI, and a real change being marked landed. Those need a repository // and a credential, so they are exercised by hand — see doc/howto/QUICKSTART.md. package e2e_test @@ -59,7 +59,7 @@ const gitQueue = "e2e-git-queue" // configured remote, so this identifies the change rather than routing to it. const sandboxRemote = "git.example.com" -type GitMergeSuite struct { +type GitLandSuite struct { suite.Suite ctx context.Context log *testutil.TestLogger @@ -77,11 +77,11 @@ type GitMergeSuite struct { work string } -func TestGitMergeE2E(t *testing.T) { - suite.Run(t, new(GitMergeSuite)) +func TestGitLandE2E(t *testing.T) { + suite.Run(t, new(GitLandSuite)) } -func (s *GitMergeSuite) SetupSuite() { +func (s *GitLandSuite) SetupSuite() { t := s.T() s.ctx = context.Background() s.log = testutil.NewTestLogger(t) @@ -142,10 +142,10 @@ func (s *GitMergeSuite) SetupSuite() { require.NoError(t, err) s.gatewayClient = gatewaypb.NewSubmitQueueGatewayClient(conn) - s.log.Logf("git merge E2E suite ready (bare repo at %s)", s.bare) + s.log.Logf("git land E2E suite ready (bare repo at %s)", s.bare) } -func (s *GitMergeSuite) TearDownSuite() { +func (s *GitLandSuite) TearDownSuite() { if s.db != nil { s.db.Close() } @@ -156,7 +156,7 @@ func (s *GitMergeSuite) TearDownSuite() { // --- assertions against the repository itself --- -func (s *GitMergeSuite) TestLand_SingleChange_ReachesTheTargetBranch() { +func (s *GitLandSuite) TestLand_SingleChange_ReachesTheTargetBranch() { before := s.mainSHA() head := s.pushChange("feature/single", map[string]string{"single.txt": "single\n"}, "add single") @@ -171,8 +171,8 @@ func (s *GitMergeSuite) TestLand_SingleChange_ReachesTheTargetBranch() { s.Equal("single\n", s.fileOnMain("single.txt")) } -func (s *GitMergeSuite) TestLand_Stack_LandsInOrderInOneRefUpdate() { - // The property that distinguishes a submit queue from merging changes one +func (s *GitLandSuite) TestLand_Stack_LandsInOrderInOneRefUpdate() { + // The property that distinguishes a submit queue from landing changes one // at a time: a stack reaches the target as a single atomic ref update, so // no reader ever observes it half-landed. before := s.mainSHA() @@ -195,8 +195,8 @@ func (s *GitMergeSuite) TestLand_Stack_LandsInOrderInOneRefUpdate() { "the whole stack must reach the target in exactly one ref update") } -func (s *GitMergeSuite) TestLand_MovesEachChangeHeadBranchToItsLandedCommit() { - // What makes a provider mark a rebased change merged: its head branch is moved +func (s *GitLandSuite) TestLand_MovesEachChangeHeadBranchToItsLandedCommit() { + // What makes a provider mark a rebased change landed: its head branch is moved // to the commit the change became, so the head is reachable from the target. before := s.mainSHA() first := s.pushChange("feature/head-1", map[string]string{"h1.txt": "h1\n"}, "add h1") @@ -218,7 +218,7 @@ func (s *GitMergeSuite) TestLand_MovesEachChangeHeadBranchToItsLandedCommit() { s.True(s.isAncestorOfMain(s.branchSHA("feature/head-2"))) } -func (s *GitMergeSuite) TestLand_Conflict_FailsAndLeavesTheTargetUntouched() { +func (s *GitLandSuite) TestLand_Conflict_FailsAndLeavesTheTargetUntouched() { // Two changes editing the same line from the same base: the first lands, // the second cannot be replayed onto it. base := s.mainSHA() @@ -236,7 +236,7 @@ func (s *GitMergeSuite) TestLand_Conflict_FailsAndLeavesTheTargetUntouched() { s.Equal(loser, s.branchSHA("feature/conflict-b")) } -func (s *GitMergeSuite) TestLand_ResubmittedAfterLanding_IsRejectedAsStale() { +func (s *GitLandSuite) TestLand_ResubmittedAfterLanding_IsRejectedAsStale() { // Landing a change moves its head branch to the commit it became, so the // URI that was submitted no longer describes where that branch points. The // staleness check catches exactly that, which is what stops a change from @@ -259,7 +259,7 @@ func (s *GitMergeSuite) TestLand_ResubmittedAfterLanding_IsRejectedAsStale() { // land submits a request and returns its sqid. Repeated URIs are the stack, in // the order they must be applied. -func (s *GitMergeSuite) land(queue string, uris ...string) string { +func (s *GitLandSuite) land(queue string, uris ...string) string { resp, err := s.gatewayClient.Land(s.ctx, &gatewaypb.LandRequest{ Queue: queue, Change: &changepb.Change{Uris: uris}, @@ -272,7 +272,7 @@ func (s *GitMergeSuite) land(queue string, uris ...string) string { // requireStatus waits for the request to reach a terminal status and asserts // which one. Bazel's test timeout is the only deadline. -func (s *GitMergeSuite) requireStatus(sqid string, want entity.RequestStatus) { +func (s *GitLandSuite) requireStatus(sqid string, want entity.RequestStatus) { var got entity.RequestStatus pollUntil(persistPollInterval, func() bool { resp, err := s.gatewayClient.GetRequestSummaryByID(s.ctx, &gatewaypb.GetRequestSummaryByIDRequest{Sqid: sqid, Queue: gitQueue}) @@ -288,7 +288,7 @@ func (s *GitMergeSuite) requireStatus(sqid string, want entity.RequestStatus) { // uri builds the git:// change URI for a branch pinned at a commit. The ref is // percent-encoded so a branch name containing slashes stays one path segment. -func (s *GitMergeSuite) uri(branch, sha string) string { +func (s *GitLandSuite) uri(branch, sha string) string { ref := "refs/heads/" + branch return fmt.Sprintf("git://%s/sandbox/%s/%s", sandboxRemote, url.PathEscape(ref), sha) } @@ -297,7 +297,7 @@ func (s *GitMergeSuite) uri(branch, sha string) string { // stageProviderConfig copies the committed example configuration into a directory // the containers can bind-mount, and returns its path. -func (s *GitMergeSuite) stageProviderConfig() string { +func (s *GitLandSuite) stageProviderConfig() string { t := s.T() staged := t.TempDir() for _, name := range []string{"merge.yaml", "profiles.yaml"} { @@ -310,7 +310,7 @@ func (s *GitMergeSuite) stageProviderConfig() string { // seedRepository creates the bare repository Runway merges into, plus a working // clone the test authors changes in. -func (s *GitMergeSuite) seedRepository() { +func (s *GitLandSuite) seedRepository() { t := s.T() s.runGit(filepath.Dir(s.bare), "init", "--bare", "-b", "main", s.bare) // Bare repositories do not log ref updates by default, and the reflog is @@ -326,7 +326,7 @@ func (s *GitMergeSuite) seedRepository() { s.runGit(s.work, "push", "origin", "main") } -func (s *GitMergeSuite) configureWorkClone() { +func (s *GitLandSuite) configureWorkClone() { for _, kv := range [][2]string{ {"user.name", "E2E Author"}, {"user.email", "author@example.com"}, @@ -342,13 +342,13 @@ func (s *GitMergeSuite) configureWorkClone() { // pushChange authors a change branched off the current target tip and pushes // it, returning its head SHA — all a change URI ever carries. -func (s *GitMergeSuite) pushChange(branch string, files map[string]string, message string) string { +func (s *GitLandSuite) pushChange(branch string, files map[string]string, message string) string { return s.pushChangeOnto("origin/main", branch, files, message) } // pushChangeOnto is pushChange based at an explicit start point, for building a // change that stacks on another rather than on the target. -func (s *GitMergeSuite) pushChangeOnto(base, branch string, files map[string]string, message string) string { +func (s *GitLandSuite) pushChangeOnto(base, branch string, files map[string]string, message string) string { s.runGit(s.work, "fetch", "origin") s.runGit(s.work, "checkout", "-B", branch, base) for path, contents := range files { @@ -361,18 +361,18 @@ func (s *GitMergeSuite) pushChangeOnto(base, branch string, files map[string]str } // mainSHA is the current tip of the target branch on the bare repository. -func (s *GitMergeSuite) mainSHA() string { +func (s *GitLandSuite) mainSHA() string { return s.runGit(s.bare, "rev-parse", "refs/heads/main") } // branchSHA is the current tip of a change's head branch. -func (s *GitMergeSuite) branchSHA(branch string) string { +func (s *GitLandSuite) branchSHA(branch string) string { return s.runGit(s.bare, "rev-parse", "refs/heads/"+branch) } // shasSince lists the commits added to the target since a known point, oldest // first. -func (s *GitMergeSuite) shasSince(since string) []string { +func (s *GitLandSuite) shasSince(since string) []string { out := s.runGit(s.bare, "rev-list", "--reverse", since+"..refs/heads/main") return strings.Fields(out) } @@ -380,7 +380,7 @@ func (s *GitMergeSuite) shasSince(since string) []string { // subjectsSince lists the messages of the commits added to the target since a // known point, oldest first — the readable form of what landed and in what // order. -func (s *GitMergeSuite) subjectsSince(since string) []string { +func (s *GitLandSuite) subjectsSince(since string) []string { out := s.runGit(s.bare, "log", "--reverse", "--format=%s", since+"..refs/heads/main") var subjects []string for _, line := range strings.Split(out, "\n") { @@ -392,13 +392,13 @@ func (s *GitMergeSuite) subjectsSince(since string) []string { } // fileOnMain reads a file's contents at the target tip. -func (s *GitMergeSuite) fileOnMain(path string) string { +func (s *GitLandSuite) fileOnMain(path string) string { return s.runGit(s.bare, "show", "refs/heads/main:"+path) + "\n" } // isAncestorOfMain reports whether a commit is reachable from the target — the -// property a provider reads to decide a change has merged. -func (s *GitMergeSuite) isAncestorOfMain(sha string) bool { +// property a provider reads to decide a change has landed. +func (s *GitLandSuite) isAncestorOfMain(sha string) bool { cmd := exec.Command(s.git, "merge-base", "--is-ancestor", sha, "refs/heads/main") cmd.Dir = s.bare return cmd.Run() == nil @@ -407,7 +407,7 @@ func (s *GitMergeSuite) isAncestorOfMain(sha string) bool { // mainRefUpdateCount is how many times the target branch has been updated, // read from the bare repository's reflog. One land must cost exactly one, // however many changes it carried. -func (s *GitMergeSuite) mainRefUpdateCount() int { +func (s *GitLandSuite) mainRefUpdateCount() int { out := s.runGit(s.bare, "reflog", "show", "--format=%H", "refs/heads/main") count := 0 for _, line := range strings.Split(out, "\n") { @@ -420,7 +420,7 @@ func (s *GitMergeSuite) mainRefUpdateCount() int { // runGit runs the pinned git and returns its trimmed stdout, failing the test // on a non-zero exit. -func (s *GitMergeSuite) runGit(dir string, args ...string) string { +func (s *GitLandSuite) runGit(dir string, args ...string) string { s.T().Helper() cmd := exec.Command(s.git, args...) cmd.Dir = dir diff --git a/test/e2e/submitqueue/suite_test.go b/test/e2e/submitqueue/suite_test.go index 2ecc0973d..38d3c2dc4 100644 --- a/test/e2e/submitqueue/suite_test.go +++ b/test/e2e/submitqueue/suite_test.go @@ -255,7 +255,7 @@ func (s *E2EIntegrationSuite) TestConflictAnalyzerFailure_ReconcilesFromDLQ() { // TestLand_HappyPath_ReachesLanded drives a single request through the whole // pipeline to terminal success on the fully-hermetic e2e-test-queue (no // conflicts, fake build succeeds, noop runway signals SUCCEEDED for both the -// merge-conflict check and the merge). It asserts three views: the black-box +// land-conflict check and the land). It asserts three views: the black-box // terminal request summary, the public GetRequestHistoryByID timeline, and the internal RequestState // in the operating store. // @@ -315,31 +315,31 @@ func (s *E2EIntegrationSuite) TestLand_HappyPath_ReachesLanded() { "operating store should show request %s in terminal state landed", req.sqid) } -// TestDependentBatch_BypassesMergingDependency proves that a dependency still -// waiting on Runway is unresolved for strict merge but can be bypassed once +// TestDependentBatch_BypassesLandingDependency proves that a dependency still +// waiting on Runway is unresolved for strict land but can be bypassed once // passed paths cover both of its possible outcomes. -func (s *E2EIntegrationSuite) TestDependentBatch_BypassesMergingDependency() { +func (s *E2EIntegrationSuite) TestDependentBatch_BypassesLandingDependency() { t := s.T() const queue = "e2e-chain-queue" const gateGroup = "runway-merge" gateTopic := runwaymq.TopicKeyMerge.String() - s.closeGate(gateGroup, queue, "e2e: hold the lead merge so the dependent finishes building first") + s.closeGate(gateGroup, queue, "e2e: hold the lead land so the dependent finishes building first") // Reopen even if an assertion below fails, so teardown does not stop the // stack with a delivery still parked. Opening twice is a no-op. defer s.openGate(gateGroup, queue) lead := s.land(queue, "github://github.example.com/uber/e2e-chain/pull/1/abcdef0123456789abcdef0123456789abcdef01") - s.log.Logf("Landed lead request %s; awaiting its merge to park", lead.sqid) + s.log.Logf("Landed lead request %s; awaiting its land to park", lead.sqid) - // The merge request is keyed by batch, so name the batch to prove the - // parked delivery is this request's merge and not some other. + // The land request is keyed by batch, so name the batch to prove the + // parked delivery is this request's land and not some other. leadBatch := s.awaitBatchID(lead) parked := s.awaitParked(gateGroup, gateTopic, leadBatch) - assert.Equal(t, queue, parked.PartitionKey, "merge request should be partitioned by queue") + assert.Equal(t, queue, parked.PartitionKey, "land request should be partitioned by queue") - // The lead is provably stopped mid-merge. A request landed now serializes + // The lead is provably stopped mid-land. A request landed now serializes // behind it. dependent := s.land(queue, "github://github.example.com/uber/e2e-chain/pull/2/1234567890abcdef1234567890abcdef12345678") dependentBatch := s.awaitBatchID(dependent) @@ -352,13 +352,13 @@ func (s *E2EIntegrationSuite) TestDependentBatch_BypassesMergingDependency() { require.Contains(t, got.Dependencies, leadBatch, "batch %s must depend on the in-flight %s for this test to exercise anything", dependentBatch, leadBatch) - // Both paths pass while the lead is still parked. A Merging dependency is - // unresolved because its merge can fail, so the durable Merging state proves - // the dependent advanced through complete coverage rather than strict merge. - s.awaitBatchState(queue, dependentBatch, entity.BatchStateMerging) - s.log.Logf("Dependent %s bypassed merging batch %s", dependent.sqid, leadBatch) + // Both paths pass while the lead is still parked. A Landing dependency is + // unresolved because its land can fail, so the durable Landing state proves + // the dependent advanced through complete coverage rather than strict land. + s.awaitBatchState(queue, dependentBatch, entity.BatchStateLanding) + s.log.Logf("Dependent %s bypassed landing batch %s", dependent.sqid, leadBatch) - // Start: the lead merges, and its fan-out is now the only thing that can + // Start: the lead lands, and its fan-out is now the only thing that can // move the dependent. s.openGate(gateGroup, queue) s.awaitUnparked(gateGroup, gateTopic, leadBatch) @@ -370,7 +370,7 @@ func (s *E2EIntegrationSuite) TestDependentBatch_BypassesMergingDependency() { } // TestDependentBatch_BypassedHeadLandsFirst proves the bypass does not just -// dispatch a merge: the dependent lands while its dependency is still held +// dispatch a land: the dependent lands while its dependency is still held // mid-build, and only then does the dependency proceed. func (s *E2EIntegrationSuite) TestDependentBatch_BypassedHeadLandsFirst() { t := s.T() @@ -389,8 +389,8 @@ func (s *E2EIntegrationSuite) TestDependentBatch_BypassedHeadLandsFirst() { require.NotEqual(t, heldBatch, followerBatch) // The follower builds with and without the held leader while the leader's - // own build is parked; complete coverage hands it to the merge stage. - s.awaitBatchState(queue, followerBatch, entity.BatchStateMerging) + // own build is parked; complete coverage hands it to the land stage. + s.awaitBatchState(queue, followerBatch, entity.BatchStateLanding) s.awaitStatus(follower, entity.RequestStatusLanded) assert.Equal(t, entity.RequestStatusSpeculating, s.mustStatus(lead), "the follower must land while its dependency is still held") @@ -408,7 +408,7 @@ func (s *E2EIntegrationSuite) TestDependentBatch_BypassedHeadLandsFirst() { // TestDependentBatch_NoBypassWhenCoverageIsIncomplete proves the complement: // with only the "dependency succeeds" side passed, a head waits for its -// dependency to resolve and merges strictly, never dispatching ahead of it. +// dependency to resolve and lands strictly, never dispatching ahead of it. // // Partial coverage is seeded directly: the follower's batch is stranded in // Created (build held), its "succeeds" path is written as passed, and a @@ -444,15 +444,15 @@ func (s *E2EIntegrationSuite) TestDependentBatch_NoBypassWhenCoverageIsIncomplet s.awaitBatchID(trigger) // The run that admits the follower also reports the wait: its one passed - // path covers only one of the lead's two outcomes, so it cannot merge + // path covers only one of the lead's two outcomes, so it cannot land // until the lead resolves. s.awaitEvent(follower, entity.RequestEventWaiting) // "No bypass" must not be read off the follower's batch state: waiting is // a recorded event, not a moment, and the lead is ungated — it can land - // before this test looks, at which point the seeded path is mergeable - // *strictly* and the batch rightly moves to Merging. The bypass would - // show up differently: the follower merging while the lead is still + // before this test looks, at which point the seeded path is landable + // *strictly* and the batch rightly moves to Landing. The bypass would + // show up differently: the follower landing while the lead is still // unresolved. Assert the true invariant instead — the follower only lands // after the lead has landed. s.awaitStatus(lead, entity.RequestStatusLanded) @@ -465,6 +465,102 @@ func (s *E2EIntegrationSuite) TestDependentBatch_NoBypassWhenCoverageIsIncomplet s.awaitStatus(trigger, entity.RequestStatusLanded) } +// TestDependentBatch_IsWokenByTheLandAhead proves that a batch waiting on +// another is woken when that one lands — the edge CODEM-303 was silently +// dropping. +// +// A landed batch fans out to speculate so its dependents can re-plan. That +// message used to reuse the bare batch ID, which the batch controller had +// already published to the same topic and partition when the batch was +// created. The queue deduplicates against rows it has not collected yet, +// consumed ones included, so the wake-up was reported as a success, stored +// nothing, and never arrived. +// +// Ordinarily something else re-plans the queue soon enough to hide that. This +// test removes every other source of a wake-up, as stop → observe → start: +// +// 1. Stop: close the gate for runway-merge on this queue, before landing, so +// the lead batch cannot complete its land. +// 2. Land the lead. It runs to the land hand-off and parks there. +// 3. Land the dependent. The queue's analyzer serializes conservatively, so +// its batch depends on the lead's, which is in-flight (Landing counts). +// 4. Fund only the path that assumes the lead succeeds, and hold every real +// build for the dependent so complete coverage cannot bypass the lead. +// 5. Observe: wait for the dependent to record "waiting". From here the only +// event that can make its funded path landable is the lead landing. +// 6. Start: open the gate. The lead lands and fans out. +// +// The dependent reaching "landed" is therefore attributable to the fan-out +// alone. Against the old code it rests at "speculating" and the suite runs to +// Bazel's timeout, which is how the harness reports a pipeline that stalled. +func (s *E2EIntegrationSuite) TestDependentBatch_IsWokenByTheLandAhead() { + t := s.T() + + const queue = "e2e-chain-queue" + const runwayGateGroup = "runway-merge" + const orchestratorGateGroup = "orchestrator" + gateTopic := runwaymq.TopicKeyMerge.String() + + s.closeGate(runwayGateGroup, queue, "e2e: hold the lead land while the dependent waits") + // Reopen even if an assertion below fails, so teardown does not stop the + // stack with a delivery still parked. Opening twice is a no-op. + defer s.openGate(runwayGateGroup, queue) + + lead := s.land(queue, "github://github.example.com/uber/e2e-chain/pull/1/abcdef0123456789abcdef0123456789abcdef01") + s.log.Logf("Landed lead request %s; awaiting its land to park", lead.sqid) + + // The land request is keyed by batch, so name the batch to prove the + // parked delivery is this request's land and not some other. + leadBatch := s.awaitBatchID(lead) + parked := s.awaitParked(runwayGateGroup, gateTopic, leadBatch) + assert.Equal(t, queue, parked.PartitionKey, "land request should be partitioned by queue") + + // The lead is provably stopped mid-land. A request landed now serializes + // behind it. + dependent := s.land(queue, "github://github.example.com/uber/e2e-chain/pull/2/1234567890abcdef1234567890abcdef12345678") + dependentBatch := s.awaitBatchID(dependent) + require.NotEqual(t, leadBatch, dependentBatch, "the two requests must be carried by different batches") + s.closeGate(orchestratorGateGroup, dependentBatch, "e2e: hold dependent builds so only the seeded path exists") + defer s.openGate(orchestratorGateGroup, dependentBatch) + + leadState, err := s.appStorage.For(queue) + require.NoError(t, err) + got, err := leadState.GetBatchStore().Get(s.ctx, dependentBatch) + require.NoError(t, err, "failed to read the dependent batch") + require.Contains(t, got.Dependencies, leadBatch, + "batch %s must depend on the in-flight %s for this test to exercise anything", dependentBatch, leadBatch) + + // Leave the real builds parked and fund only the world where the lead lands. + // A later request wakes the queue so it admits the stranded batch and + // reports the wait without adding the missing failure-assumption path. + s.strandInCreated(queue, dependentBatch) + s.seedPassedPath(queue, entity.SpeculationPath{ + Head: dependentBatch, + Dependencies: []entity.PathDependency{ + {Batch: leadBatch, Assumption: entity.DependencyAssumptionSucceeds}, + }, + }) + trigger := s.land(queue, "github://github.example.com/uber/e2e-chain/pull/3/fedcba9876543210fedcba9876543210fedcba98") + s.awaitBatchID(trigger) + + s.awaitEvent(dependent, entity.RequestEventWaiting) + s.log.Logf("Dependent %s has passed its build and waits only on %s", dependent.sqid, leadBatch) + + // Start: the lead lands, and its fan-out is now the only thing that can + // move the dependent. + s.openGate(runwayGateGroup, queue) + s.awaitUnparked(runwayGateGroup, gateTopic, leadBatch) + + s.awaitStatus(lead, entity.RequestStatusLanded) + s.awaitStatus(dependent, entity.RequestStatusLanded) + + assert.Equal(t, entity.RequestStateLanded, s.terminalState(dependent), + "the dependent must land once the batch it waited on landed") + + s.openGate(orchestratorGateGroup, dependentBatch) + s.awaitStatus(trigger, entity.RequestStatusLanded) +} + // TestReadAPIs validates all five request read endpoints against receipts // created through the public Land API. func (s *E2EIntegrationSuite) TestReadAPIs() { @@ -677,7 +773,7 @@ func (s *E2EIntegrationSuite) TestCancel_CaughtPreBatch_NeverLands() { // A batch delivery that is retried after its ack was lost must not enrol the // request into a second batch. Both batches would be analyzed, promoted and -// admitted, and both would merge the same change. +// admitted, and both would land the same change. // // The build gate is what makes the redelivery land in the window that matters: // with the build for the first batch held, the request cannot reach a terminal diff --git a/test/integration/extension/messagequeue/mysql/queue_test.go b/test/integration/extension/messagequeue/mysql/queue_test.go index a87503019..b0fb4a6df 100644 --- a/test/integration/extension/messagequeue/mysql/queue_test.go +++ b/test/integration/extension/messagequeue/mysql/queue_test.go @@ -779,9 +779,9 @@ func (s *SQLQueueIntegrationSuite) TestDedupOutlivesConsumption() { // Naming the cause is what gets it through. require.NoError(t, publisher.Publish(s.ctx, topic, - entityqueue.NewMessage("batch-1/merged", []byte("woken"), "queue-1", nil))) + entityqueue.NewMessage("batch-1/landed", []byte("woken"), "queue-1", nil))) second := receive(t, deliveryChan) - assert.Equal(t, "batch-1/merged", second.Message().ID) + assert.Equal(t, "batch-1/landed", second.Message().ID) assert.Equal(t, []byte("woken"), second.Message().Payload) require.NoError(t, second.Ack(s.ctx)) } diff --git a/test/integration/submitqueue/extension/storage/suite.go b/test/integration/submitqueue/extension/storage/suite.go index ff40e3efb..9a3b104f7 100644 --- a/test/integration/submitqueue/extension/storage/suite.go +++ b/test/integration/submitqueue/extension/storage/suite.go @@ -319,7 +319,7 @@ func (s *StorageContractSuite) TestStorage_BatchUpdateReplacesAllNonKeyFields() emptyCollections := got emptyCollections.Contains = []string{} emptyCollections.Dependencies = []string{} - emptyCollections.State = entity.BatchStateMerging + emptyCollections.State = entity.BatchStateLanding require.NoError(t, store.Update(ctx, emptyCollections, 2, 3)) got, err = store.Get(ctx, batch.ID) @@ -329,7 +329,7 @@ func (s *StorageContractSuite) TestStorage_BatchUpdateReplacesAllNonKeyFields() assert.Empty(t, got.Contains) assert.NotNil(t, got.Dependencies) assert.Empty(t, got.Dependencies) - assert.Equal(t, entity.BatchStateMerging, got.State) + assert.Equal(t, entity.BatchStateLanding, got.State) assert.Equal(t, int32(3), got.Version) stale := got @@ -375,7 +375,7 @@ func (s *StorageContractSuite) TestStorage_QueueBatchStateRecordLifecycle() { assert.ElementsMatch(t, []entity.QueueBatchState{speculating}, got) // An empty bucket lists empty, not an error. - got, err = storeA.List(ctx, entity.BatchStateMerging) + got, err = storeA.List(ctx, entity.BatchStateLanding) require.NoError(t, err) assert.Empty(t, got) diff --git a/tool/linter/messageid/main_test.go b/tool/linter/messageid/main_test.go index 9fcc6282e..99018bafa 100644 --- a/tool/linter/messageid/main_test.go +++ b/tool/linter/messageid/main_test.go @@ -50,7 +50,7 @@ func f() { _ = messagequeue.NewMessage("id", nil, "part", nil) }`, name: "publishing through the helper passes", src: `package p import "github.com/uber/submitqueue/platform/publish" -func f() { _ = publish.IntentID("batch-1", "merged") }`, +func f() { _ = publish.IntentID("batch-1", "landed") }`, }, { name: "NewMessage of an unrelated package passes",