From 844f3ac9e863916cbaa5a7e0a490fbfdb7dfdd7a Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Sat, 31 Oct 2026 20:10:27 +0800 Subject: [PATCH] E3.1.2: Bind fresh replacement and lifecycle handoff Retire the exact previous mapped workload before reloading current admission facts and constructing a fresh native replacement. Reuse the existing lifecycle record and owned waiter for trusted startup and qualified persistent handoff. Fence transfer claims, detachment and final lifecycle promotion against owner loss. Preserve public mapped inhibition and require future production F/G evidence. Tests cover both profiles, stale bindings, retirement, startup, transfer uncertainty and owner-loss refusals. --- .../current_workload_lifecycle_mapped.go | 272 +++++++++ .../current_workload_lifecycle_mapped_test.go | 572 ++++++++++++++++++ 2 files changed, 844 insertions(+) create mode 100644 internal/dockerdeploy/current_workload_lifecycle_mapped.go create mode 100644 internal/dockerdeploy/current_workload_lifecycle_mapped_test.go diff --git a/internal/dockerdeploy/current_workload_lifecycle_mapped.go b/internal/dockerdeploy/current_workload_lifecycle_mapped.go new file mode 100644 index 00000000..1b370971 --- /dev/null +++ b/internal/dockerdeploy/current_workload_lifecycle_mapped.go @@ -0,0 +1,272 @@ +package dockerdeploy + +import ( + "context" + "errors" + "fmt" + "reflect" + "strconv" + "strings" + + "github.com/omry/reploy/internal/canonical" + "github.com/omry/reploy/internal/deploy" + "github.com/omry/reploy/internal/mappedlaunch" +) + +// F/G's trusted effect owner supplies these observations, never an operator, +// application marker or engine-running assertion. Retire stops the exact owned +// instance and proves its retirement, including an already absent instance. +// A detached observation must follow qualified stream detachment and freshly +// observe the same live instance. E supplies no production provider. +// Calls retain the operation lock. Detach retains the waiter's ownership fence: +// it detaches streams, not the lifetime owner, and must not recursively close it. +type mappedWorkloadEvidenceV1 interface { + RetireMappedWorkload(context.Context, deploy.StateV1, mappedWorkloadRequestV1) (mappedWorkloadProofV1, error) + DetachMappedWorkload(context.Context, deploy.StateV1, mappedWorkloadRequestV1) error + MappedWorkload(context.Context, deploy.StateV1, mappedWorkloadRequestV1) (mappedWorkloadProofV1, error) +} + +type mappedWorkloadRequestV1 struct { + Phase, Installation, Revision, Engine, Run string + Generation canonical.Digest + Record deploy.WorkloadLifecycleV1 + Binding mappedlaunch.Binding + Container, PID string +} + +type mappedWorkloadProofV1 struct { + Request canonical.Digest + Container, StartedAt string + Running, Retired, StartupComplete, Detached bool + AutoRemove bool + RestartPolicy string + RestartCount uint64 +} + +func mappedWorkloadEndpointV1(engine string) (string, error) { + digest, err := canonical.Sum("mapped-workload-endpoint", "v1", engine) + return strings.TrimPrefix(string(digest), "sha256:"), err +} + +func requireMappedWorkloadOwnerV1(ctx context.Context, ops deploy.IdentityMappingOperationsV1) (mappedWorkloadEvidenceV1, error) { + if ctx == nil || ops.Operation == nil { + return nil, fmt.Errorf("mapped workload requires bounded context and operation lock") + } + if _, bounded := ctx.Deadline(); !bounded || ctx.Err() != nil { + return nil, fmt.Errorf("mapped workload requires live bounded context") + } + if err := ops.Operation.RequireWritable(); err != nil { + return nil, err + } + owner, ok := ops.Evidence.(mappedWorkloadEvidenceV1) + if !ok { + return nil, fmt.Errorf("mapped workload trusted observation provider is unavailable") + } + return owner, nil +} + +func verifyMappedWorkloadProofV1(ctx context.Context, request mappedWorkloadRequestV1, proof mappedWorkloadProofV1) error { + digest, err := canonical.Sum("mapped-workload-observation", "v1", request) + if err != nil || ctx.Err() != nil || proof.Request != digest || proof.Container != request.Container || + proof.RestartPolicy != "no" || proof.RestartCount != 0 || proof.AutoRemove { + return fmt.Errorf("mapped workload observation is missing, stale or substituted") + } + identity := request.Record + identity.ContainerID, identity.StartedAt = proof.Container, proof.StartedAt + if identity.ContainerID == "" || identity.Validate() != nil { + return fmt.Errorf("mapped workload observation lacks exact process/start identity") + } + if request.Record.StartedAt != "" && request.Record.StartedAt != proof.StartedAt { + return fmt.Errorf("mapped workload start identity changed") + } + if request.Phase == "retirement" { + if proof.Running || !proof.Retired { + return fmt.Errorf("mapped workload exact retirement is unverified") + } + } else if !proof.Running || proof.Retired || !proof.StartupComplete || proof.Detached != (request.Phase == "detached") { + return fmt.Errorf("mapped workload startup or stream handoff is unverified") + } + return nil +} + +// reload is the existing trusted held-lock admission reader/writer: it reloads +// current generation, host principal/delegation and policy, then prepares a new +// inhibited native operation. It runs only after exact old retirement. The +// returned plan still requires E3.1.1's owned-waiter gate; it is no admission token. +func prepareMappedWorkloadReplacementV1(ctx context.Context, ops deploy.IdentityMappingOperationsV1, reload func(context.Context, deploy.IdentityMappingOperationsV1) (mappedEntryInputV1, error)) (mappedEntryPlanV1, error) { + var empty mappedEntryPlanV1 + owner, err := requireMappedWorkloadOwnerV1(ctx, ops) + if err != nil || reload == nil { + return empty, errors.Join(err, fmt.Errorf("mapped replacement requires a fresh trusted admission reader")) + } + old, found, err := ops.Operation.ReadWorkloadLifecycleV1() + if err != nil { + return empty, err + } + status, err := ops.Status() + if err != nil || status.State.IdentityMapping == nil { + return empty, errors.Join(err, fmt.Errorf("mapped replacement requires selected native ownership")) + } + state := status.State + m := state.IdentityMapping + oldRun, oldOperation := "", "" + if m.NativeProcess != nil { + oldRun = *m.NativeProcess + } + if p := m.PendingOperation; p != nil { + if p.Kind != "native-execute" || p.Phase != "release-attempted" || p.Native == nil || !found || old.ContainerID == "" { + return empty, fmt.Errorf("mapped replacement has an unresolved native attempt") + } + oldOperation = p.OperationID + } + if found && old.ContainerID != "" { + if m.Engine == nil || oldRun == "" || m.AdmittedGeneration == nil { + return empty, fmt.Errorf("mapped replacement lacks old native ownership") + } + endpoint, err := mappedWorkloadEndpointV1(*m.Engine) + if err != nil || endpoint != old.Endpoint { + return empty, fmt.Errorf("mapped replacement old endpoint differs") + } + old.State = "stopping" + if err := ops.Operation.WriteWorkloadLifecycleV1(old); err != nil { + return empty, err + } + request := mappedWorkloadRequestV1{Phase: "retirement", Installation: m.InstallationID, Revision: m.Revision, + Engine: *m.Engine, Run: oldRun, Generation: m.AdmittedGeneration.BuildLockDigest, Record: old, Container: old.ContainerID} + proof, err := owner.RetireMappedWorkload(ctx, state, request) + if err != nil { + return empty, fmt.Errorf("mapped replacement retirement requires recovery: %w", err) + } + if err := verifyMappedWorkloadProofV1(ctx, request, proof); err != nil { + return empty, err + } + if oldOperation != "" { + state, err = ops.Recover(ctx, state, oldOperation, false) + if err != nil || state.IdentityMapping == nil || state.IdentityMapping.PendingOperation != nil { + return empty, errors.Join(err, fmt.Errorf("mapped replacement native retirement remains uncertain")) + } + } + old.State = "down" + if err := ops.Operation.WriteWorkloadLifecycleV1(old); err != nil { + return empty, err + } + } else if found && old.State != "down" { + return empty, fmt.Errorf("mapped replacement has unbound startup uncertainty") + } + input, err := reload(ctx, ops) + if err != nil || ctx.Err() != nil { + return empty, errors.Join(err, ctx.Err()) + } + current, err := ops.Status() + if err != nil || !reflect.DeepEqual(current.State, input.Current.State) || input.Kind != mappedHostReplacementV1 { + return empty, fmt.Errorf("mapped replacement admission is not the current locked snapshot: %v", err) + } + plan, err := constructMappedEntryPlanV1(input) + if err != nil { + return empty, err + } + facts, err := plan.facts() + if err != nil || facts.Run == oldRun || facts.Operation == oldOperation || found && strings.TrimPrefix(string(plan.binding.Plan), "sha256:") == old.Runtime { + return empty, fmt.Errorf("mapped replacement cannot reuse an old attempt or plan: %v", err) + } + endpoint, err := mappedWorkloadEndpointV1(facts.Engine) + if err != nil { + return empty, err + } + record := deploy.WorkloadLifecycleV1{Schema: "workload-lifecycle-v1", State: "starting", Endpoint: endpoint, Runtime: strings.TrimPrefix(string(plan.binding.Plan), "sha256:")} + if err := ops.Operation.WriteWorkloadLifecycleV1(record); err != nil { + return empty, err + } + return plan, nil +} + +// This is called by the trusted lifetime owner after its one-shot release, +// while it still holds the operation lock. A failed confirmation keeps starting +// ownership; the lifetime owner must perform its normal bounded stop/retirement. +// A detached proof includes actual qualified detachment followed by a fresh +// exact live observation. No production detach transport is introduced here. +func confirmMappedWorkloadStartupV1(ctx context.Context, ops deploy.IdentityMappingOperationsV1, plan mappedEntryPlanV1, waiter *ownedMappedWaiterV1, detached bool) error { + owner, err := requireMappedWorkloadOwnerV1(ctx, ops) + if err != nil { + return err + } + facts, err := plan.facts() + if err != nil || facts.Kind != strconv.Itoa(int(mappedHostReplacementV1)) || waiter == nil || waiter.prepared == nil || + waiter.prepared.ops.Operation != ops.Operation || waiter.binding != plan.binding || waiter.container == "" || waiter.pid <= 0 { + return fmt.Errorf("mapped startup requires its exact owned replacement waiter: %v", err) + } + waiter.mu.Lock() + released := waiter.consumed && !waiter.closed + waiter.mu.Unlock() + if !released { + return fmt.Errorf("mapped startup requires an attempted owned release") + } + status, err := ops.Status() + if err != nil || status.State.Current == nil || status.State.IdentityMapping == nil { + return fmt.Errorf("mapped startup current ownership is unavailable: %v", err) + } + state, m := status.State, status.State.IdentityMapping + p := m.PendingOperation + revision, err := strconv.ParseUint(facts.Revision, 10, 64) + if err != nil || revision == ^uint64(0) || m.Revision != strconv.FormatUint(revision+1, 10) || + m.InstallationID != facts.Installation || state.Current.BuildLockDigest != facts.Generation || + p == nil || p.Kind != "native-execute" || p.Phase != "release-attempted" || p.OperationID != facts.Operation || + p.Native == nil || p.Native.Engine != facts.Engine || p.Native.Process != facts.Run { + return fmt.Errorf("mapped startup native release owner changed") + } + record, found, err := ops.Operation.ReadWorkloadLifecycleV1() + endpoint, endpointErr := mappedWorkloadEndpointV1(facts.Engine) + if err != nil || endpointErr != nil || !found || record.State != "starting" || record.Endpoint != endpoint || + record.Runtime != strings.TrimPrefix(string(plan.binding.Plan), "sha256:") || record.ContainerID != "" || record.TransferClaimed || record.StartupComplete || record.HandoffComplete { + return fmt.Errorf("mapped startup lifecycle owner changed") + } + request := mappedWorkloadRequestV1{Phase: "attached", Installation: facts.Installation, Revision: m.Revision, Engine: facts.Engine, + Run: facts.Run, Generation: facts.Generation, Record: record, Binding: plan.binding, Container: waiter.container, PID: strconv.Itoa(waiter.pid)} + proof, err := owner.MappedWorkload(ctx, state, request) + if err != nil { + return fmt.Errorf("mapped startup completion requires recovery: %w", err) + } + if err := verifyMappedWorkloadProofV1(ctx, request, proof); err != nil { + return err + } + record.ContainerID, record.StartedAt, record.StartupComplete = proof.Container, proof.StartedAt, true + if detached { + // Claim this exact transfer before detaching any stream. A failed or + // ambiguous detach cannot be retried as another successful handoff. + if err := func() error { + waiter.mu.Lock() + defer waiter.mu.Unlock() + if waiter.closed || !waiter.consumed || ctx.Err() != nil { + return fmt.Errorf("mapped lifetime owner ended before stream detachment") + } + record.TransferClaimed = true + if err := ops.Operation.WriteWorkloadLifecycleV1(record); err != nil { + return err + } + request.Phase, request.Record = "detached", record + return owner.DetachMappedWorkload(ctx, state, request) + }(); err != nil { + return fmt.Errorf("mapped stream detachment requires recovery: %w", err) + } + proof, err = owner.MappedWorkload(ctx, state, request) + if err != nil { + return fmt.Errorf("mapped detached instance requires recovery: %w", err) + } + if err := verifyMappedWorkloadProofV1(ctx, request, proof); err != nil { + return err + } + } + // Never translate a frame write, engine-running result or application marker + // into startup success. Only the exact trusted observation reaches this write. + // Serialize the commit with close: an owner that closed during observation + // cannot publish up, and cleanup cannot start inside the commit interval. + waiter.mu.Lock() + defer waiter.mu.Unlock() + if waiter.closed || !waiter.consumed || ctx.Err() != nil { + return fmt.Errorf("mapped lifetime owner ended before startup commit") + } + record.State, record.ContainerID, record.StartedAt = "up", proof.Container, proof.StartedAt + record.StartupComplete = true + record.TransferClaimed, record.HandoffComplete = detached, detached + return ops.Operation.WriteWorkloadLifecycleV1(record) +} diff --git a/internal/dockerdeploy/current_workload_lifecycle_mapped_test.go b/internal/dockerdeploy/current_workload_lifecycle_mapped_test.go new file mode 100644 index 00000000..8d85f052 --- /dev/null +++ b/internal/dockerdeploy/current_workload_lifecycle_mapped_test.go @@ -0,0 +1,572 @@ +package dockerdeploy + +import ( + "context" + "errors" + "reflect" + "slices" + "strconv" + "strings" + "testing" + "time" + + "github.com/omry/reploy/internal/canonical" + "github.com/omry/reploy/internal/deploy" +) + +// These are private orchestration fixtures with real lock/state/lifecycle and +// retained acknowledged native history. Engine, admission and F/G observations are injected; +// no live-engine startup or detachment qualification is claimed. +type mappedWorkloadFixtureV1 struct { + *waiterEffectFixtureV1 + t *testing.T + live *mappedLiveFixtureV1 + ops deploy.IdentityMappingOperationsV1 + pending *deploy.IdentityMappingOperationV1 + old deploy.WorkloadLifecycleV1 + trace []string + retireErr error + observeErr error + detachErr error + changeProof func(mappedWorkloadRequestV1, *mappedWorkloadProofV1) + changeInput func(*mappedEntryInputV1) + afterObserve func(mappedWorkloadRequestV1) + cancelOnRetire context.CancelFunc +} + +func cloneMappedWorkloadStateFixtureV1(t *testing.T, state deploy.StateV1) deploy.StateV1 { + t.Helper() + data, err := deploy.EncodeStateV1(state) + if err != nil { + t.Fatal(err) + } + copy, err := deploy.DecodeStateV1(data) + if err != nil { + t.Fatal(err) + } + return copy +} + +func newMappedWorkloadFixtureV1(t *testing.T, profile string) *mappedWorkloadFixtureV1 { + t.Helper() + live := newMappedLiveFixtureV1(t, profile, mappedHostReplacementV1) + f := &mappedWorkloadFixtureV1{waiterEffectFixtureV1: live.waiter.effect, t: t, live: live, ops: live.waiter.prepared.ops} + f.ops.Evidence = f + live.waiter.prepared.ops = f.ops + state := cloneMappedWorkloadStateFixtureV1(t, live.input.Current.State) + m := state.IdentityMapping + p := m.PendingOperation + p.OperationID, p.Native.Process = "old-operation", "old-process" + p.Phase, p.ExpectedRevision, m.Revision = "release-attempted", "2", "2" + a := p.Native.Active + m.UIDAssignments, m.GIDAssignments = a.UIDs, a.GIDs + m.AcceptedUIDs, m.AcceptedGIDs = &a.UIDs, &a.GIDs + m.AdmittedGeneration, m.Engine, m.NativeProcess = state.Current, &p.Native.Engine, &p.Native.Process + f.pending = cloneMappedWorkloadStateFixtureV1(t, state).IdentityMapping.PendingOperation + // The old release was acknowledged. Pending-release recovery also invokes + // the host preflight; that refusal is tested separately on this fixture. + m.PendingOperation = nil + writeMappedLiveStateFixtureV1(t, f.ops.Operation, state) + endpoint, err := mappedWorkloadEndpointV1(*m.Engine) + if err != nil { + t.Fatal(err) + } + f.old = deploy.WorkloadLifecycleV1{Schema: "workload-lifecycle-v1", State: "up", Endpoint: endpoint, + Runtime: strings.Repeat("1", 64), ContainerID: strings.Repeat("a", 64), StartedAt: "2026-01-01T00:00:00Z", StartupComplete: true} + if err := f.ops.Operation.WriteWorkloadLifecycleV1(f.old); err != nil { + t.Fatal(err) + } + return f +} + +func (f *mappedWorkloadFixtureV1) proof(request mappedWorkloadRequestV1) mappedWorkloadProofV1 { + f.t.Helper() + digest, err := canonical.Sum("mapped-workload-observation", "v1", request) + if err != nil { + f.t.Fatal(err) + } + started := request.Record.StartedAt + if started == "" { + started = "2026-01-02T00:00:00Z" + } + proof := mappedWorkloadProofV1{Request: digest, Container: request.Container, StartedAt: started, RestartPolicy: "no", + Running: request.Phase != "retirement", Retired: request.Phase == "retirement", StartupComplete: request.Phase != "retirement", Detached: request.Phase == "detached"} + if f.changeProof != nil { + f.changeProof(request, &proof) + } + return proof +} + +func (f *mappedWorkloadFixtureV1) RetireMappedWorkload(ctx context.Context, state deploy.StateV1, request mappedWorkloadRequestV1) (mappedWorkloadProofV1, error) { + f.t.Helper() + if err := f.ops.Operation.RequireHeld(); err != nil || ctx.Err() != nil { + f.t.Fatalf("retirement lost its lock/context: %v", err) + } + record, found, err := f.ops.Operation.ReadWorkloadLifecycleV1() + if err != nil || !found || record.State != "stopping" || record.ContainerID != f.old.ContainerID || request.Container != f.old.ContainerID || + request.Record != record || request.Engine != *state.IdentityMapping.Engine || request.Run != *state.IdentityMapping.NativeProcess { + f.t.Fatal("retirement did not select the persisted exact owner") + } + f.trace = append(f.trace, "retire") + if f.cancelOnRetire != nil { + f.cancelOnRetire() + } + return f.proof(request), f.retireErr +} + +func (f *mappedWorkloadFixtureV1) MappedWorkload(_ context.Context, _ deploy.StateV1, request mappedWorkloadRequestV1) (mappedWorkloadProofV1, error) { + f.trace = append(f.trace, request.Phase) + proof := f.proof(request) + if f.afterObserve != nil { + f.afterObserve(request) + } + return proof, f.observeErr +} + +func (f *mappedWorkloadFixtureV1) DetachMappedWorkload(_ context.Context, _ deploy.StateV1, request mappedWorkloadRequestV1) error { + f.t.Helper() + record, found, err := f.ops.Operation.ReadWorkloadLifecycleV1() + if err != nil || !found || record != request.Record || record.State != "starting" || record.ContainerID != f.live.waiter.id || + !record.TransferClaimed || record.HandoffComplete || !record.StartupComplete { + f.t.Fatal("stream detachment preceded its durable exact-instance claim") + } + f.trace = append(f.trace, "detach") + return f.detachErr +} + +func (f *mappedWorkloadFixtureV1) reload(ctx context.Context, ops deploy.IdentityMappingOperationsV1) (mappedEntryInputV1, error) { + f.t.Helper() + f.trace = append(f.trace, "reload") + if ctx.Err() != nil || ops.Operation.RequireHeld() != nil { + f.t.Fatal("fresh reader lost its held lock/context") + } + s, err := ops.Status() + if err != nil || s.State.IdentityMapping.PendingOperation != nil { + f.t.Fatal("fresh reader preceded exact native retirement") + } + record, found, err := ops.Operation.ReadWorkloadLifecycleV1() + if err != nil || !found || record.State != "down" || record.ContainerID != f.old.ContainerID { + f.t.Fatal("fresh reader preceded persisted old completion") + } + input := f.live.input + state := cloneMappedWorkloadStateFixtureV1(f.t, s.State) + template := cloneMappedWorkloadStateFixtureV1(f.t, f.live.input.Current.State).IdentityMapping.PendingOperation + template.ExpectedRevision, template.ExpectedGeneration = state.IdentityMapping.Revision, state.Current + state.IdentityMapping.PendingOperation = template + input.Current.State = state + input.Current.Generation = *state.Current + if f.changeInput != nil { + f.changeInput(&input) + } + writeMappedLiveStateFixtureV1(f.t, ops.Operation, state) + return input, nil +} + +func (f *mappedWorkloadFixtureV1) prepare() (mappedEntryPlanV1, error) { + return prepareMappedWorkloadReplacementV1(f.live.waiter.ctx, f.ops, f.reload) +} + +func (f *mappedWorkloadFixtureV1) released(plan mappedEntryPlanV1) *ownedMappedWaiterV1 { + f.t.Helper() + w := f.live.waiter + facts, err := plan.facts() + if err != nil { + f.t.Fatal(err) + } + status, err := f.ops.Status() + if err != nil { + f.t.Fatal(err) + } + // Reuse the existing owned-waiter transport fixture. This models the + // atomic writer result only; actual host-checked admission is tested in E1.2. + w.backend.admit = func(ops deploy.IdentityMappingOperationsV1, ctx context.Context, state deploy.StateV1, id string) (deploy.StateV1, error) { + if ctx.Err() != nil || ops.Operation.RequireHeld() != nil || id != facts.Operation { + return deploy.StateV1{}, errors.New("wrong admission owner") + } + m := state.IdentityMapping + p := m.PendingOperation + a := p.Native.Active + m.UIDAssignments, m.GIDAssignments = a.UIDs, a.GIDs + m.AcceptedUIDs, m.AcceptedGIDs = &a.UIDs, &a.GIDs + m.AdmittedGeneration, m.Engine, m.NativeProcess = state.Current, &p.Native.Engine, &p.Native.Process + r, _ := strconv.ParseUint(m.Revision, 10, 64) + m.Revision = strconv.FormatUint(r+1, 10) + p.ExpectedRevision, p.Phase = m.Revision, "release-attempted" + writeMappedLiveStateFixtureV1(f.t, ops.Operation, state) + return state, nil + } + f.live.untilCancel = true + f.live.plan = plan + spec, policy, err := mappedLiveCreateSpecV1(f.ops, status.State, f.live.directory, facts) + if err != nil { + f.t.Fatal(err) + } + waiter, err := f.live.backend.start(w.ctx, f.ops, status.State, mappedStateGeometryV1(facts.Active), facts.Operation, spec, policy, plan.binding, w.options) + if err != nil { + f.t.Fatal(err) + } + f.t.Cleanup(func() { _ = waiter.close() }) + if err := f.ops.Operation.RecordLiveRunContainerV1(f.live.run.ID, waiter.container); err != nil { + f.t.Fatal(err) + } + if _, err := waiter.release(); err != nil { + f.t.Fatal(err) + } + return waiter +} + +func TestMappedWorkloadReplacementOrdersRetirementAndFreshAdmissionV1(t *testing.T) { + for _, profile := range []string{"exact", "bounded"} { + t.Run(profile, func(t *testing.T) { + f := newMappedWorkloadFixtureV1(t, profile) + before, _ := f.ops.Status() + plan, err := f.prepare() + if err != nil { + t.Fatal(err) + } + if !slices.Equal(f.trace, []string{"retire", "reload"}) || f.live.waiter.created != 0 { + t.Fatal("replacement bypassed ordered trusted retirement or created early") + } + record, _, err := f.ops.Operation.ReadWorkloadLifecycleV1() + if err != nil || record.State != "starting" || record.ContainerID != "" || record.StartupComplete || record.Runtime != strings.TrimPrefix(string(plan.binding.Plan), "sha256:") { + t.Fatal("replacement persisted stale startup evidence") + } + current, _ := f.ops.Status() + if !reflect.DeepEqual(before.State.IdentityMapping.UIDAssignments, current.State.IdentityMapping.UIDAssignments) || + !reflect.DeepEqual(before.State.IdentityMapping.AcceptedUIDs, current.State.IdentityMapping.AcceptedUIDs) || + current.State.IdentityMapping.PendingOperation.OperationID == "old-operation" || plan.binding == f.live.plan.binding { + t.Fatal("replacement lost history or reused original final plan") + } + }) + } +} + +func TestMappedWorkloadRetirementUncertaintyBlocksReloadV1(t *testing.T) { + for _, profile := range []string{"exact", "bounded"} { + for _, name := range []string{"stop error", "still running", "not retired", "wrong digest", "wrong instance", "wrong start", "restart", "autoremove", "native recovery"} { + t.Run(profile+"/"+name, func(t *testing.T) { + f := newMappedWorkloadFixtureV1(t, profile) + switch name { + case "stop error": + f.retireErr = errors.New("owned stop unproved") + case "native recovery": + f.fail = true + state, _ := f.ops.Status() + state.State.IdentityMapping.PendingOperation = f.pending + writeMappedLiveStateFixtureV1(t, f.ops.Operation, state.State) + default: + f.changeProof = func(_ mappedWorkloadRequestV1, p *mappedWorkloadProofV1) { + switch name { + case "still running": + p.Running = true + case "not retired": + p.Retired = false + case "wrong digest": + p.Request = canonical.Digest(strings.Repeat("f", 64)) + case "wrong instance": + p.Container = strings.Repeat("b", 64) + case "wrong start": + p.StartedAt = "2026-01-03T00:00:00Z" + case "restart": + p.RestartPolicy = "always" + case "autoremove": + p.AutoRemove = true + } + } + } + if _, err := f.prepare(); err == nil || !slices.Equal(f.trace, []string{"retire"}) { + t.Fatal("uncertain retirement reached fresh admission") + } + r, _, _ := f.ops.Operation.ReadWorkloadLifecycleV1() + s, _ := f.ops.Status() + if r.State != "stopping" || r.ContainerID != f.old.ContainerID || name == "native recovery" && s.State.IdentityMapping.PendingOperation == nil || len(s.State.IdentityMapping.UIDAssignments) == 0 { + t.Fatal("uncertainty erased old ownership or reservations") + } + }) + } + } +} + +func TestMappedWorkloadReplacementRejectsCachedOrWrongEntryV1(t *testing.T) { + for _, name := range []string{"old process", "application entry", "unlocked snapshot", "wrong generation"} { + t.Run(name, func(t *testing.T) { + f := newMappedWorkloadFixtureV1(t, "exact") + f.changeInput = func(input *mappedEntryInputV1) { + switch name { + case "old process": + input.Current.State.IdentityMapping.PendingOperation.Native.Process = "old-process" + case "application entry": + input.Kind = mappedApplicationCommandV1 + case "unlocked snapshot": + input.Current.State = cloneMappedWorkloadStateFixtureV1(t, input.Current.State) + input.Current.State.BlueprintSource = "not the locked owner" + case "wrong generation": + input.Current.Generation.Reference = "changed" + } + } + if _, err := f.prepare(); err == nil || f.live.waiter.created != 0 { + t.Fatal("cached or unrelated replacement accepted") + } + record, _, _ := f.ops.Operation.ReadWorkloadLifecycleV1() + if record.State != "down" || record.StartedAt != f.old.StartedAt { + t.Fatal("rejected fresh admission replaced old completed record") + } + }) + } +} + +func TestMappedWorkloadRequiresTrustedBoundedOwnerBeforeEffectsV1(t *testing.T) { + for _, name := range []string{"missing provider", "no context", "unbounded", "cancelled", "no reader", "old endpoint", "old identity missing"} { + t.Run(name, func(t *testing.T) { + f := newMappedWorkloadFixtureV1(t, "exact") + ctx, reload := f.live.waiter.ctx, f.reload + switch name { + case "missing provider": + f.ops.Evidence = f.waiterEffectFixtureV1 + case "no context": + ctx = nil + case "unbounded": + ctx = context.Background() + case "cancelled": + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, time.Second) + cancel() + case "no reader": + reload = nil + case "old endpoint": + r := f.old + r.Endpoint = strings.Repeat("e", 64) + if err := f.ops.Operation.WriteWorkloadLifecycleV1(r); err != nil { + t.Fatal(err) + } + case "old identity missing": + r := f.old + r.ContainerID, r.StartedAt, r.State, r.StartupComplete = "", "", "starting", false + if err := f.ops.Operation.WriteWorkloadLifecycleV1(r); err != nil { + t.Fatal(err) + } + } + if _, err := prepareMappedWorkloadReplacementV1(ctx, f.ops, reload); err == nil || len(f.trace) != 0 { + t.Fatal("unqualified owner reached lifecycle effects") + } + }) + } +} + +func TestMappedWorkloadStartupAndQualifiedHandoffV1(t *testing.T) { + for _, profile := range []string{"exact", "bounded"} { + for _, detached := range []bool{false, true} { + t.Run(profile+"/"+strconv.FormatBool(detached), func(t *testing.T) { + f := newMappedWorkloadFixtureV1(t, profile) + plan, err := f.prepare() + if err != nil { + t.Fatal(err) + } + waiter := f.released(plan) + if err := confirmMappedWorkloadStartupV1(f.live.waiter.ctx, f.ops, plan, waiter, detached); err != nil { + t.Fatal(err) + } + record, _, _ := f.ops.Operation.ReadWorkloadLifecycleV1() + if record.State != "up" || !record.StartupComplete || record.ContainerID != waiter.container || record.StartedAt == f.old.StartedAt || record.HandoffComplete != detached || record.TransferClaimed != detached { + t.Fatal("lifecycle did not preserve exact startup/handoff identity") + } + want := []string{"retire", "reload", "attached"} + if detached { + want = append(want, "detach", "detached") + } + if !slices.Equal(f.trace, want) { + t.Fatalf("startup/detachment observation order: %v", f.trace) + } + if err := confirmMappedWorkloadStartupV1(f.live.waiter.ctx, f.ops, plan, waiter, detached); err == nil || !slices.Equal(f.trace, want) { + t.Fatal("completed instance resent startup/detachment") + } + }) + } + } +} + +func TestMappedWorkloadMissingStartupOrDetachProofRetainsUncertaintyV1(t *testing.T) { + for _, name := range []string{"provider error", "running only", "exited", "wrong instance", "wrong digest", "invalid start", "restart policy", "restart count", "autoremove", "early detach", "detach error", "detach unproved", "post-detach identity", "post-detach exited"} { + t.Run(name, func(t *testing.T) { + f := newMappedWorkloadFixtureV1(t, "exact") + plan, err := f.prepare() + if err != nil { + t.Fatal(err) + } + waiter := f.released(plan) + detach := strings.HasPrefix(name, "detach") || strings.HasPrefix(name, "post-detach") + if name == "provider error" { + f.observeErr = errors.New("F unavailable") + } + if name == "detach error" { + f.detachErr = errors.New("transport uncertain") + } + f.changeProof = func(r mappedWorkloadRequestV1, p *mappedWorkloadProofV1) { + switch name { + case "running only": + p.StartupComplete = false + case "exited": + p.Running = false + case "wrong instance": + p.Container = f.old.ContainerID + case "wrong digest": + p.Request = canonical.Digest(strings.Repeat("f", 64)) + case "invalid start": + p.StartedAt = "not-a-time" + case "restart policy": + p.RestartPolicy = "always" + case "restart count": + p.RestartCount = 1 + case "autoremove": + p.AutoRemove = true + case "early detach": + p.Detached = true + case "detach unproved": + p.Detached = false + case "post-detach identity": + if r.Phase == "detached" { + p.StartedAt = "2026-01-03T00:00:00Z" + } + case "post-detach exited": + if r.Phase == "detached" { + p.Running = false + } + } + } + if err := confirmMappedWorkloadStartupV1(f.live.waiter.ctx, f.ops, plan, waiter, detach); err == nil { + t.Fatal("unqualified startup/handoff was accepted") + } + record, _, _ := f.ops.Operation.ReadWorkloadLifecycleV1() + s, _ := f.ops.Status() + if record.State != "starting" || record.HandoffComplete || s.State.IdentityMapping.PendingOperation == nil || len(s.State.IdentityMapping.UIDAssignments) == 0 { + t.Fatal("startup uncertainty lost recovery owner or reserves") + } + before := slices.Clone(f.trace) + if detach && record.TransferClaimed { + if err := confirmMappedWorkloadStartupV1(f.live.waiter.ctx, f.ops, plan, waiter, true); err == nil || !slices.Equal(before, f.trace) { + t.Fatal("uncertain detachment repeated its transfer") + } + } + }) + } +} + +func TestMappedWorkloadStartupRefusesChangedOwnerBeforeObservationV1(t *testing.T) { + for _, name := range []string{"no release", "closed waiter", "wrong binding", "wrong operation lock", "missing provider", "changed native process", "changed revision", "changed lifecycle"} { + t.Run(name, func(t *testing.T) { + f := newMappedWorkloadFixtureV1(t, "bounded") + plan, err := f.prepare() + if err != nil { + t.Fatal(err) + } + waiter := f.released(plan) + switch name { + case "no release": + waiter.mu.Lock() + waiter.consumed = false + waiter.mu.Unlock() + case "closed waiter": + if err := waiter.close(); err != nil { + t.Fatal(err) + } + case "wrong binding": + plan.binding.Operation = "foreign" + case "wrong operation lock": + f.ops.Operation = nil + case "missing provider": + f.ops.Evidence = f.waiterEffectFixtureV1 + case "changed native process", "changed revision": + status, _ := f.ops.Status() + if name == "changed native process" { + status.State.IdentityMapping.PendingOperation.Native.Process = "foreign" + *status.State.IdentityMapping.NativeProcess = "foreign" + } else { + status.State.IdentityMapping.Revision = "42" + status.State.IdentityMapping.PendingOperation.ExpectedRevision = "42" + } + writeMappedLiveStateFixtureV1(t, f.ops.Operation, status.State) + case "changed lifecycle": + record, _, _ := f.ops.Operation.ReadWorkloadLifecycleV1() + record.Runtime = strings.Repeat("e", 64) + if err := f.ops.Operation.WriteWorkloadLifecycleV1(record); err != nil { + t.Fatal(err) + } + } + trace := slices.Clone(f.trace) + if err := confirmMappedWorkloadStartupV1(f.live.waiter.ctx, f.ops, plan, waiter, true); err == nil || !slices.Equal(trace, f.trace) { + t.Fatal("changed owner reached startup or detachment observation") + } + }) + } +} + +func TestMappedWorkloadCancellationCannotRetireOrReloadV1(t *testing.T) { + f := newMappedWorkloadFixtureV1(t, "exact") + ctx, cancel := context.WithTimeout(f.live.waiter.ctx, time.Second) + defer cancel() + f.cancelOnRetire = cancel + if _, err := prepareMappedWorkloadReplacementV1(ctx, f.ops, f.reload); err == nil || !slices.Equal(f.trace, []string{"retire"}) { + t.Fatal("cancelled retirement reached fresh admission") + } + record, _, err := f.ops.Operation.ReadWorkloadLifecycleV1() + if err != nil || record.State != "stopping" || record.ContainerID != f.old.ContainerID { + t.Fatal("cancellation cleared exact old ownership") + } +} + +func TestMappedWorkloadOwnerCloseDuringObservationCannotCommitUpV1(t *testing.T) { + for _, detached := range []bool{false, true} { + t.Run(strconv.FormatBool(detached), func(t *testing.T) { + f := newMappedWorkloadFixtureV1(t, "exact") + plan, err := f.prepare() + if err != nil { + t.Fatal(err) + } + waiter := f.released(plan) + f.afterObserve = func(request mappedWorkloadRequestV1) { + if request.Phase == "detached" || !detached { + if err := waiter.close(); err != nil { + t.Fatal(err) + } + } + } + if err := confirmMappedWorkloadStartupV1(f.live.waiter.ctx, f.ops, plan, waiter, detached); err == nil { + t.Fatal("closed lifetime owner was recorded up") + } + record, _, err := f.ops.Operation.ReadWorkloadLifecycleV1() + if err != nil || record.State != "starting" || record.HandoffComplete { + t.Fatal("owner loss completed lifecycle handoff") + } + }) + } +} + +func TestMappedWorkloadOwnerLossBeforeDetachHasNoTransferEffectsV1(t *testing.T) { + for _, profile := range []string{"exact", "bounded"} { + t.Run(profile, func(t *testing.T) { + f := newMappedWorkloadFixtureV1(t, profile) + plan, err := f.prepare() + if err != nil { + t.Fatal(err) + } + waiter := f.released(plan) + f.afterObserve = func(request mappedWorkloadRequestV1) { + if request.Phase == "attached" { + if err := waiter.close(); err != nil { + t.Fatal(err) + } + } + } + if err := confirmMappedWorkloadStartupV1(f.live.waiter.ctx, f.ops, plan, waiter, true); err == nil { + t.Fatal("lost foreground owner completed detached startup") + } + if !slices.Equal(f.trace, []string{"retire", "reload", "attached"}) { + t.Fatalf("lost foreground owner reached detach effects: %v", f.trace) + } + record, _, err := f.ops.Operation.ReadWorkloadLifecycleV1() + if err != nil || record.State != "starting" || record.TransferClaimed || record.HandoffComplete || record.ContainerID != "" { + t.Fatal("lost foreground owner persisted a transfer claim") + } + }) + } +}