diff --git a/internal/dockerdeploy/live_run_mapped.go b/internal/dockerdeploy/live_run_mapped.go new file mode 100644 index 00000000..57e535c1 --- /dev/null +++ b/internal/dockerdeploy/live_run_mapped.go @@ -0,0 +1,376 @@ +package dockerdeploy + +import ( + "context" + "errors" + "fmt" + "io" + "path/filepath" + "reflect" + "slices" + "strconv" + "strings" + "sync/atomic" + "time" + + "github.com/omry/reploy/internal/canonical" + "github.com/omry/reploy/internal/deploy" + "github.com/omry/reploy/internal/identitymapping" + "github.com/omry/reploy/internal/mappedlaunch" +) + +var errMappedMasterExitedV1 = errors.New("required mapped master exited; instance retired") + +type mappedLiveOwnedInputV1 struct { + io.ReadCloser + stopping atomic.Bool +} + +func (input *mappedLiveOwnedInputV1) Read(buffer []byte) (int, error) { + n, err := input.ReadCloser.Read(buffer) + if err != nil && input.stopping.Load() { + return n, io.EOF + } + return n, err +} + +func (input *mappedLiveOwnedInputV1) Close() error { + input.stopping.Store(true) + return input.ReadCloser.Close() +} + +type mappedLiveRunBackendV1 struct { + acquire func(context.Context, string) (*deploy.OperationLock, error) + start func(context.Context, deploy.IdentityMappingOperationsV1, deploy.StateV1, identitymapping.Geometry, string, CommandSpec, mappedSeccompPolicyV1, mappedlaunch.Binding, RunOptions) (*ownedMappedWaiterV1, error) + recover func(deploy.IdentityMappingOperationsV1, context.Context, deploy.StateV1, string) (deploy.StateV1, error) +} + +// The caller owns the existing queue-entry lease. This function takes the +// operation lock and retains it through live ownership and the release attempt. +// Enabled application stdin transfers ownership to this private operation: its +// Close must interrupt Read. Borrowed readers are refused before engine effects. +// It is private plumbing: no public mapped guard or F qualification is supplied. +func runMappedLiveInstanceV1(admission context.Context, directory string, ops deploy.IdentityMappingOperationsV1, runID string, plan mappedEntryPlanV1, options RunOptions) error { + return runMappedLiveInstanceWithBackendV1(admission, directory, ops, runID, plan, options, mappedLiveRunBackendV1{ + acquire: deploy.AcquireOperationLock, + start: func(ctx context.Context, ops deploy.IdentityMappingOperationsV1, state deploy.StateV1, active identitymapping.Geometry, operation string, spec CommandSpec, policy mappedSeccompPolicyV1, binding mappedlaunch.Binding, options RunOptions) (*ownedMappedWaiterV1, error) { + p, err := prepareMappedPodmanCreateV1(ctx, ops, state, active, operation, spec, policy) + if err != nil { + return nil, err + } + return startOwnedMappedWaiterV1(ctx, p, binding, options) + }, + recover: func(ops deploy.IdentityMappingOperationsV1, ctx context.Context, state deploy.StateV1, operation string) (deploy.StateV1, error) { + return ops.Recover(ctx, state, operation, false) + }, + }) +} + +func runMappedLiveInstanceWithBackendV1(admission context.Context, directory string, ops deploy.IdentityMappingOperationsV1, runID string, plan mappedEntryPlanV1, options RunOptions, backend mappedLiveRunBackendV1) (failure error) { + operation := ops.Operation + if operation == nil { + return fmt.Errorf("mapped instance requires its operation lock") + } + defer func() { + if operation != nil { + failure = errors.Join(failure, operation.Unlock()) + } + }() + if err := operation.RequireWritable(); err != nil { + return err + } + if admission == nil || options.Context == nil || backend.acquire == nil || backend.start == nil || backend.recover == nil { + return fmt.Errorf("mapped instance requires admission, lifetime and complete backend") + } + if _, bounded := admission.Deadline(); !bounded || admission.Err() != nil || options.Context.Err() != nil { + return fmt.Errorf("mapped instance requires live lifetime and bounded admission") + } + absolute, err := filepath.Abs(directory) + if err != nil || directory == "" || filepath.Dir(filepath.Dir(operation.Path())) != absolute { + return fmt.Errorf("mapped instance lock belongs to a different deployment") + } + facts, err := plan.facts() + if err != nil { + return err + } + if !facts.Stdin && options.Stdin != nil { + return fmt.Errorf("mapped instance stdin is disabled by its entry plan") + } + var input io.ReadCloser + if options.Stdin != nil { + var owned bool + input, owned = options.Stdin.(io.ReadCloser) + if !owned { + return fmt.Errorf("mapped instance stdin requires owned interruptible input") + } + input = &mappedLiveOwnedInputV1{ReadCloser: input} + options.Stdin = input + } + stopInput := func() error { + if input == nil { + return nil + } + owned := input + input = nil + return owned.Close() + } + defer func() { failure = errors.Join(failure, stopInput()) }() + kind, err := strconv.Atoi(facts.Kind) + if err != nil || facts.Mode != strconv.Itoa(int(mappedEntryFreshInstanceV1)) { + return fmt.Errorf("mapped instance requires a fresh entry plan") + } + master := mappedEntryKindV1(kind) == mappedWorkloadStartV1 || mappedEntryKindV1(kind) == mappedHostReplacementV1 + if !master && mappedEntryKindV1(kind) != mappedApplicationCommandV1 && mappedEntryKindV1(kind) != mappedRuntimeTestV1 && mappedEntryKindV1(kind) != mappedLifecycleHookV1 { + return fmt.Errorf("mapped instance entry kind is unsupported") + } + if err := operation.RequireQueueEntryLeaseHeldV1(runID); err != nil { + return err + } + queue, _, err := operation.ReadLiveRunQueueV1() + if err != nil { + return err + } + run, found := findLiveRunV1(queue, runID) + if !found || run.Status != deploy.LiveRunStatusActiveV1 || run.Container != "" || run.Name != facts.Command { + return fmt.Errorf("mapped instance requires its unstarted active live entry") + } + status, err := ops.Status() + if err != nil { + return err + } + state := status.State + if state.Current == nil || run.GenerationReference != state.Current.Reference || state.Current.BuildLockDigest != facts.Generation || state.IdentityMapping == nil || state.IdentityMapping.InstallationID != facts.Installation || state.IdentityMapping.Revision != facts.Revision { + return fmt.Errorf("mapped instance generation or mapping changed") + } + active := mappedStateGeometryV1(facts.Active) + pending := state.IdentityMapping.PendingOperation + if pending == nil || pending.Kind != "native-execute" || pending.Phase != "effects-pending" || !pending.Inhibited || pending.OperationID != facts.Operation || pending.Native == nil || pending.Native.Engine != facts.Engine || pending.Native.Process != facts.Run || !reflect.DeepEqual(pending.Native.Active, facts.Active) { + return fmt.Errorf("mapped instance differs from native operation owner") + } + spec, policy, err := mappedLiveCreateSpecV1(ops, state, absolute, facts) + if err != nil { + return err + } + waiter, err := backend.start(admission, ops, state, active, facts.Operation, spec, policy, plan.binding, options) + if err != nil { + // Create may already have happened. Keep the existing native intent and + // queue; only the trusted effect owner can reconcile this uncertainty. + return fmt.Errorf("mapped create/start requires recovery: %w", err) + } + if waiter == nil || waiter.binding != plan.binding || waiter.prepared == nil || waiter.prepared.ops.Operation != operation || waiter.container == "" { + // Never clean an object returned under a different owner. + return fmt.Errorf("mapped waiter ownership differs; recovery required") + } + if err := operation.RecordLiveRunContainerV1(runID, waiter.container); err != nil { + return errors.Join(fmt.Errorf("record mapped live ownership: %w", err), waiter.close()) + } + run.Container = waiter.container + if _, err := waiter.release(); err != nil { + // release closes on failure, including uncertain frame writes. Its + // error is retained; a second idempotent close is not cleanup proof. + return fmt.Errorf("mapped release requires recovery: %w", err) + } + if err := operation.Unlock(); err != nil { + return errors.Join(fmt.Errorf("unlock mapped attached instance: %w", err), waiter.close()) + } + operation = nil + completionContext, cancelCompletion := context.WithCancel(options.Context) + defer cancelCompletion() + completion := make(chan error, 1) + go func() { completion <- waitMappedLiveExitV1(completionContext, waiter) }() + var runErr error + attachDone, completionDone := false, false + select { + case runErr = <-waiter.done: + attachDone = true + case runErr = <-completion: + completionDone = true + case <-options.Context.Done(): + runErr = options.Context.Err() + } + if attachDone && runErr == nil { + select { + case runErr = <-completion: + completionDone = true + case <-options.Context.Done(): + runErr = options.Context.Err() + case <-time.After(5 * time.Second): + runErr = fmt.Errorf("mapped engine completion is unverified") + } + } + // Cmd.Wait can await its stdin copier after the attach child exits. Stop + // the owned reader independently, rather than requiring application EOF. + inputErr := stopInput() + cancelCompletion() + if completionDone && !attachDone { + // Let the exited attach drain output after the owned input becomes EOF. + // Do not cancel it first and silently lose output or its real errors. + select { + case attachErr := <-waiter.done: + attachDone = true + runErr = errors.Join(runErr, attachErr) + case <-time.After(5 * time.Second): + runErr = errors.Join(runErr, fmt.Errorf("mapped attach drain is unverified")) + } + } + // The owned container is the lifetime boundary, including a lost attach. + // Stop/remove that exact instance before touching queue or journal state. + if err := waiter.close(); err != nil { + return errors.Join(runErr, inputErr, fmt.Errorf("mapped exact cleanup requires recovery: %w", err)) + } + if !attachDone { + select { + case attachErr := <-waiter.done: + if !completionDone { + runErr = errors.Join(runErr, attachErr) + } + case <-time.After(5 * time.Second): + return errors.Join(runErr, inputErr, fmt.Errorf("mapped attach did not stop; ownership retained")) + } + } + if !completionDone { + select { + case <-completion: + case <-time.After(5 * time.Second): + return errors.Join(runErr, inputErr, fmt.Errorf("mapped completion observer did not stop; ownership retained")) + } + } + if inputErr != nil { + return errors.Join(runErr, fmt.Errorf("mapped input closure unverified; ownership retained: %w", inputErr)) + } + cleanup, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + operation, err = backend.acquire(cleanup, absolute) + if err != nil { + return errors.Join(runErr, fmt.Errorf("reacquire mapped completion lock: %w", err)) + } + if operation == nil { + return errors.Join(runErr, fmt.Errorf("mapped completion has no operation lock")) + } + queue, _, err = operation.ReadLiveRunQueueV1() + if err != nil { + return errors.Join(runErr, err) + } + currentRun, found := findLiveRunV1(queue, runID) + if !found || !reflect.DeepEqual(currentRun, run) { + return errors.Join(runErr, fmt.Errorf("mapped completion owner changed; recovery required")) + } + if err := operation.RequireQueueEntryLeaseHeldV1(runID); err != nil { + return errors.Join(runErr, err) + } + ops.Operation = operation + status, err = ops.Status() + if err != nil { + return errors.Join(runErr, err) + } + current := status.State + m := current.IdentityMapping + if current.Current == nil || !reflect.DeepEqual(current.Current, state.Current) || m == nil || m.InstallationID != facts.Installation || m.PendingOperation == nil || m.PendingOperation.OperationID != facts.Operation || m.PendingOperation.Phase != "release-attempted" || m.PendingOperation.Native == nil || m.PendingOperation.Native.Engine != facts.Engine || m.PendingOperation.Native.Process != facts.Run { + return errors.Join(runErr, fmt.Errorf("mapped retirement journal changed; recovery required")) + } + // An engine removal result alone cannot clear release-attempted. Recover + // must independently verify the exact trusted effect-owner evidence. + retired, err := backend.recover(ops, cleanup, current, facts.Operation) + if err != nil { + return errors.Join(runErr, fmt.Errorf("mapped retirement unverified; ownership retained: %w", err)) + } + if retired.IdentityMapping == nil || retired.IdentityMapping.PendingOperation != nil { + return errors.Join(runErr, fmt.Errorf("mapped retirement did not complete its journal; ownership retained")) + } + if _, removed, err := operation.RemoveLiveRunV1(runID); err != nil || !removed { + return errors.Join(runErr, fmt.Errorf("complete mapped live entry requires recovery: %v", err)) + } + if master { + runErr = errors.Join(runErr, errMappedMasterExitedV1) + } + return runErr +} + +func waitMappedLiveExitV1(ctx context.Context, waiter *ownedMappedWaiterV1) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + status := newBoundedPortableToolProbeOutput(16, cancel) + spec := waiter.prepared.spec + spec.Args = []string{"wait", waiter.container} + if err := waiter.backend.run(spec, RunOptions{Context: ctx, Stdout: status}); err != nil { + return err + } + value := strings.TrimSpace(status.content.String()) + code, err := strconv.Atoi(value) + if ctx.Err() != nil || status.Truncated() || err != nil || code < 0 || code > 255 || strconv.Itoa(code) != value { + return fmt.Errorf("mapped engine completion is unverified") + } + if code != 0 { + return fmt.Errorf("mapped entry exited with status %d", code) + } + return nil +} + +// Render from frozen plan facts using the existing application renderer. No +// caller-provided raw command can substitute mounts, environment or startup. +func mappedLiveCreateSpecV1(ops deploy.IdentityMappingOperationsV1, state deploy.StateV1, directory string, facts mappedEntryPayloadV1) (CommandSpec, mappedSeccompPolicyV1, error) { + var empty mappedSeccompPolicyV1 + if ops.ValidateProfileOwner == nil { + return CommandSpec{}, empty, fmt.Errorf("mapped live plan requires locked build evidence") + } + lock, found, err := ops.Operation.ReadBuildLock(state.Current.BuildLockDigest, ops.ValidateProfileOwner) + if err != nil || !found || lock.FinalImage.ConfigDigest != facts.Image || state.Current.RuntimePolicyDigest != facts.RuntimePolicy { + return CommandSpec{}, empty, fmt.Errorf("mapped live plan build/image changed: %v", err) + } + network, _ := canonical.Sum("mapped-network", "v1", lock.RuntimePolicy.Network) + if network != facts.NetworkPolicy || facts.Kernel.validate() != nil { + return CommandSpec{}, empty, fmt.Errorf("mapped live plan policy changed") + } + uid, err := strconv.ParseUint(facts.UID, 10, 32) + if err != nil { + return CommandSpec{}, empty, err + } + gid, err := strconv.ParseUint(facts.GID, 10, 32) + if err != nil { + return CommandSpec{}, empty, err + } + groups := make([]uint32, len(facts.Groups)) + for i, value := range facts.Groups { + n, err := strconv.ParseUint(value, 10, 32) + if err != nil { + return CommandSpec{}, empty, err + } + groups[i] = uint32(n) + } + final, _ := strconv.ParseUint(facts.Kernel.FinalCapabilities, 16, 64) + setup, _ := strconv.ParseUint(facts.Kernel.SetupCapabilities, 16, 64) + last, _ := strconv.Atoi(facts.Kernel.KernelLastCapability) + policy, err := compileMappedSeccompV1(podmanDockerSeccompProfile, facts.Kernel.ABI, final, setup, last) + if err != nil { + return CommandSpec{}, empty, err + } + sandbox := newApplicationSandboxPlanWithNetworkV1(RuntimeUserPlan{UID: uint32(uid), GID: uint32(gid), DockerUser: facts.UID + ":" + facts.GID, SupplementaryGIDs: groups}, lock.RuntimePolicy.Network) + runtime := DockerExecutionPlan{DeploymentDir: directory, ContainerName: facts.Run, Image: state.Current.Reference, Mounts: facts.Mounts, Sandbox: sandbox} + spec, err := transientContainerCommandSpecV1("create", facts.Run, runtime, ResolvedEnvironmentCommand{Argv: facts.Argv}, nil, facts.Stdin, false) + if err != nil { + return CommandSpec{}, empty, err + } + // Reuse the existing Podman home/seed translation before replacing its + // ordinary startup with the complete already-bound mapped startup. + spec, err = preparePodmanSeccompCommand(spec) + if err != nil { + return CommandSpec{}, empty, err + } + // HOME/TMPDIR defaults from the ordinary renderer must not silently alter + // a mapped entry's frozen environment. All environment bytes are explicit. + entry := slices.Index(spec.Args, "--entrypoint") + args := make([]string, 0, len(spec.Args)+2*len(facts.Environment)) + for i := 0; i < entry; i++ { + if spec.Args[i] == "--env" { + i++ + continue + } + args = append(args, spec.Args[i]) + } + for _, value := range facts.Environment { + args = append(args, "--env", value) + } + args = append(args, spec.Args[entry:entry+3]...) + spec.Args = append(args, facts.Startup...) + return spec, policy, nil +} diff --git a/internal/dockerdeploy/live_run_mapped_test.go b/internal/dockerdeploy/live_run_mapped_test.go new file mode 100644 index 00000000..2f3e9394 --- /dev/null +++ b/internal/dockerdeploy/live_run_mapped_test.go @@ -0,0 +1,662 @@ +package dockerdeploy + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "slices" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/omry/reploy/internal/canonical" + "github.com/omry/reploy/internal/deploy" + "github.com/omry/reploy/internal/identitymapping" + "github.com/omry/reploy/internal/mappedlaunch" + "github.com/omry/reploy/internal/providers/registry" +) + +// Real locks, leases, queue and immutable build readers surround the existing +// waiter fixture. Host admission/F/kernel effects remain explicitly injected. +// These are private orchestration proofs, not production runtime qualification. +type mappedLiveFixtureV1 struct { + waiter *waiterFixtureV1 + plan mappedEntryPlanV1 + input mappedEntryInputV1 + run deploy.LiveRunV1 + lease *deploy.QueueEntryLeaseV1 + backend mappedLiveRunBackendV1 + directory string + recoveries, starts int + retirementError error + incompleteRetirement bool + attachErr error + exitStatus string + untilCancel bool + beforeRecovery func(*deploy.OperationLock) + beforeStart func() + exited chan struct{} +} + +func writeMappedLiveStateFixtureV1(t *testing.T, operation *deploy.OperationLock, state deploy.StateV1) { + t.Helper() + encoded, err := deploy.EncodeStateV1(state) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(filepath.Dir(operation.Path()), "state.json"), encoded, 0600); err != nil { + t.Fatal(err) + } +} + +func newMappedLiveFixtureV1(t *testing.T, profile string, kind mappedEntryKindV1) *mappedLiveFixtureV1 { + t.Helper() + f := &mappedLiveFixtureV1{waiter: newWaiterFixtureV1(t), exited: make(chan struct{})} + f.exitStatus = "0\n" + w := f.waiter + w.options.Stdin = io.NopCloser(w.options.Stdin) + operation := w.prepared.ops.Operation + f.directory = filepath.Dir(filepath.Dir(operation.Path())) + i := mappedEntryPlanFixtureV1(t, profile) + i.Kind = kind + i.Environment = []string{"HOME=/tmp/explicit", "TMPDIR=/tmp/explicit", "TOKEN=dummy"} + i.Current.State.IdentityMapping.OwnerDir = f.directory + i.Current.Generation.Reference = w.prepared.state.Current.Reference + i.Current.State.Current = &i.Current.Generation + i.Current.State.IdentityMapping.PendingOperation.ExpectedGeneration = i.Current.State.Current + i.Runtime.Image = i.Current.Generation.Reference + i.Current.State.IdentityMapping.PendingOperation.Native.Engine = w.prepared.endpoint + i.Current.State.IdentityMapping.PendingOperation.Native.Process = "mapped-fixture-instance" + plan, err := constructMappedEntryPlanV1(i) + if err != nil { + t.Fatal(err) + } + if _, err := operation.PublishBuildLock(i.Current.Lock, registry.ValidateRequirementProfileV1); err != nil { + t.Fatal(err) + } + writeMappedLiveStateFixtureV1(t, operation, i.Current.State) + f.input, f.plan = i, plan + f.run = liveRunAdmissionFixtureV1("run-0000000000000041", false) + f.run.Name, f.run.GenerationReference = i.Command, i.Current.Generation.Reference + f.lease, err = operation.AcquireLiveRunLeaseV1(f.run.ID) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = f.lease.Close() }) + if _, err := operation.AdmitLiveRunV1(f.run, false); err != nil { + t.Fatal(err) + } + queue, _, err := operation.ReadLiveRunQueueV1() + if err != nil { + t.Fatal(err) + } + f.run = queue.Runs[0] + f.backend = mappedLiveRunBackendV1{ + acquire: deploy.AcquireOperationLock, + start: func(ctx context.Context, ops deploy.IdentityMappingOperationsV1, state deploy.StateV1, active identitymapping.Geometry, id string, spec CommandSpec, policy mappedSeccompPolicyV1, binding mappedlaunch.Binding, options RunOptions) (*ownedMappedWaiterV1, error) { + f.starts++ + if f.beforeStart != nil { + f.beforeStart() + } + if err := operation.RequireHeld(); err != nil { + return nil, err + } + w.prepared.ops, w.prepared.state = ops, state + w.prepared.active, w.prepared.operation = active, id + w.prepared.spec, w.prepared.policy, w.binding = spec, policy, binding + return startMappedWaiterWithBackendV1(ctx, w.prepared, binding, options, w.backend) + }, + recover: func(ops deploy.IdentityMappingOperationsV1, ctx context.Context, state deploy.StateV1, id string) (deploy.StateV1, error) { + f.recoveries++ + if f.beforeRecovery != nil { + f.beforeRecovery(ops.Operation) + } + if err := ops.Operation.RequireHeld(); err != nil { + return deploy.StateV1{}, err + } + if ctx.Err() != nil || w.removed != 1 || state.IdentityMapping.PendingOperation.Phase != "release-attempted" || id != bindingOperation(f.plan) { + return deploy.StateV1{}, errors.New("retirement not bound to exact cleaned attempt") + } + if f.retirementError != nil { + return deploy.StateV1{}, f.retirementError + } + if f.incompleteRetirement { + return state, nil + } + state.IdentityMapping.PendingOperation = nil + state.IdentityMapping.Revision = "3" + writeMappedLiveStateFixtureV1(t, ops.Operation, state) + return state, nil + }, + } + originalAdmit := w.backend.admit + w.backend.admit = func(ops deploy.IdentityMappingOperationsV1, ctx context.Context, state deploy.StateV1, id string) (deploy.StateV1, error) { + queue, _, err := operation.ReadLiveRunQueueV1() + if err != nil || len(queue.Runs) == 0 || queue.Runs[0].Container != w.id { + return deploy.StateV1{}, errors.New("release preceded durable exact live ownership") + } + if _, err := originalAdmit(ops, ctx, state, id); err != nil { + return deploy.StateV1{}, err + } + // E1.2 tests the actual host-checked atomic writer. This injected + // effect models only its validated release-attempted result. + m := state.IdentityMapping + active := m.PendingOperation.Native.Active + m.UIDAssignments, m.GIDAssignments = active.UIDs, active.GIDs + m.AcceptedUIDs, m.AcceptedGIDs = &active.UIDs, &active.GIDs + m.AdmittedGeneration, m.Engine, m.NativeProcess = state.Current, &m.PendingOperation.Native.Engine, &m.PendingOperation.Native.Process + m.Revision, m.PendingOperation.ExpectedRevision, m.PendingOperation.Phase = "2", "2", "release-attempted" + writeMappedLiveStateFixtureV1(t, operation, state) + return state, nil + } + originalRun := w.backend.run + w.backend.run = func(spec CommandSpec, options RunOptions) error { + if slices.Equal(spec.Args, []string{"wait", w.id}) { + select { + case <-f.exited: + case <-options.Context.Done(): + return options.Context.Err() + } + if options.Stdin != nil || options.Stdout == nil { + return errors.New("wait must use engine status, not application streams") + } + _, err := fmt.Fprint(options.Stdout, f.exitStatus) + return err + } + if spec.Args[0] != "start" { + return originalRun(spec, options) + } + defer close(f.exited) + if err := originalRun(spec, options); err != nil { + return err + } + // The frame/stream worker can finish before the caller unlocks. + // No blocking helper reads the operation lock during release. + if f.untilCancel { + <-options.Context.Done() + return options.Context.Err() + } + return f.attachErr + } + return f +} + +func bindingOperation(p mappedEntryPlanV1) string { return p.binding.Operation } + +func (f *mappedLiveFixtureV1) execute() error { + return runMappedLiveInstanceWithBackendV1(f.waiter.ctx, f.directory, f.waiter.prepared.ops, f.run.ID, f.plan, f.waiter.options, f.backend) +} + +func (f *mappedLiveFixtureV1) inspect(t *testing.T) (*deploy.OperationLock, deploy.LiveRunQueueV1, deploy.StateV1) { + t.Helper() + operation, err := deploy.AcquireOperationLock(t.Context(), f.directory) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = operation.Unlock() }) + queue, _, err := operation.ReadLiveRunQueueV1() + if err != nil { + t.Fatal(err) + } + state, _, err := operation.ReadStateV1() + if err != nil { + t.Fatal(err) + } + return operation, queue, state +} + +func TestMappedLiveInstanceOwnsReleaseAndFiniteCompletionV1(t *testing.T) { + for _, profile := range []string{"exact", "bounded"} { + for _, kind := range []mappedEntryKindV1{mappedApplicationCommandV1, mappedRuntimeTestV1, mappedLifecycleHookV1, mappedWorkloadStartV1, mappedHostReplacementV1} { + t.Run(fmt.Sprint(profile, kind), func(t *testing.T) { + f := newMappedLiveFixtureV1(t, profile, kind) + err := f.execute() + master := kind == mappedWorkloadStartV1 || kind == mappedHostReplacementV1 + if master && !errors.Is(err, errMappedMasterExitedV1) || !master && err != nil { + t.Fatalf("entry completion misclassified: %v", err) + } + _, queue, state := f.inspect(t) + if len(queue.Runs) != 0 || state.IdentityMapping.PendingOperation != nil || f.starts != 1 || f.recoveries != 1 || f.waiter.created != 1 || f.waiter.removed != 1 || f.waiter.admissions != 1 { + t.Fatal("instance did not complete exactly once") + } + if len(state.IdentityMapping.UIDAssignments) == 0 || state.IdentityMapping.AcceptedUIDs == nil || len(*state.IdentityMapping.AcceptedUIDs) == 0 { + t.Fatal("retirement released retained mapping history") + } + select { + case frame := <-f.waiter.frame: + if frame.Binding != f.plan.binding || frame.Container != f.waiter.id { + t.Fatal("wrong release frame") + } + default: + t.Fatal("release was not delivered") + } + }) + } + } +} + +func TestMappedLiveInstanceCancellationReacquiresLockAndRetainsLeaseV1(t *testing.T) { + f := newMappedLiveFixtureV1(t, "exact", mappedApplicationCommandV1) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + f.waiter.options.Context, f.untilCancel = ctx, true + f.beforeRecovery = func(operation *deploy.OperationLock) { + if err := operation.RequireQueueEntryLeaseHeldV1(f.run.ID); err != nil { + t.Error("live lease lost during attached run", err) + } + } + result := make(chan error, 1) + go func() { result <- f.execute() }() + select { + case <-f.waiter.frame: + case <-time.After(2 * time.Second): + t.Fatal("release blocked") + } + // An actual independent lock acquisition proves the attached path yielded + // the operation lock, while its live lease and exact owner remain durable. + operation, err := deploy.AcquireOperationLock(t.Context(), f.directory) + if err != nil { + t.Fatal(err) + } + if err := operation.RequireQueueEntryLeaseHeldV1(f.run.ID); err != nil { + t.Fatal(err) + } + queue, _, err := operation.ReadLiveRunQueueV1() + if err != nil || len(queue.Runs) != 1 || queue.Runs[0].Container != f.waiter.id { + t.Fatal("attached run lost exact ownership", err) + } + _ = operation.Unlock() + cancel() + select { + case err := <-result: + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation lost: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("cancelled instance did not finish") + } + _, queue, state := f.inspect(t) + if len(queue.Runs) != 0 || state.IdentityMapping.PendingOperation != nil || f.waiter.removed != 1 { + t.Fatal("cancelled instance not retired") + } +} + +func TestMappedLiveInstanceUncertaintyRetainsExactOwnershipV1(t *testing.T) { + for _, mode := range []string{"create", "F", "admission", "partial frame", "cleanup", "retirement", "incomplete retirement", "reacquire", "changed journal", "changed queue", "input close"} { + t.Run(mode, func(t *testing.T) { + f := newMappedLiveFixtureV1(t, "exact", mappedApplicationCommandV1) + w := f.waiter + switch mode { + case "input close": + w.options.Stdin = mappedLiveCloseFailureV1{w.options.Stdin.(io.ReadCloser)} + case "create": + w.createFailure = true + case "F": + w.effect.fail = true + case "admission": + w.backend.admit = func(deploy.IdentityMappingOperationsV1, context.Context, deploy.StateV1, string) (deploy.StateV1, error) { + return deploy.StateV1{}, errors.New("uncertain admission persistence") + } + case "partial frame": + original := w.backend.run + w.backend.run = func(s CommandSpec, o RunOptions) error { + if s.Args[0] != "start" { + return original(s, o) + } + b := make([]byte, 7) + _, err := o.Stdin.Read(b) + return errors.Join(err, errors.New("partial frame disconnect")) + } + case "cleanup": + w.cleanupFailure = true + case "retirement": + f.retirementError = errors.New("trusted retirement proof unavailable") + case "incomplete retirement": + f.incompleteRetirement = true + case "reacquire": + f.backend.acquire = func(context.Context, string) (*deploy.OperationLock, error) { + return nil, errors.New("completion lock unavailable") + } + case "changed journal", "changed queue": + f.backend.acquire = func(ctx context.Context, dir string) (*deploy.OperationLock, error) { + operation, err := deploy.AcquireOperationLock(ctx, dir) + if err != nil { + return nil, err + } + if mode == "changed journal" { + state, _, _ := operation.ReadStateV1() + state.IdentityMapping.PendingOperation.OperationID = "other" + writeMappedLiveStateFixtureV1(t, operation, state) + } else { + _, _, _ = operation.RemoveLiveRunV1(f.run.ID) + } + return operation, nil + } + } + if err := f.execute(); err == nil { + t.Fatal("uncertain outcome claimed success") + } + _, queue, state := f.inspect(t) + if state.IdentityMapping.PendingOperation == nil || mode != "changed queue" && len(queue.Runs) != 1 || f.recoveries > 1 || w.created > 1 || w.admissions > 1 { + t.Fatal("uncertainty lost ownership or retried admission") + } + if mode != "create" && mode != "changed queue" && queue.Runs[0].Container != w.id { + t.Fatal("uncertain exact ID not retained") + } + if mode == "create" || mode == "F" || mode == "admission" || mode == "partial frame" { + if w.application.Load() != 0 { + t.Fatal("failed release executed application") + } + } + }) + } +} + +func TestMappedLiveInstanceRejectsBeforeEffectsV1(t *testing.T) { + for _, mode := range []string{"nil context", "unbounded admission", "cancelled admission", "nil lifetime", "cancelled lifetime", "wrong directory", "no lease", "missing run", "occupied run", "wrong command", "stale generation", "stale revision", "wrong operation", "wrong engine", "wrong instance", "wrong maps", "changed build", "unknown entry", "occupied entry", "corrupt plan", "backend", "borrowed input"} { + t.Run(mode, func(t *testing.T) { + f := newMappedLiveFixtureV1(t, "exact", mappedApplicationCommandV1) + ctx, dir, plan := f.waiter.ctx, f.directory, f.plan + ops, options, backend := f.waiter.prepared.ops, f.waiter.options, f.backend + state := f.input.Current.State + switch mode { + case "borrowed input": + options.Stdin = strings.NewReader("borrowed input") + case "nil context": + ctx = nil + case "unbounded admission": + ctx = t.Context() + case "cancelled admission": + f.waiter.cancelAdmission() + case "nil lifetime": + options.Context = nil + case "cancelled lifetime": + ctx, cancel := context.WithCancel(t.Context()) + cancel() + options.Context = ctx + case "wrong directory": + dir = t.TempDir() + case "no lease": + _ = f.lease.Close() + case "missing run": + _, _, _ = ops.Operation.RemoveLiveRunV1(f.run.ID) + case "occupied run": + _ = ops.Operation.RecordLiveRunContainerV1(f.run.ID, "foreign") + case "wrong command": + facts, _ := plan.facts() + facts.Command = "foreign" + plan = reboundMappedLivePlanFixtureV1(t, plan, facts) + case "stale generation": + facts, _ := plan.facts() + facts.Generation = rendererDigest("f") + plan = reboundMappedLivePlanFixtureV1(t, plan, facts) + case "stale revision": + state.IdentityMapping.Revision, state.IdentityMapping.PendingOperation.ExpectedRevision = "2", "2" + case "wrong operation": + state.IdentityMapping.PendingOperation.OperationID = "other" + case "wrong engine": + state.IdentityMapping.PendingOperation.Native.Engine = "other" + case "wrong instance": + state.IdentityMapping.PendingOperation.Native.Process = "other" + case "wrong maps": + facts, _ := plan.facts() + facts.Active.Profile = "bounded" + plan = reboundMappedLivePlanFixtureV1(t, plan, facts) + case "changed build": + ops.ValidateProfileOwner = nil + case "unknown entry", "occupied entry": + facts, _ := plan.facts() + facts.Kind = "255" + if mode == "occupied entry" { + facts.Mode = strconv.Itoa(int(mappedEntryOccupiedExecV1)) + } + plan = reboundMappedLivePlanFixtureV1(t, plan, facts) + case "corrupt plan": + plan.payload += "x" + case "backend": + backend.start = nil + } + writeMappedLiveStateFixtureV1(t, ops.Operation, state) + if err := runMappedLiveInstanceWithBackendV1(ctx, dir, ops, f.run.ID, plan, options, backend); err == nil { + t.Fatal("invalid owner reached execution") + } + if f.starts != 0 || f.waiter.created != 0 || f.waiter.application.Load() != 0 { + t.Fatal("rejection had application or engine effects") + } + }) + } +} + +func TestMappedLiveInstanceEnforcesDisabledStdinV1(t *testing.T) { + for _, profile := range []string{"exact", "bounded"} { + for _, supplied := range []bool{false, true} { + t.Run(fmt.Sprint(profile, supplied), func(t *testing.T) { + f := newMappedLiveFixtureV1(t, profile, mappedApplicationCommandV1) + f.input.Stdin = false + var err error + f.plan, err = constructMappedEntryPlanV1(f.input) + if err != nil { + t.Fatal(err) + } + if !supplied { + f.waiter.options.Stdin = nil + f.waiter.expectedInput = "" + } + err = f.execute() + if supplied { + if err == nil || !strings.Contains(err.Error(), "stdin") { + t.Fatalf("disabled stdin accepted a reader: %v", err) + } + if f.starts != 0 || f.waiter.created != 0 || f.waiter.application.Load() != 0 { + t.Fatal("stdin refusal had engine or application effects") + } + } else if err != nil { + t.Fatalf("disabled stdin without a reader failed: %v", err) + } + }) + } + } +} + +type mappedLiveBlockedInputV1 struct { + io.ReadCloser + reading chan struct{} + once sync.Once +} + +type mappedLiveCloseFailureV1 struct{ io.ReadCloser } + +func (input mappedLiveCloseFailureV1) Close() error { + return errors.Join(input.ReadCloser.Close(), errors.New("input closure failed")) +} + +func (input *mappedLiveBlockedInputV1) Read(buffer []byte) (int, error) { + input.once.Do(func() { close(input.reading) }) + return input.ReadCloser.Read(buffer) +} + +type mappedLiveExitOutputV1 struct { + exited chan struct{} + once sync.Once +} + +func (output *mappedLiveExitOutputV1) Write(buffer []byte) (int, error) { + if len(buffer) > 0 { + output.once.Do(func() { close(output.exited) }) + } + return len(buffer), nil +} + +func TestMappedLiveExitBeforeInputEOFHelperV1(t *testing.T) { + if os.Getenv("REPLOY_MAPPED_EXIT_INPUT_FIXTURE") == "1" { + fmt.Println("child exiting without reading application stdin") + os.Exit(0) + } +} + +func TestMappedLiveInstanceCompletesBeforeInputEOFV1(t *testing.T) { + testMappedLiveOpenInputV1(t, false) +} + +func TestMappedLiveInstanceCancelsBlockedInputV1(t *testing.T) { + testMappedLiveOpenInputV1(t, true) +} + +func testMappedLiveOpenInputV1(t *testing.T, cancelled bool) { + t.Helper() + for _, profile := range []string{"exact", "bounded"} { + t.Run(profile, func(t *testing.T) { + f := newMappedLiveFixtureV1(t, profile, mappedApplicationCommandV1) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + reader, writer := io.Pipe() + defer writer.Close() + input := &mappedLiveBlockedInputV1{ReadCloser: reader, reading: make(chan struct{})} + defer input.Close() + output := &mappedLiveExitOutputV1{exited: make(chan struct{})} + f.waiter.options.Context, f.waiter.options.Stdin = ctx, input + originalRun := f.waiter.backend.run + f.waiter.backend.run = func(spec CommandSpec, options RunOptions) error { + switch spec.Args[0] { + case "start": + frame, err := mappedlaunch.Read(options.Stdin, f.waiter.binding) + if err != nil { + return err + } + f.waiter.frame <- frame + options.Stdout = output + return runCommandRaw(CommandSpec{Name: os.Args[0], Args: []string{"-test.run=^TestMappedLiveExitBeforeInputEOFHelperV1$"}, Env: []string{"REPLOY_MAPPED_EXIT_INPUT_FIXTURE=1"}}, options) + case "wait": + // Only engine observation is injected. The stdin copier and + // exited child are real, and input EOF is never supplied. + if cancelled { + <-options.Context.Done() + return options.Context.Err() + } + select { + case <-output.exited: + _, err := io.WriteString(options.Stdout, "0\n") + return err + case <-options.Context.Done(): + return options.Context.Err() + } + default: + return originalRun(spec, options) + } + } + result := make(chan error, 1) + go func() { result <- f.execute() }() + for _, ready := range []chan struct{}{input.reading, output.exited} { + select { + case <-ready: + case <-time.After(5 * time.Second): + cancel() + _ = input.Close() + t.Fatal("real child/stdin copier did not start") + } + } + if cancelled { + cancel() + } + select { + case err := <-result: + if cancelled && !errors.Is(err, context.Canceled) || !cancelled && err != nil { + t.Fatalf("completed instance failed: %v", err) + } + case <-time.After(2 * time.Second): + cancel() + _ = input.Close() + <-result + t.Fatal("completed instance waited for application stdin EOF") + } + _, queue, state := f.inspect(t) + if len(queue.Runs) != 0 || state.IdentityMapping.PendingOperation != nil || f.waiter.removed != 1 { + t.Fatal("completed instance did not retire exactly once") + } + }) + } +} + +func reboundMappedLivePlanFixtureV1(t *testing.T, plan mappedEntryPlanV1, facts mappedEntryPayloadV1) mappedEntryPlanV1 { + t.Helper() + digest, err := canonical.Sum("mapped-entry", "v1", facts) + if err != nil { + t.Fatal(err) + } + encoded, err := canonical.Marshal(facts) + if err != nil { + t.Fatal(err) + } + plan.payload, plan.binding.Plan = string(encoded), digest + plan.binding.Generation = facts.Generation + return plan +} + +func TestMappedLiveCreateUsesOnlyFrozenLaunchFactsV1(t *testing.T) { + f := newMappedLiveFixtureV1(t, "exact", mappedApplicationCommandV1) + facts, _ := f.plan.facts() + spec, policy, err := mappedLiveCreateSpecV1(f.waiter.prepared.ops, f.input.Current.State, f.directory, facts) + if err != nil { + t.Fatal(err) + } + request, err := mappedPodmanApplicationCreateRequestV1(spec, "fixture", policy) + if err != nil { + t.Fatal(err) + } + body := request.body + if request.name != facts.Run || !slices.Equal(body.Cmd, facts.Startup) || !slices.Equal(body.Env, facts.Environment) || body.Tty || body.HostConfig.AutoRemove || body.HostConfig.RestartPolicy.Name != "no" || policy.facts() != facts.Kernel || !body.HostConfig.ReadonlyRootfs { + t.Fatalf("frozen launch facts changed: %s", mustMappedLiveJSONFixtureV1(t, body)) + } + if !strings.Contains(strings.Join(body.Env, "\n"), "TOKEN=dummy") || !reflect.DeepEqual(facts.Mounts, f.input.Runtime.Mounts) { + t.Fatal("expected input bytes changed") + } + if _, err := runMappedNilFixtureV1(); err == nil { + t.Fatal("production entry accepted missing ownership") + } +} + +func mustMappedLiveJSONFixtureV1(t *testing.T, value any) string { + t.Helper() + data, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + return string(data) +} + +func runMappedNilFixtureV1() (bool, error) { + return false, runMappedLiveInstanceV1(nil, "", deploy.IdentityMappingOperationsV1{}, "", mappedEntryPlanV1{}, RunOptions{}) +} + +func TestMappedLiveCompletionRequiresEngineExitStatusV1(t *testing.T) { + for _, value := range []string{"7\n", "", "00\n", "-1\n", "256\n", "0\n0\n", strings.Repeat("0", 40)} { + t.Run(fmt.Sprintf("%q", value), func(t *testing.T) { + f := newMappedLiveFixtureV1(t, "exact", mappedApplicationCommandV1) + f.exitStatus = value + if err := f.execute(); err == nil { + t.Fatal("bad or nonzero engine status became successful completion") + } + _, queue, state := f.inspect(t) + if len(queue.Runs) != 0 || state.IdentityMapping.PendingOperation != nil || f.waiter.removed != 1 { + t.Fatal("failed command status prevented exact retirement") + } + }) + } + f := newMappedLiveFixtureV1(t, "exact", mappedApplicationCommandV1) + cause := errors.New("attach transport lost") + f.attachErr = cause + if err := f.execute(); !errors.Is(err, cause) { + t.Fatalf("attach failure lost: %v", err) + } + _, queue, state := f.inspect(t) + if len(queue.Runs) != 0 || state.IdentityMapping.PendingOperation != nil { + t.Fatal("attach failure did not retire owned instance") + } +} diff --git a/internal/dockerdeploy/mapped_entry_plan.go b/internal/dockerdeploy/mapped_entry_plan.go index 46d7a781..aa8b89bd 100644 --- a/internal/dockerdeploy/mapped_entry_plan.go +++ b/internal/dockerdeploy/mapped_entry_plan.go @@ -204,7 +204,9 @@ func constructMappedEntryPlanV1(input mappedEntryInputV1) (mappedEntryPlanV1, er final |= uint64(1) << bit } sandbox := newApplicationSandboxPlanWithNetworkV1(RuntimeUserPlan{UID: uint32(uid), GID: uint32(gid), SupplementaryGIDs: groups}, c.Lock.RuntimePolicy.Network) - setup := final + // The supported Podman renderer prepares the fixed private home before + // dropping setup authority. Bind that existing step before hashing the plan. + setup := final | uint64(1) // CHOWN, used only by temporary-home setup. for _, name := range applicationSetupCapabilitiesV1(sandbox) { setup |= uint64(1) << slices.Index(mappedCapabilityNamesV1, name) } @@ -217,7 +219,7 @@ func constructMappedEntryPlanV1(input mappedEntryInputV1) (mappedEntryPlanV1, er ports[i], _ = strconv.Atoi(port) } startup := sandboxApplicationArgvV1(DockerExecutionPlan{Sandbox: sandbox}, command.Argv, true, ports) - startup = slices.Insert(startup, 1, fmt.Sprintf("--mapped-capabilities=%016x", final)) + startup = slices.Insert(startup, 1, fmt.Sprintf("--mapped-capabilities=%016x", final), podmanPrepareTemporaryHome) if _, err := mappedStartupCommandV1(startup, kernel); err != nil { return fail(err) } diff --git a/internal/dockerdeploy/mapped_entry_plan_test.go b/internal/dockerdeploy/mapped_entry_plan_test.go index ae9f167c..74464d0b 100644 --- a/internal/dockerdeploy/mapped_entry_plan_test.go +++ b/internal/dockerdeploy/mapped_entry_plan_test.go @@ -402,9 +402,12 @@ func TestMappedEntryPlanV1CapabilityTupleComesFromLockedRole(t *testing.T) { t.Fatalf("wrong root tuple: %+v", f) } kernel := mustCompileEntryKernelV1(t, f) - if kernel.setupCapabilities != kernel.finalCapabilities|1<<6|1<<8|1<<12 { + if kernel.setupCapabilities != kernel.finalCapabilities|1<<0|1<<6|1<<8|1<<12 { t.Fatalf("wrong exact setup union: %x", kernel.setupCapabilities) } + if !slices.Contains(f.Startup, podmanPrepareTemporaryHome) { + t.Fatal("Podman temporary-home setup is not bound into the plan") + } if _, err := mappedStartupCommandV1(f.Startup, kernel); err != nil { t.Fatal(err) }