From 87899abecced4c5f8b3513ecee66793ecdd2df37 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Sun, 1 Nov 2026 01:13:20 +0800 Subject: [PATCH] E4.2: Qualify private mapped execution and retain conformance evidence Add repeated integrated private journeys for both mapping profiles, with authority refusal, interrupted release, cancellation, owned cleanup and retained identity history. Reuse the existing production planner, lifetime and startup mechanism fixtures. Add a local evidence collector and integrity tests that bind source, compiler, profiles and exact test results. Keep injected engine/native/F observations separate from production exec mechanism evidence and explicitly leave live-engine/OCI, F/G and public activation unqualified. No production runtime behavior changes. --- .../dockerdeploy/mapped_conformance_test.go | 257 ++++++++++++++++++ internal/probe/mapped_exec_linux_test.go | 19 ++ .../multiidentity/E_CONFORMANCE.md | 56 ++++ .../multiidentity/e_conformance.py | 196 +++++++++++++ .../multiidentity/test_e_conformance.py | 194 +++++++++++++ 5 files changed, 722 insertions(+) create mode 100644 internal/dockerdeploy/mapped_conformance_test.go create mode 100644 tools/conformance/multiidentity/E_CONFORMANCE.md create mode 100644 tools/conformance/multiidentity/e_conformance.py create mode 100644 tools/conformance/multiidentity/test_e_conformance.py diff --git a/internal/dockerdeploy/mapped_conformance_test.go b/internal/dockerdeploy/mapped_conformance_test.go new file mode 100644 index 00000000..64dd7aae --- /dev/null +++ b/internal/dockerdeploy/mapped_conformance_test.go @@ -0,0 +1,257 @@ +package dockerdeploy + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "reflect" + "runtime" + "testing" + + "github.com/omry/reploy/internal/deploy" +) + +// These journeys use production planners, locks, leases, stream framing and +// lifetime code. Engine, process pinning, native admission and F observations +// remain injected. The separate probe suite exercises the real exec mechanism. +func TestMappedConformanceFreshJourneyV1(t *testing.T) { + for _, profile := range []string{"exact", "bounded"} { + for _, entry := range []struct { + name string + kind mappedEntryKindV1 + }{ + {"workload", mappedWorkloadStartV1}, + {"application", mappedApplicationCommandV1}, + {"runtime-test", mappedRuntimeTestV1}, + {"hook", mappedLifecycleHookV1}, + } { + for repetition := range 3 { + t.Run(fmt.Sprintf("%s/%s/%d", profile, entry.name, repetition), func(t *testing.T) { + f := newMappedLiveFixtureV1(t, profile, entry.kind) + var stdout, stderr bytes.Buffer + f.waiter.options.Stdout, f.waiter.options.Stderr = &stdout, &stderr + original := f.waiter.backend.run + f.waiter.backend.run = func(s CommandSpec, o RunOptions) error { + err := original(s, o) + if err == nil && s.Args[0] == "start" { + if _, err = io.WriteString(o.Stdout, "application output\n"); err == nil { + _, err = io.WriteString(o.Stderr, "application diagnostic\n") + } + } + return err + } + f.beforeRecovery = func(op *deploy.OperationLock) { + if err := op.RequireQueueEntryLeaseHeldV1(f.run.ID); err != nil { + t.Error("owned lease lost before recovery", err) + } + } + err := runMappedControlledEntryWithBackendV1(f.waiter.ctx, f.directory, f.waiter.prepared.ops, f.run.ID, f.input, f.waiter.options, f.backend) + if entry.kind == mappedWorkloadStartV1 { + if !errors.Is(err, errMappedMasterExitedV1) { + t.Fatalf("required master exit lost: %v", err) + } + } else if err != nil { + t.Fatal(err) + } + _, queue, state := f.inspect(t) + if len(queue.Runs) != 0 || state.IdentityMapping.PendingOperation != nil || f.starts != 1 || f.recoveries != 1 || f.waiter.created != 1 || f.waiter.admissions != 1 || f.waiter.removed != 1 || f.waiter.process.closed != 1 { + t.Fatal("journey did not release and retire exactly one owned instance") + } + if f.waiter.application.Load() == 0 || stdout.String() != "application output\n" || stderr.String() != "application diagnostic\n" { + t.Fatal("application streams lost, mixed, or contain release metadata") + } + if state.IdentityMapping.AcceptedUIDs == nil || len(*state.IdentityMapping.AcceptedUIDs) == 0 || !reflect.DeepEqual(state.IdentityMapping.UIDAssignments, f.input.Current.State.IdentityMapping.PendingOperation.Native.Active.UIDs) { + t.Fatal("retirement discarded admitted identity history") + } + select { + case frame := <-f.waiter.frame: + if frame.Binding != f.plan.binding || frame.Container != f.waiter.id { + t.Fatal("release substituted generation, role, image or plan") + } + default: + t.Fatal("missing release frame") + } + select { + case <-f.waiter.frame: + t.Fatal("duplicate release frame") + default: + } + facts, err := f.plan.facts() + if err != nil { + t.Fatal(err) + } + evidence, err := json.Marshal(struct { + Profile, Entry, Compiler, OS, Architecture string + Facts mappedEntryPayloadV1 + Qualification string + }{profile, entry.name, runtime.Version(), runtime.GOOS, runtime.GOARCH, facts, "private-orchestration; injected engine/native/F"}) + if err != nil { + t.Fatal(err) + } + t.Logf("E conformance receipt: %s", evidence) + }) + } + } + } +} + +func TestMappedConformanceRefusalJourneyV1(t *testing.T) { + for _, profile := range []string{"exact", "bounded"} { + for _, mode := range []string{"role", "image", "revision", "runtime-account", "maps", "owner", "occupied", "host-native"} { + t.Run(profile+"/"+mode, func(t *testing.T) { + f := newMappedLiveFixtureV1(t, profile, mappedApplicationCommandV1) + op := f.waiter.prepared.ops.Operation + before, _, err := op.ReadStateV1() + if err != nil { + t.Fatal(err) + } + switch mode { + case "role": + f.input.Command = "undeclared-role" + case "image": + f.input.Runtime.Image += "-stale" + case "revision": + f.input.Current.State.IdentityMapping.Revision = "99" + case "runtime-account": + f.input.Current.State.IdentityMapping.PendingOperation.Native.RuntimeUsers = []string{"999"} + case "maps": + f.input.Current.State.IdentityMapping.PendingOperation.Native.Active.UIDs[0].HostStart = "999999" + case "owner": + f.input.Current.State.IdentityMapping.PendingOperation.Native.Engine = "foreign" + case "occupied": + f.input.Kind = mappedControlledSessionV1 + case "host-native": + f.input.Kind = mappedHostNativeCommandV1 + } + if err := runMappedControlledEntryWithBackendV1(f.waiter.ctx, f.directory, f.waiter.prepared.ops, f.run.ID, f.input, f.waiter.options, f.backend); err == nil { + t.Fatal("invalid authority accepted") + } + if f.starts != 0 || f.waiter.created != 0 || f.waiter.admissions != 0 || f.waiter.application.Load() != 0 || f.recoveries != 0 { + t.Fatal("refusal reached an engine or application effect") + } + // Pure construction keeps the caller lock. Once construction + // succeeds, the lifetime runner owns and releases that lock even + // when its durable owner check refuses before create. + if op.RequireHeld() != nil { + op, err = deploy.AcquireOperationLock(t.Context(), f.directory) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = op.Unlock() }) + } + after, _, err := op.ReadStateV1() + if err != nil || !reflect.DeepEqual(before, after) { + t.Fatal("refusal changed durable state", err) + } + }) + } + } +} + +func TestMappedConformanceInterruptedJourneyV1(t *testing.T) { + for _, profile := range []string{"exact", "bounded"} { + for _, mode := range []string{"engine", "create", "pin", "pid", "F", "sandbox", "admission", "partial-frame", "cleanup", "retirement"} { + t.Run(profile+"/"+mode, func(t *testing.T) { + f := newMappedLiveFixtureV1(t, profile, mappedApplicationCommandV1) + w := f.waiter + switch mode { + case "engine": + w.observeFailure = true + case "create": + w.createFailure = true + case "pin": + w.pinFailure = true + case "pid": + w.process.fail = true + case "F": + w.effect.fail = true + case "sandbox": + w.effect.sandboxFailure = errors.New("missing trusted kernel/code observation") + case "admission": + w.backend.admit = func(deploy.IdentityMappingOperationsV1, context.Context, deploy.StateV1, string) (deploy.StateV1, error) { + return deploy.StateV1{}, errors.New("uncertain native admission") + } + case "partial-frame": + original := w.backend.run + w.backend.run = func(s CommandSpec, o RunOptions) error { + if s.Args[0] != "start" { + return original(s, o) + } + _, err := io.ReadFull(o.Stdin, make([]byte, 7)) + return errors.Join(err, errors.New("release interrupted")) + } + case "cleanup": + w.cleanupFailure = true + case "retirement": + f.retirementError = errors.New("retirement proof unavailable") + } + if err := runMappedControlledEntryWithBackendV1(w.ctx, f.directory, w.prepared.ops, f.run.ID, f.input, w.options, f.backend); err == nil { + t.Fatal("interrupted journey claimed success") + } + _, queue, state := f.inspect(t) + if state.IdentityMapping.PendingOperation == nil || len(queue.Runs) != 1 || w.created > 1 || w.admissions > 1 || f.recoveries > 1 { + t.Fatal("uncertainty discarded ownership or retried effects") + } + if w.created == 1 && mode != "create" && queue.Runs[0].Container != w.id && w.removed != 1 { + t.Fatal("uncertainty neither retained nor cleaned the exact instance") + } + if mode != "cleanup" && mode != "retirement" { + if w.application.Load() != 0 { + t.Fatal("refused or incomplete release reached application input") + } + select { + case <-w.frame: + t.Fatal("refused journey delivered a complete release frame") + default: + } + } + }) + } + } +} + +func TestMappedConformanceCancellationJourneyV1(t *testing.T) { + for _, profile := range []string{"exact", "bounded"} { + t.Run(profile, func(t *testing.T) { + f := newMappedLiveFixtureV1(t, profile, mappedApplicationCommandV1) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + f.waiter.options.Context, f.untilCancel = ctx, true + result := make(chan error, 1) + go func() { + result <- runMappedControlledEntryWithBackendV1(f.waiter.ctx, f.directory, f.waiter.prepared.ops, f.run.ID, f.input, f.waiter.options, f.backend) + }() + select { + case <-f.waiter.frame: + case <-f.waiter.ctx.Done(): + t.Fatal("release did not complete within admission deadline") + } + op, err := deploy.AcquireOperationLock(f.waiter.ctx, f.directory) + if err != nil { + t.Fatal(err) + } + queue, _, err := op.ReadLiveRunQueueV1() + owned := err == nil && len(queue.Runs) == 1 && queue.Runs[0].Container == f.waiter.id && op.RequireQueueEntryLeaseHeldV1(f.run.ID) == nil + _ = op.Unlock() + if !owned { + t.Fatal("attached run did not retain exact ownership and lease", err) + } + cancel() + select { + case err := <-result: + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation lost: %v", err) + } + case <-f.waiter.ctx.Done(): + t.Fatal("cancellation did not finish within fixture deadline") + } + _, queue, state := f.inspect(t) + if len(queue.Runs) != 0 || state.IdentityMapping.PendingOperation != nil || f.waiter.removed != 1 || f.recoveries != 1 || state.IdentityMapping.AcceptedUIDs == nil || len(*state.IdentityMapping.AcceptedUIDs) == 0 { + t.Fatal("cancelled journey lost cleanup or admitted mapping history") + } + }) + } +} diff --git a/internal/probe/mapped_exec_linux_test.go b/internal/probe/mapped_exec_linux_test.go index 3bcdf833..212c987a 100644 --- a/internal/probe/mapped_exec_linux_test.go +++ b/internal/probe/mapped_exec_linux_test.go @@ -599,6 +599,9 @@ func TestMappedDescriptorRaceHelperV1(t *testing.T) { panic(err) } close(ready) + if args[0] == "descriptor-shared-close" { + mappedSharedCloseObserverV1(&damaged) + } for { if _, _, errno := syscall.RawSyscall(unix.SYS_FCNTL, 4096, unix.F_GETFD, 0); errno == unix.EBADF { atomic.StoreUint32(&damaged, 1) @@ -645,6 +648,22 @@ func TestMappedDescriptorRaceHelperV1(t *testing.T) { os.Exit(Main(args[2:], os.Stdin, os.Stdout, os.Stderr)) } +// The deliberately wrong shared closer damages runtime descriptors too. Its +// observer must not depend on Go scheduling or descriptor operations afterward. +// Other modes retain the real concurrent opener above. +// +//go:nosplit +//go:norace +func mappedSharedCloseObserverV1(damaged *uint32) { + for { + if _, _, errno := syscall.RawSyscall(unix.SYS_FCNTL, 4096, unix.F_GETFD, 0); errno == unix.EBADF { + atomic.StoreUint32(damaged, 1) + for { + } + } + } +} + // Deliberately wrong test-only closer. A peer detects the shared table's lost // high fd. Stay in a raw tail so a runtime poller failure cannot masquerade as // that specific independent observation. The parent bounds/reaps the process. diff --git a/tools/conformance/multiidentity/E_CONFORMANCE.md b/tools/conformance/multiidentity/E_CONFORMANCE.md new file mode 100644 index 00000000..75deccb4 --- /dev/null +++ b/tools/conformance/multiidentity/E_CONFORMANCE.md @@ -0,0 +1,56 @@ +# Private milestone E conformance + +Run from the Reploy checkout with a native Go toolchain. Use a new output +directory under an existing private evidence parent: + +```sh +python3 tools/conformance/multiidentity/e_conformance.py --output-dir /absolute/new/evidence +python3 -m unittest discover -s tools/conformance/multiidentity -p test_e_conformance.py +``` + +The runner creates no engine resources. Its existing production-startup tests +use disposable Linux user/network namespaces and bounded child processes. +They need `cc`, `unshare`, `newuidmap`, `newgidmap` and delegated subordinate +IDs. Missing prerequisites remain visible test skips, not qualification. +Mapped runtime support remains limited by the approved Linux/Podman cohort; +running the portable fixture suites elsewhere does not expand that cohort. + +`report.json` binds source bytes and file modes, Go configuration and compiler +and linker hashes, host kernel/architecture, fixture helpers and the pinned +OCI seccomp seed. It checks source/toolchain stability after collection. +Each suite has its exact command, elapsed time, JSON event log digest and +complete terminal test statuses. Only stdout is parsed as JSON. Each suite's +stderr is retained separately in `.stderr` with its own digest, including +partial timeout diagnostics. The two metadata commands retain `go-env-before.stderr` +and `go-env-after.stderr`; their digests are recorded separately from toolchain facts. +Private fresh journeys also emit their exact +closed plan, including profile, maps, immutable image, role, generation, +runtime/kernel/network policies and stream mode. These are fixture facts. +The actual production exec tests identify their compiled source/toolchain +and verify the production exec tail; this is mechanism evidence, not a +retained launched-image binary or live-engine receipt. + +The integrated private journey repeats workload, application, runtime-test and +hook cases three times for each of `exact` and `bounded`. It checks one release, +separate application streams, owned completion/cleanup and retained mapping +history. Invalid authority and interrupted proof/release paths check absence of +application effects. Existing suites cover native atomic admission, canonical +policy, capabilities, private/high/concurrent descriptors, occupied refusal, +replacement refusal, cancellation and cleanup ownership. + +Collection exit zero means the required private suites passed, their fresh +matrix is complete, no selected test failed and the source/toolchain stayed +stable. A partially skipped production-exec suite remains `partial`, with +each gap listed; it cannot become a full kernel qualification. A missing +required test, failed test, timeout, malformed log or changed source fails +collection. Existing evidence directories are never overwritten. + +No result from this runner qualifies a live engine or public mapped execution. +`mapped_runtime_qualified` is always false. Production F code/mount/network +observations, an exact engine/OCI-policy receipt, unresolved kernel tuples and +G activation remain separate gates. Working replacement and occupied exec +remain refused. The older `conformance.py` Podman probe is a distinct lower +level experiment and cannot stand in for this integrated E evidence. +The opt-in `TestMappedPodmanCreateLiveV1` is explicitly excluded from this +private runner, even when its enabling environment is set. The exact exclusion +is retained in the report and command; live-engine qualification stays open. diff --git a/tools/conformance/multiidentity/e_conformance.py b/tools/conformance/multiidentity/e_conformance.py new file mode 100644 index 00000000..bb3ae5ae --- /dev/null +++ b/tools/conformance/multiidentity/e_conformance.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Collect private E test evidence. Never qualify a live mapped engine.""" + +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import os +from pathlib import Path +import platform +import shutil +import subprocess +import sys +import time + + +ROOT = Path(__file__).resolve().parents[3] +PACKAGE = "github.com/omry/reploy/internal/" +SUITES = ( + ("identity", "identitymapping", ".", ("TestHostPreflightRequiresCompleteSources",)), + ("native-state", "deploy", "^TestNativeMappingV1", ( + "TestNativeMappingV1AdmissionIsOneDurableTransaction", + "TestNativeMappingV1AdmissionWriteFailureNeverGrantsRelease", + )), + ("private-orchestration", "dockerdeploy", "Mapped", ( + "TestMappedConformanceFreshJourneyV1", + "TestMappedConformanceRefusalJourneyV1", + "TestMappedConformanceInterruptedJourneyV1", + "TestMappedConformanceCancellationJourneyV1", + "TestMappedReplacementEntryV1QualifiedOwnerRefusesBeforeEffects", + "TestMappedEntryAdaptersV1CheckBothSessionOwners", + "TestMappedSeccompCompilerRemovesEveryEscapeAllowance", + )), + ("production-exec-mechanism", "probe", "^TestMapped(Production|StartupPreflight|StandardDescriptors|KernelCapabilityLimit)", ( + "TestMappedProductionExecTailV1", "TestMappedProductionStartupV1", + )), +) + + +def digest(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def command(argv: list[str], stderr_path: Path, *, timeout: int = 60) -> bytes: + try: + process = subprocess.run(argv, cwd=ROOT, timeout=timeout, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + except subprocess.TimeoutExpired as error: + stderr_path.write_bytes(error.stderr or b"") + raise + stderr_path.write_bytes(process.stderr) + process.check_returncode() + return process.stdout + + +def source_inventory(root: Path = ROOT) -> dict[str, dict]: + # Include embedded policy/data, fixture code and build inputs, not just the + # new test. Ignore Python caches and evidence directories by construction. + paths = {root / name for name in ("go.mod", "go.sum", "AGENTS.md", ".github/workflows/ci.yml")} + for directory in ("internal", "cmd"): + paths.update(p for p in (root / directory).rglob("*") if p.is_file()) + here = root / "tools/conformance/multiidentity" + paths.update(here / name for name in ("e_conformance.py", "test_e_conformance.py", "E_CONFORMANCE.md")) + return { + p.relative_to(root).as_posix(): {"sha256": digest(p.read_bytes()), "mode": p.stat().st_mode & 0o777} + for p in sorted(paths) + } + + +def toolchain(output: Path, phase: str) -> dict: + env = json.loads(command(["go", "env", "-json", "GOOS", "GOARCH", "GOVERSION", "GOTOOLDIR", "GOEXPERIMENT", "GOFLAGS", "CGO_ENABLED"], + output / f"go-env-{phase}.stderr")) + tools = {} + suffix = ".exe" if os.name == "nt" else "" + for name in ("compile", "link"): + path = Path(env["GOTOOLDIR"]) / (name + suffix) + tools[name] = {"path": str(path), "sha256": digest(path.read_bytes())} + for name in ("go", "cc", "unshare", "newuidmap", "newgidmap"): + located = shutil.which(name) + tools[name] = None if located is None else {"path": located, "sha256": digest(Path(located).read_bytes())} + return {"go_env": env, "tools": tools, + "runtime_flags": {name: os.environ.get(name) for name in ("GODEBUG", "GOMAXPROCS", "GOAMD64", "GOARM64")}, + "host": {"system": platform.system(), "release": platform.release(), "machine": platform.machine()}} + + +def summarize(raw: bytes, package: str, required: tuple[str, ...], returncode: int | None) -> dict: + events = [json.loads(line) for line in raw.splitlines() if line.strip()] + if any(not isinstance(e, dict) or e.get("Package") != PACKAGE + package for e in events): + raise ValueError("unexpected Go event package or shape") + terminal = {} + package_actions = [] + for event in events: + action = event.get("Action") + if action in ("pass", "fail", "skip"): + if "Test" in event: + if event["Test"] in terminal: + raise ValueError("duplicate terminal test event") + terminal[event["Test"]] = action + else: + package_actions.append(action) + missing = [name for name in required if name not in terminal] + failed = sorted(name for name, action in terminal.items() if action == "fail") + skipped = sorted(name for name, action in terminal.items() if action == "skip") + required_gaps = [name for name in required if terminal.get(name) != "pass"] + ok = returncode == 0 and package_actions == ["pass"] and not missing and not failed + return {"status": "failed" if not ok else "partial" if skipped or required_gaps else "passed", + "missing": missing, "failed": failed, "skipped": skipped, + "required_gaps": required_gaps, "terminal_tests": terminal, + "returncode": returncode, "log_sha256": digest(raw)} + + +def fresh_matrix(result: dict) -> list[str]: + terminals = result["terminal_tests"] + return [f"{profile}/{entry}/{repeat}" + for profile in ("exact", "bounded") + for entry in ("workload", "application", "runtime-test", "hook") + for repeat in range(3) + if terminals.get(f"TestMappedConformanceFreshJourneyV1/{profile}/{entry}/{repeat}") != "pass"] + + +def report_results(results: dict, stable: bool) -> dict: + private_names = ("identity", "native-state", "private-orchestration") + private = all(results[name]["status"] == "passed" for name in private_names) + matrix_gaps = fresh_matrix(results["private-orchestration"]) + kernel = results["production-exec-mechanism"] + # A zero process exit is collection success, never a mapped-support claim. + collection_ok = stable and not matrix_gaps and private and kernel["status"] != "failed" + return {"collection_ok": collection_ok, "source_and_toolchain_stable": stable, + "private_conformance_passed": stable and private and not matrix_gaps, + "fresh_instance_matrix_gaps": matrix_gaps, + "production_exec_mechanism": kernel["status"], + "live_engine": {"status": "not-qualified", "engine": None, "oci_runtime": None, + "reason": "No live engine is exercised by this runner; fixture engine/native/F observations are injected."}, + "mapped_runtime_qualified": False, "public_activation": "inhibited", + "remaining_gates": ["Production F code/mount/network observation and exact live engine/OCI policy qualification.", + "Working replacement and occupied entry remain refused; G public activation is separate.", + "Every skipped/missing kernel case remains an explicit qualification gap."]} + + +def run(output: Path) -> dict: + output.mkdir(mode=0o700, parents=False, exist_ok=False) + inventory = source_inventory() + context = toolchain(output, "before") + results = {} + for name, package, selection, required in SUITES: + argv = ["go", "test", "-json", "-count=1", "-timeout=10m", "./internal/" + package, "-run", selection] + if name == "private-orchestration": + # The opt-in engine test belongs to separate live qualification. + # Never run it here, even if its enabling environment is inherited. + argv += ["-skip", "^TestMappedPodmanCreateLiveV1$"] + started = time.monotonic() + try: + process = subprocess.run(argv, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=660, check=False) + raw, stderr, code = process.stdout, process.stderr, process.returncode + except subprocess.TimeoutExpired as error: + raw, stderr, code = error.stdout or b"", error.stderr or b"", None + (output / (name + ".jsonl")).write_bytes(raw) + (output / (name + ".stderr")).write_bytes(stderr) + try: + result = summarize(raw, package, required, code) + except (ValueError, TypeError, KeyError) as error: + result = {"status": "failed", "error": str(error), "terminal_tests": {}, "log_sha256": digest(raw), "returncode": code} + result.update(command=argv, elapsed_seconds=time.monotonic() - started, + stderr_sha256=digest(stderr)) + results[name] = result + print(json.dumps({"suite": name, "status": result["status"], "elapsed_seconds": result["elapsed_seconds"]}), flush=True) + final_context = toolchain(output, "after") + stable = inventory == source_inventory() and context == final_context + report = {"schema_version": 1, "collected_at": dt.datetime.now(dt.timezone.utc).isoformat(), + "repository": str(ROOT), "source": inventory, "context": context, + "metadata_stderr_sha256": {phase: digest((output / f"go-env-{phase}.stderr").read_bytes()) + for phase in ("before", "after")}, + "policy_source": "internal/dockerdeploy/seccomp/docker-v0.2.4.json", + "selection_exclusions": {"TestMappedPodmanCreateLiveV1": "Separate opt-in live-engine qualification; not exercised or qualified here."}, + "suites": results, "summary": report_results(results, stable)} + (output / "report.json").write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return report + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", required=True, type=Path, help="new evidence directory; its parent must exist") + args = parser.parse_args() + try: + report = run(args.output_dir) + except (OSError, subprocess.SubprocessError, ValueError) as error: + print(f"E conformance collection failed: {error}", file=sys.stderr) + return 1 + print(json.dumps(report["summary"], indent=2)) + return 0 if report["summary"]["collection_ok"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/conformance/multiidentity/test_e_conformance.py b/tools/conformance/multiidentity/test_e_conformance.py new file mode 100644 index 00000000..41b2d797 --- /dev/null +++ b/tools/conformance/multiidentity/test_e_conformance.py @@ -0,0 +1,194 @@ +from __future__ import annotations + +import copy +import importlib.util +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + + +HERE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location("e_conformance", HERE / "e_conformance.py") +e = importlib.util.module_from_spec(spec) +spec.loader.exec_module(e) + + +def events(package: str, actions: list[tuple[str | None, str]]) -> bytes: + return b"\n".join(json.dumps({"Package": e.PACKAGE + package, "Action": action, + **({"Test": test} if test is not None else {})}).encode() + for test, action in actions) + + +def complete_results() -> dict: + results = {name: {"status": "passed", "terminal_tests": {}} for name, _, _, _ in e.SUITES} + terminals = results["private-orchestration"]["terminal_tests"] + for profile in ("exact", "bounded"): + for entry in ("workload", "application", "runtime-test", "hook"): + for repeat in range(3): + terminals[f"TestMappedConformanceFreshJourneyV1/{profile}/{entry}/{repeat}"] = "pass" + return results + + +class EvidenceTests(unittest.TestCase): + def test_metadata_json_remains_separate_from_stderr_on_success_and_failure(self): + with tempfile.TemporaryDirectory() as temporary: + stderr = Path(temporary) / "metadata.stderr" + for code in (0, 3): + argv = [sys.executable, "-c", + 'import sys; print("{\\\"GOOS\\\":\\\"linux\\\"}"); ' + 'sys.stderr.write("go: downloading example.invalid/module v1\\n"); ' + f'sys.exit({code})'] + if code: + with self.assertRaises(subprocess.CalledProcessError): + e.command(argv, stderr) + else: + self.assertEqual(json.loads(e.command(argv, stderr)), {"GOOS": "linux"}) + self.assertEqual(stderr.read_bytes(), b"go: downloading example.invalid/module v1\n") + + def test_metadata_timeout_retains_stderr_and_stays_a_failure(self): + with tempfile.TemporaryDirectory() as temporary: + stderr = Path(temporary) / "metadata.stderr" + error = subprocess.TimeoutExpired(["go", "env"], 60, output=b"{}", stderr=b"partial diagnostic") + with mock.patch.object(e.subprocess, "run", side_effect=error): + with self.assertRaises(subprocess.TimeoutExpired): + e.command(["go", "env"], stderr) + self.assertEqual(stderr.read_bytes(), b"partial diagnostic") + + def test_suite_chatter_and_timeout_keep_separate_digest_bound_artifacts(self): + for timeout in (False, True): + with self.subTest(timeout=timeout), tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) / "evidence" + responses = [] + for index, (_, package, _, required) in enumerate(e.SUITES): + cases = list(required) + if package == "dockerdeploy": + cases += list(complete_results()["private-orchestration"]["terminal_tests"]) + raw = events(package, [(case, "pass") for case in cases] + [(None, "pass")]) + stderr = b"go: downloading example.invalid/module v1\n" + if timeout and index == 0: + responses.append(subprocess.TimeoutExpired(["go", "test"], 660, output=raw, stderr=stderr)) + else: + responses.append(subprocess.CompletedProcess(["go", "test"], 0, raw, stderr)) + + def metadata(directory, phase): + (directory / f"go-env-{phase}.stderr").write_bytes(b"metadata chatter\n") + return {"same": "toolchain"} + + with mock.patch.object(e, "source_inventory", return_value={}), \ + mock.patch.object(e, "toolchain", side_effect=metadata), \ + mock.patch.object(e.subprocess, "run", side_effect=responses) as run: + report = e.run(output) + self.assertEqual(report["summary"]["collection_ok"], not timeout) + for call in run.call_args_list: + self.assertEqual(call.kwargs["stderr"], subprocess.PIPE) + for name, _, _, _ in e.SUITES: + self.assertEqual((output / (name + ".stderr")).read_bytes(), stderr) + self.assertEqual(report["suites"][name]["stderr_sha256"], e.digest(stderr)) + self.assertEqual(report["suites"][name]["log_sha256"], e.digest((output / (name + ".jsonl")).read_bytes())) + for phase in ("before", "after"): + self.assertEqual(report["metadata_stderr_sha256"][phase], e.digest(b"metadata chatter\n")) + + def test_missing_required_case_cannot_pass_package_only(self): + result = e.summarize(events("probe", [(None, "pass")]), "probe", ("TestRequired",), 0) + self.assertEqual(result["status"], "failed") + self.assertEqual(result["missing"], ["TestRequired"]) + + def test_skipped_root_and_child_remain_visible(self): + for actions in ([('TestRequired', 'skip'), (None, 'pass')], + [('TestRequired/blocked', 'skip'), ('TestRequired', 'pass'), (None, 'pass')]): + with self.subTest(actions=actions): + raw = events("probe", actions) + result = e.summarize(raw, "probe", ("TestRequired",), 0) + self.assertEqual(result["status"], "partial") + self.assertTrue(result["skipped"]) + self.assertEqual(result["log_sha256"], e.digest(raw)) + + def test_failed_subtest_nonzero_timeout_and_truncated_log_refuse(self): + for code, actions in ((0, [('TestRequired/child', 'fail'), ('TestRequired', 'pass'), (None, 'pass')]), + (1, [('TestRequired', 'pass'), (None, 'pass')]), + (None, [('TestRequired', 'pass'), (None, 'pass')]), + (0, [('TestRequired', 'pass')])): + with self.subTest(code=code, actions=actions): + result = e.summarize(events("probe", actions), "probe", ("TestRequired",), code) + self.assertEqual(result["status"], "failed") + + def test_duplicate_terminal_wrong_package_and_invalid_json_refuse(self): + samples = [events("probe", [('TestRequired', 'pass'), ('TestRequired', 'pass'), (None, 'pass')]), + events("dockerdeploy", [('TestRequired', 'pass'), (None, 'pass')]), + b'not JSON', b'[]'] + for raw in samples: + with self.subTest(raw=raw): + with self.assertRaises((ValueError, TypeError)): + e.summarize(raw, "probe", ("TestRequired",), 0) + + def test_each_fresh_profile_entry_and_repetition_required(self): + result = complete_results()["private-orchestration"] + self.assertEqual(e.fresh_matrix(result), []) + for case in list(result["terminal_tests"]): + for status in ("skip", "fail", None): + changed = copy.deepcopy(result) + if status is None: + del changed["terminal_tests"][case] + else: + changed["terminal_tests"][case] = status + self.assertEqual(len(e.fresh_matrix(changed)), 1) + + def test_injected_pass_cannot_become_live_runtime_qualification(self): + result = e.report_results(complete_results(), True) + self.assertTrue(result["collection_ok"]) + self.assertTrue(result["private_conformance_passed"]) + self.assertFalse(result["mapped_runtime_qualified"]) + self.assertEqual(result["live_engine"]["status"], "not-qualified") + self.assertIsNone(result["live_engine"]["engine"]) + self.assertEqual(result["public_activation"], "inhibited") + + def test_kernel_gap_is_explicit_and_not_an_injected_pass(self): + results = complete_results() + results["production-exec-mechanism"]["status"] = "partial" + result = e.report_results(results, True) + self.assertTrue(result["private_conformance_passed"]) + self.assertEqual(result["production_exec_mechanism"], "partial") + self.assertFalse(result["mapped_runtime_qualified"]) + results["production-exec-mechanism"]["status"] = "failed" + self.assertFalse(e.report_results(results, True)["collection_ok"]) + + def test_any_private_gap_or_source_change_blocks_collection_success(self): + for name in ("identity", "native-state", "private-orchestration"): + results = complete_results() + results[name]["status"] = "partial" + self.assertFalse(e.report_results(results, True)["collection_ok"]) + result = e.report_results(complete_results(), False) + self.assertFalse(result["collection_ok"]) + self.assertFalse(result["private_conformance_passed"]) + + def test_source_inventory_binds_dependencies_modes_and_new_files(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + names = ["go.mod", "go.sum", "AGENTS.md", ".github/workflows/ci.yml", + "internal/dependency/policy.json", "cmd/reploy/main.go", + "tools/conformance/multiidentity/e_conformance.py", + "tools/conformance/multiidentity/test_e_conformance.py", + "tools/conformance/multiidentity/E_CONFORMANCE.md"] + for name in names: + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(name) + initial = e.source_inventory(root) + dependency = root / "internal/dependency/policy.json" + dependency.write_text("changed policy") + self.assertNotEqual(initial, e.source_inventory(root)) + dependency.write_text("internal/dependency/policy.json") + dependency.chmod(0o700) + self.assertNotEqual(initial, e.source_inventory(root)) + (root / "internal/new.go").write_text("new source") + self.assertIn("internal/new.go", e.source_inventory(root)) + dependency.unlink() + self.assertNotIn("internal/dependency/policy.json", e.source_inventory(root)) + + +if __name__ == "__main__": + unittest.main()