diff --git a/internal/dockerdeploy/mapped_conformance_test.go b/internal/dockerdeploy/mapped_conformance_test.go new file mode 100644 index 00000000..64dd7aae --- /dev/null +++ b/internal/dockerdeploy/mapped_conformance_test.go @@ -0,0 +1,257 @@ +package dockerdeploy + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "reflect" + "runtime" + "testing" + + "github.com/omry/reploy/internal/deploy" +) + +// These journeys use production planners, locks, leases, stream framing and +// lifetime code. Engine, process pinning, native admission and F observations +// remain injected. The separate probe suite exercises the real exec mechanism. +func TestMappedConformanceFreshJourneyV1(t *testing.T) { + for _, profile := range []string{"exact", "bounded"} { + for _, entry := range []struct { + name string + kind mappedEntryKindV1 + }{ + {"workload", mappedWorkloadStartV1}, + {"application", mappedApplicationCommandV1}, + {"runtime-test", mappedRuntimeTestV1}, + {"hook", mappedLifecycleHookV1}, + } { + for repetition := range 3 { + t.Run(fmt.Sprintf("%s/%s/%d", profile, entry.name, repetition), func(t *testing.T) { + f := newMappedLiveFixtureV1(t, profile, entry.kind) + var stdout, stderr bytes.Buffer + f.waiter.options.Stdout, f.waiter.options.Stderr = &stdout, &stderr + original := f.waiter.backend.run + f.waiter.backend.run = func(s CommandSpec, o RunOptions) error { + err := original(s, o) + if err == nil && s.Args[0] == "start" { + if _, err = io.WriteString(o.Stdout, "application output\n"); err == nil { + _, err = io.WriteString(o.Stderr, "application diagnostic\n") + } + } + return err + } + f.beforeRecovery = func(op *deploy.OperationLock) { + if err := op.RequireQueueEntryLeaseHeldV1(f.run.ID); err != nil { + t.Error("owned lease lost before recovery", err) + } + } + err := runMappedControlledEntryWithBackendV1(f.waiter.ctx, f.directory, f.waiter.prepared.ops, f.run.ID, f.input, f.waiter.options, f.backend) + if entry.kind == mappedWorkloadStartV1 { + if !errors.Is(err, errMappedMasterExitedV1) { + t.Fatalf("required master exit lost: %v", err) + } + } else if err != nil { + t.Fatal(err) + } + _, queue, state := f.inspect(t) + if len(queue.Runs) != 0 || state.IdentityMapping.PendingOperation != nil || f.starts != 1 || f.recoveries != 1 || f.waiter.created != 1 || f.waiter.admissions != 1 || f.waiter.removed != 1 || f.waiter.process.closed != 1 { + t.Fatal("journey did not release and retire exactly one owned instance") + } + if f.waiter.application.Load() == 0 || stdout.String() != "application output\n" || stderr.String() != "application diagnostic\n" { + t.Fatal("application streams lost, mixed, or contain release metadata") + } + if state.IdentityMapping.AcceptedUIDs == nil || len(*state.IdentityMapping.AcceptedUIDs) == 0 || !reflect.DeepEqual(state.IdentityMapping.UIDAssignments, f.input.Current.State.IdentityMapping.PendingOperation.Native.Active.UIDs) { + t.Fatal("retirement discarded admitted identity history") + } + select { + case frame := <-f.waiter.frame: + if frame.Binding != f.plan.binding || frame.Container != f.waiter.id { + t.Fatal("release substituted generation, role, image or plan") + } + default: + t.Fatal("missing release frame") + } + select { + case <-f.waiter.frame: + t.Fatal("duplicate release frame") + default: + } + facts, err := f.plan.facts() + if err != nil { + t.Fatal(err) + } + evidence, err := json.Marshal(struct { + Profile, Entry, Compiler, OS, Architecture string + Facts mappedEntryPayloadV1 + Qualification string + }{profile, entry.name, runtime.Version(), runtime.GOOS, runtime.GOARCH, facts, "private-orchestration; injected engine/native/F"}) + if err != nil { + t.Fatal(err) + } + t.Logf("E conformance receipt: %s", evidence) + }) + } + } + } +} + +func TestMappedConformanceRefusalJourneyV1(t *testing.T) { + for _, profile := range []string{"exact", "bounded"} { + for _, mode := range []string{"role", "image", "revision", "runtime-account", "maps", "owner", "occupied", "host-native"} { + t.Run(profile+"/"+mode, func(t *testing.T) { + f := newMappedLiveFixtureV1(t, profile, mappedApplicationCommandV1) + op := f.waiter.prepared.ops.Operation + before, _, err := op.ReadStateV1() + if err != nil { + t.Fatal(err) + } + switch mode { + case "role": + f.input.Command = "undeclared-role" + case "image": + f.input.Runtime.Image += "-stale" + case "revision": + f.input.Current.State.IdentityMapping.Revision = "99" + case "runtime-account": + f.input.Current.State.IdentityMapping.PendingOperation.Native.RuntimeUsers = []string{"999"} + case "maps": + f.input.Current.State.IdentityMapping.PendingOperation.Native.Active.UIDs[0].HostStart = "999999" + case "owner": + f.input.Current.State.IdentityMapping.PendingOperation.Native.Engine = "foreign" + case "occupied": + f.input.Kind = mappedControlledSessionV1 + case "host-native": + f.input.Kind = mappedHostNativeCommandV1 + } + if err := runMappedControlledEntryWithBackendV1(f.waiter.ctx, f.directory, f.waiter.prepared.ops, f.run.ID, f.input, f.waiter.options, f.backend); err == nil { + t.Fatal("invalid authority accepted") + } + if f.starts != 0 || f.waiter.created != 0 || f.waiter.admissions != 0 || f.waiter.application.Load() != 0 || f.recoveries != 0 { + t.Fatal("refusal reached an engine or application effect") + } + // Pure construction keeps the caller lock. Once construction + // succeeds, the lifetime runner owns and releases that lock even + // when its durable owner check refuses before create. + if op.RequireHeld() != nil { + op, err = deploy.AcquireOperationLock(t.Context(), f.directory) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = op.Unlock() }) + } + after, _, err := op.ReadStateV1() + if err != nil || !reflect.DeepEqual(before, after) { + t.Fatal("refusal changed durable state", err) + } + }) + } + } +} + +func TestMappedConformanceInterruptedJourneyV1(t *testing.T) { + for _, profile := range []string{"exact", "bounded"} { + for _, mode := range []string{"engine", "create", "pin", "pid", "F", "sandbox", "admission", "partial-frame", "cleanup", "retirement"} { + t.Run(profile+"/"+mode, func(t *testing.T) { + f := newMappedLiveFixtureV1(t, profile, mappedApplicationCommandV1) + w := f.waiter + switch mode { + case "engine": + w.observeFailure = true + case "create": + w.createFailure = true + case "pin": + w.pinFailure = true + case "pid": + w.process.fail = true + case "F": + w.effect.fail = true + case "sandbox": + w.effect.sandboxFailure = errors.New("missing trusted kernel/code observation") + case "admission": + w.backend.admit = func(deploy.IdentityMappingOperationsV1, context.Context, deploy.StateV1, string) (deploy.StateV1, error) { + return deploy.StateV1{}, errors.New("uncertain native admission") + } + case "partial-frame": + original := w.backend.run + w.backend.run = func(s CommandSpec, o RunOptions) error { + if s.Args[0] != "start" { + return original(s, o) + } + _, err := io.ReadFull(o.Stdin, make([]byte, 7)) + return errors.Join(err, errors.New("release interrupted")) + } + case "cleanup": + w.cleanupFailure = true + case "retirement": + f.retirementError = errors.New("retirement proof unavailable") + } + if err := runMappedControlledEntryWithBackendV1(w.ctx, f.directory, w.prepared.ops, f.run.ID, f.input, w.options, f.backend); err == nil { + t.Fatal("interrupted journey claimed success") + } + _, queue, state := f.inspect(t) + if state.IdentityMapping.PendingOperation == nil || len(queue.Runs) != 1 || w.created > 1 || w.admissions > 1 || f.recoveries > 1 { + t.Fatal("uncertainty discarded ownership or retried effects") + } + if w.created == 1 && mode != "create" && queue.Runs[0].Container != w.id && w.removed != 1 { + t.Fatal("uncertainty neither retained nor cleaned the exact instance") + } + if mode != "cleanup" && mode != "retirement" { + if w.application.Load() != 0 { + t.Fatal("refused or incomplete release reached application input") + } + select { + case <-w.frame: + t.Fatal("refused journey delivered a complete release frame") + default: + } + } + }) + } + } +} + +func TestMappedConformanceCancellationJourneyV1(t *testing.T) { + for _, profile := range []string{"exact", "bounded"} { + t.Run(profile, func(t *testing.T) { + f := newMappedLiveFixtureV1(t, profile, mappedApplicationCommandV1) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + f.waiter.options.Context, f.untilCancel = ctx, true + result := make(chan error, 1) + go func() { + result <- runMappedControlledEntryWithBackendV1(f.waiter.ctx, f.directory, f.waiter.prepared.ops, f.run.ID, f.input, f.waiter.options, f.backend) + }() + select { + case <-f.waiter.frame: + case <-f.waiter.ctx.Done(): + t.Fatal("release did not complete within admission deadline") + } + op, err := deploy.AcquireOperationLock(f.waiter.ctx, f.directory) + if err != nil { + t.Fatal(err) + } + queue, _, err := op.ReadLiveRunQueueV1() + owned := err == nil && len(queue.Runs) == 1 && queue.Runs[0].Container == f.waiter.id && op.RequireQueueEntryLeaseHeldV1(f.run.ID) == nil + _ = op.Unlock() + if !owned { + t.Fatal("attached run did not retain exact ownership and lease", err) + } + cancel() + select { + case err := <-result: + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation lost: %v", err) + } + case <-f.waiter.ctx.Done(): + t.Fatal("cancellation did not finish within fixture deadline") + } + _, queue, state := f.inspect(t) + if len(queue.Runs) != 0 || state.IdentityMapping.PendingOperation != nil || f.waiter.removed != 1 || f.recoveries != 1 || state.IdentityMapping.AcceptedUIDs == nil || len(*state.IdentityMapping.AcceptedUIDs) == 0 { + t.Fatal("cancelled journey lost cleanup or admitted mapping history") + } + }) + } +} diff --git a/internal/probe/mapped_exec_linux_test.go b/internal/probe/mapped_exec_linux_test.go index 3bcdf833..212c987a 100644 --- a/internal/probe/mapped_exec_linux_test.go +++ b/internal/probe/mapped_exec_linux_test.go @@ -599,6 +599,9 @@ func TestMappedDescriptorRaceHelperV1(t *testing.T) { panic(err) } close(ready) + if args[0] == "descriptor-shared-close" { + mappedSharedCloseObserverV1(&damaged) + } for { if _, _, errno := syscall.RawSyscall(unix.SYS_FCNTL, 4096, unix.F_GETFD, 0); errno == unix.EBADF { atomic.StoreUint32(&damaged, 1) @@ -645,6 +648,22 @@ func TestMappedDescriptorRaceHelperV1(t *testing.T) { os.Exit(Main(args[2:], os.Stdin, os.Stdout, os.Stderr)) } +// The deliberately wrong shared closer damages runtime descriptors too. Its +// observer must not depend on Go scheduling or descriptor operations afterward. +// Other modes retain the real concurrent opener above. +// +//go:nosplit +//go:norace +func mappedSharedCloseObserverV1(damaged *uint32) { + for { + if _, _, errno := syscall.RawSyscall(unix.SYS_FCNTL, 4096, unix.F_GETFD, 0); errno == unix.EBADF { + atomic.StoreUint32(damaged, 1) + for { + } + } + } +} + // Deliberately wrong test-only closer. A peer detects the shared table's lost // high fd. Stay in a raw tail so a runtime poller failure cannot masquerade as // that specific independent observation. The parent bounds/reaps the process. diff --git a/tools/conformance/multiidentity/E_CONFORMANCE.md b/tools/conformance/multiidentity/E_CONFORMANCE.md new file mode 100644 index 00000000..75deccb4 --- /dev/null +++ b/tools/conformance/multiidentity/E_CONFORMANCE.md @@ -0,0 +1,56 @@ +# Private milestone E conformance + +Run from the Reploy checkout with a native Go toolchain. Use a new output +directory under an existing private evidence parent: + +```sh +python3 tools/conformance/multiidentity/e_conformance.py --output-dir /absolute/new/evidence +python3 -m unittest discover -s tools/conformance/multiidentity -p test_e_conformance.py +``` + +The runner creates no engine resources. Its existing production-startup tests +use disposable Linux user/network namespaces and bounded child processes. +They need `cc`, `unshare`, `newuidmap`, `newgidmap` and delegated subordinate +IDs. Missing prerequisites remain visible test skips, not qualification. +Mapped runtime support remains limited by the approved Linux/Podman cohort; +running the portable fixture suites elsewhere does not expand that cohort. + +`report.json` binds source bytes and file modes, Go configuration and compiler +and linker hashes, host kernel/architecture, fixture helpers and the pinned +OCI seccomp seed. It checks source/toolchain stability after collection. +Each suite has its exact command, elapsed time, JSON event log digest and +complete terminal test statuses. Only stdout is parsed as JSON. Each suite's +stderr is retained separately in `.stderr` with its own digest, including +partial timeout diagnostics. The two metadata commands retain `go-env-before.stderr` +and `go-env-after.stderr`; their digests are recorded separately from toolchain facts. +Private fresh journeys also emit their exact +closed plan, including profile, maps, immutable image, role, generation, +runtime/kernel/network policies and stream mode. These are fixture facts. +The actual production exec tests identify their compiled source/toolchain +and verify the production exec tail; this is mechanism evidence, not a +retained launched-image binary or live-engine receipt. + +The integrated private journey repeats workload, application, runtime-test and +hook cases three times for each of `exact` and `bounded`. It checks one release, +separate application streams, owned completion/cleanup and retained mapping +history. Invalid authority and interrupted proof/release paths check absence of +application effects. Existing suites cover native atomic admission, canonical +policy, capabilities, private/high/concurrent descriptors, occupied refusal, +replacement refusal, cancellation and cleanup ownership. + +Collection exit zero means the required private suites passed, their fresh +matrix is complete, no selected test failed and the source/toolchain stayed +stable. A partially skipped production-exec suite remains `partial`, with +each gap listed; it cannot become a full kernel qualification. A missing +required test, failed test, timeout, malformed log or changed source fails +collection. Existing evidence directories are never overwritten. + +No result from this runner qualifies a live engine or public mapped execution. +`mapped_runtime_qualified` is always false. Production F code/mount/network +observations, an exact engine/OCI-policy receipt, unresolved kernel tuples and +G activation remain separate gates. Working replacement and occupied exec +remain refused. The older `conformance.py` Podman probe is a distinct lower +level experiment and cannot stand in for this integrated E evidence. +The opt-in `TestMappedPodmanCreateLiveV1` is explicitly excluded from this +private runner, even when its enabling environment is set. The exact exclusion +is retained in the report and command; live-engine qualification stays open. diff --git a/tools/conformance/multiidentity/e_conformance.py b/tools/conformance/multiidentity/e_conformance.py new file mode 100644 index 00000000..bb3ae5ae --- /dev/null +++ b/tools/conformance/multiidentity/e_conformance.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Collect private E test evidence. Never qualify a live mapped engine.""" + +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import os +from pathlib import Path +import platform +import shutil +import subprocess +import sys +import time + + +ROOT = Path(__file__).resolve().parents[3] +PACKAGE = "github.com/omry/reploy/internal/" +SUITES = ( + ("identity", "identitymapping", ".", ("TestHostPreflightRequiresCompleteSources",)), + ("native-state", "deploy", "^TestNativeMappingV1", ( + "TestNativeMappingV1AdmissionIsOneDurableTransaction", + "TestNativeMappingV1AdmissionWriteFailureNeverGrantsRelease", + )), + ("private-orchestration", "dockerdeploy", "Mapped", ( + "TestMappedConformanceFreshJourneyV1", + "TestMappedConformanceRefusalJourneyV1", + "TestMappedConformanceInterruptedJourneyV1", + "TestMappedConformanceCancellationJourneyV1", + "TestMappedReplacementEntryV1QualifiedOwnerRefusesBeforeEffects", + "TestMappedEntryAdaptersV1CheckBothSessionOwners", + "TestMappedSeccompCompilerRemovesEveryEscapeAllowance", + )), + ("production-exec-mechanism", "probe", "^TestMapped(Production|StartupPreflight|StandardDescriptors|KernelCapabilityLimit)", ( + "TestMappedProductionExecTailV1", "TestMappedProductionStartupV1", + )), +) + + +def digest(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def command(argv: list[str], stderr_path: Path, *, timeout: int = 60) -> bytes: + try: + process = subprocess.run(argv, cwd=ROOT, timeout=timeout, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + except subprocess.TimeoutExpired as error: + stderr_path.write_bytes(error.stderr or b"") + raise + stderr_path.write_bytes(process.stderr) + process.check_returncode() + return process.stdout + + +def source_inventory(root: Path = ROOT) -> dict[str, dict]: + # Include embedded policy/data, fixture code and build inputs, not just the + # new test. Ignore Python caches and evidence directories by construction. + paths = {root / name for name in ("go.mod", "go.sum", "AGENTS.md", ".github/workflows/ci.yml")} + for directory in ("internal", "cmd"): + paths.update(p for p in (root / directory).rglob("*") if p.is_file()) + here = root / "tools/conformance/multiidentity" + paths.update(here / name for name in ("e_conformance.py", "test_e_conformance.py", "E_CONFORMANCE.md")) + return { + p.relative_to(root).as_posix(): {"sha256": digest(p.read_bytes()), "mode": p.stat().st_mode & 0o777} + for p in sorted(paths) + } + + +def toolchain(output: Path, phase: str) -> dict: + env = json.loads(command(["go", "env", "-json", "GOOS", "GOARCH", "GOVERSION", "GOTOOLDIR", "GOEXPERIMENT", "GOFLAGS", "CGO_ENABLED"], + output / f"go-env-{phase}.stderr")) + tools = {} + suffix = ".exe" if os.name == "nt" else "" + for name in ("compile", "link"): + path = Path(env["GOTOOLDIR"]) / (name + suffix) + tools[name] = {"path": str(path), "sha256": digest(path.read_bytes())} + for name in ("go", "cc", "unshare", "newuidmap", "newgidmap"): + located = shutil.which(name) + tools[name] = None if located is None else {"path": located, "sha256": digest(Path(located).read_bytes())} + return {"go_env": env, "tools": tools, + "runtime_flags": {name: os.environ.get(name) for name in ("GODEBUG", "GOMAXPROCS", "GOAMD64", "GOARM64")}, + "host": {"system": platform.system(), "release": platform.release(), "machine": platform.machine()}} + + +def summarize(raw: bytes, package: str, required: tuple[str, ...], returncode: int | None) -> dict: + events = [json.loads(line) for line in raw.splitlines() if line.strip()] + if any(not isinstance(e, dict) or e.get("Package") != PACKAGE + package for e in events): + raise ValueError("unexpected Go event package or shape") + terminal = {} + package_actions = [] + for event in events: + action = event.get("Action") + if action in ("pass", "fail", "skip"): + if "Test" in event: + if event["Test"] in terminal: + raise ValueError("duplicate terminal test event") + terminal[event["Test"]] = action + else: + package_actions.append(action) + missing = [name for name in required if name not in terminal] + failed = sorted(name for name, action in terminal.items() if action == "fail") + skipped = sorted(name for name, action in terminal.items() if action == "skip") + required_gaps = [name for name in required if terminal.get(name) != "pass"] + ok = returncode == 0 and package_actions == ["pass"] and not missing and not failed + return {"status": "failed" if not ok else "partial" if skipped or required_gaps else "passed", + "missing": missing, "failed": failed, "skipped": skipped, + "required_gaps": required_gaps, "terminal_tests": terminal, + "returncode": returncode, "log_sha256": digest(raw)} + + +def fresh_matrix(result: dict) -> list[str]: + terminals = result["terminal_tests"] + return [f"{profile}/{entry}/{repeat}" + for profile in ("exact", "bounded") + for entry in ("workload", "application", "runtime-test", "hook") + for repeat in range(3) + if terminals.get(f"TestMappedConformanceFreshJourneyV1/{profile}/{entry}/{repeat}") != "pass"] + + +def report_results(results: dict, stable: bool) -> dict: + private_names = ("identity", "native-state", "private-orchestration") + private = all(results[name]["status"] == "passed" for name in private_names) + matrix_gaps = fresh_matrix(results["private-orchestration"]) + kernel = results["production-exec-mechanism"] + # A zero process exit is collection success, never a mapped-support claim. + collection_ok = stable and not matrix_gaps and private and kernel["status"] != "failed" + return {"collection_ok": collection_ok, "source_and_toolchain_stable": stable, + "private_conformance_passed": stable and private and not matrix_gaps, + "fresh_instance_matrix_gaps": matrix_gaps, + "production_exec_mechanism": kernel["status"], + "live_engine": {"status": "not-qualified", "engine": None, "oci_runtime": None, + "reason": "No live engine is exercised by this runner; fixture engine/native/F observations are injected."}, + "mapped_runtime_qualified": False, "public_activation": "inhibited", + "remaining_gates": ["Production F code/mount/network observation and exact live engine/OCI policy qualification.", + "Working replacement and occupied entry remain refused; G public activation is separate.", + "Every skipped/missing kernel case remains an explicit qualification gap."]} + + +def run(output: Path) -> dict: + output.mkdir(mode=0o700, parents=False, exist_ok=False) + inventory = source_inventory() + context = toolchain(output, "before") + results = {} + for name, package, selection, required in SUITES: + argv = ["go", "test", "-json", "-count=1", "-timeout=10m", "./internal/" + package, "-run", selection] + if name == "private-orchestration": + # The opt-in engine test belongs to separate live qualification. + # Never run it here, even if its enabling environment is inherited. + argv += ["-skip", "^TestMappedPodmanCreateLiveV1$"] + started = time.monotonic() + try: + process = subprocess.run(argv, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=660, check=False) + raw, stderr, code = process.stdout, process.stderr, process.returncode + except subprocess.TimeoutExpired as error: + raw, stderr, code = error.stdout or b"", error.stderr or b"", None + (output / (name + ".jsonl")).write_bytes(raw) + (output / (name + ".stderr")).write_bytes(stderr) + try: + result = summarize(raw, package, required, code) + except (ValueError, TypeError, KeyError) as error: + result = {"status": "failed", "error": str(error), "terminal_tests": {}, "log_sha256": digest(raw), "returncode": code} + result.update(command=argv, elapsed_seconds=time.monotonic() - started, + stderr_sha256=digest(stderr)) + results[name] = result + print(json.dumps({"suite": name, "status": result["status"], "elapsed_seconds": result["elapsed_seconds"]}), flush=True) + final_context = toolchain(output, "after") + stable = inventory == source_inventory() and context == final_context + report = {"schema_version": 1, "collected_at": dt.datetime.now(dt.timezone.utc).isoformat(), + "repository": str(ROOT), "source": inventory, "context": context, + "metadata_stderr_sha256": {phase: digest((output / f"go-env-{phase}.stderr").read_bytes()) + for phase in ("before", "after")}, + "policy_source": "internal/dockerdeploy/seccomp/docker-v0.2.4.json", + "selection_exclusions": {"TestMappedPodmanCreateLiveV1": "Separate opt-in live-engine qualification; not exercised or qualified here."}, + "suites": results, "summary": report_results(results, stable)} + (output / "report.json").write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return report + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", required=True, type=Path, help="new evidence directory; its parent must exist") + args = parser.parse_args() + try: + report = run(args.output_dir) + except (OSError, subprocess.SubprocessError, ValueError) as error: + print(f"E conformance collection failed: {error}", file=sys.stderr) + return 1 + print(json.dumps(report["summary"], indent=2)) + return 0 if report["summary"]["collection_ok"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/conformance/multiidentity/test_e_conformance.py b/tools/conformance/multiidentity/test_e_conformance.py new file mode 100644 index 00000000..41b2d797 --- /dev/null +++ b/tools/conformance/multiidentity/test_e_conformance.py @@ -0,0 +1,194 @@ +from __future__ import annotations + +import copy +import importlib.util +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + + +HERE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location("e_conformance", HERE / "e_conformance.py") +e = importlib.util.module_from_spec(spec) +spec.loader.exec_module(e) + + +def events(package: str, actions: list[tuple[str | None, str]]) -> bytes: + return b"\n".join(json.dumps({"Package": e.PACKAGE + package, "Action": action, + **({"Test": test} if test is not None else {})}).encode() + for test, action in actions) + + +def complete_results() -> dict: + results = {name: {"status": "passed", "terminal_tests": {}} for name, _, _, _ in e.SUITES} + terminals = results["private-orchestration"]["terminal_tests"] + for profile in ("exact", "bounded"): + for entry in ("workload", "application", "runtime-test", "hook"): + for repeat in range(3): + terminals[f"TestMappedConformanceFreshJourneyV1/{profile}/{entry}/{repeat}"] = "pass" + return results + + +class EvidenceTests(unittest.TestCase): + def test_metadata_json_remains_separate_from_stderr_on_success_and_failure(self): + with tempfile.TemporaryDirectory() as temporary: + stderr = Path(temporary) / "metadata.stderr" + for code in (0, 3): + argv = [sys.executable, "-c", + 'import sys; print("{\\\"GOOS\\\":\\\"linux\\\"}"); ' + 'sys.stderr.write("go: downloading example.invalid/module v1\\n"); ' + f'sys.exit({code})'] + if code: + with self.assertRaises(subprocess.CalledProcessError): + e.command(argv, stderr) + else: + self.assertEqual(json.loads(e.command(argv, stderr)), {"GOOS": "linux"}) + self.assertEqual(stderr.read_bytes(), b"go: downloading example.invalid/module v1\n") + + def test_metadata_timeout_retains_stderr_and_stays_a_failure(self): + with tempfile.TemporaryDirectory() as temporary: + stderr = Path(temporary) / "metadata.stderr" + error = subprocess.TimeoutExpired(["go", "env"], 60, output=b"{}", stderr=b"partial diagnostic") + with mock.patch.object(e.subprocess, "run", side_effect=error): + with self.assertRaises(subprocess.TimeoutExpired): + e.command(["go", "env"], stderr) + self.assertEqual(stderr.read_bytes(), b"partial diagnostic") + + def test_suite_chatter_and_timeout_keep_separate_digest_bound_artifacts(self): + for timeout in (False, True): + with self.subTest(timeout=timeout), tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) / "evidence" + responses = [] + for index, (_, package, _, required) in enumerate(e.SUITES): + cases = list(required) + if package == "dockerdeploy": + cases += list(complete_results()["private-orchestration"]["terminal_tests"]) + raw = events(package, [(case, "pass") for case in cases] + [(None, "pass")]) + stderr = b"go: downloading example.invalid/module v1\n" + if timeout and index == 0: + responses.append(subprocess.TimeoutExpired(["go", "test"], 660, output=raw, stderr=stderr)) + else: + responses.append(subprocess.CompletedProcess(["go", "test"], 0, raw, stderr)) + + def metadata(directory, phase): + (directory / f"go-env-{phase}.stderr").write_bytes(b"metadata chatter\n") + return {"same": "toolchain"} + + with mock.patch.object(e, "source_inventory", return_value={}), \ + mock.patch.object(e, "toolchain", side_effect=metadata), \ + mock.patch.object(e.subprocess, "run", side_effect=responses) as run: + report = e.run(output) + self.assertEqual(report["summary"]["collection_ok"], not timeout) + for call in run.call_args_list: + self.assertEqual(call.kwargs["stderr"], subprocess.PIPE) + for name, _, _, _ in e.SUITES: + self.assertEqual((output / (name + ".stderr")).read_bytes(), stderr) + self.assertEqual(report["suites"][name]["stderr_sha256"], e.digest(stderr)) + self.assertEqual(report["suites"][name]["log_sha256"], e.digest((output / (name + ".jsonl")).read_bytes())) + for phase in ("before", "after"): + self.assertEqual(report["metadata_stderr_sha256"][phase], e.digest(b"metadata chatter\n")) + + def test_missing_required_case_cannot_pass_package_only(self): + result = e.summarize(events("probe", [(None, "pass")]), "probe", ("TestRequired",), 0) + self.assertEqual(result["status"], "failed") + self.assertEqual(result["missing"], ["TestRequired"]) + + def test_skipped_root_and_child_remain_visible(self): + for actions in ([('TestRequired', 'skip'), (None, 'pass')], + [('TestRequired/blocked', 'skip'), ('TestRequired', 'pass'), (None, 'pass')]): + with self.subTest(actions=actions): + raw = events("probe", actions) + result = e.summarize(raw, "probe", ("TestRequired",), 0) + self.assertEqual(result["status"], "partial") + self.assertTrue(result["skipped"]) + self.assertEqual(result["log_sha256"], e.digest(raw)) + + def test_failed_subtest_nonzero_timeout_and_truncated_log_refuse(self): + for code, actions in ((0, [('TestRequired/child', 'fail'), ('TestRequired', 'pass'), (None, 'pass')]), + (1, [('TestRequired', 'pass'), (None, 'pass')]), + (None, [('TestRequired', 'pass'), (None, 'pass')]), + (0, [('TestRequired', 'pass')])): + with self.subTest(code=code, actions=actions): + result = e.summarize(events("probe", actions), "probe", ("TestRequired",), code) + self.assertEqual(result["status"], "failed") + + def test_duplicate_terminal_wrong_package_and_invalid_json_refuse(self): + samples = [events("probe", [('TestRequired', 'pass'), ('TestRequired', 'pass'), (None, 'pass')]), + events("dockerdeploy", [('TestRequired', 'pass'), (None, 'pass')]), + b'not JSON', b'[]'] + for raw in samples: + with self.subTest(raw=raw): + with self.assertRaises((ValueError, TypeError)): + e.summarize(raw, "probe", ("TestRequired",), 0) + + def test_each_fresh_profile_entry_and_repetition_required(self): + result = complete_results()["private-orchestration"] + self.assertEqual(e.fresh_matrix(result), []) + for case in list(result["terminal_tests"]): + for status in ("skip", "fail", None): + changed = copy.deepcopy(result) + if status is None: + del changed["terminal_tests"][case] + else: + changed["terminal_tests"][case] = status + self.assertEqual(len(e.fresh_matrix(changed)), 1) + + def test_injected_pass_cannot_become_live_runtime_qualification(self): + result = e.report_results(complete_results(), True) + self.assertTrue(result["collection_ok"]) + self.assertTrue(result["private_conformance_passed"]) + self.assertFalse(result["mapped_runtime_qualified"]) + self.assertEqual(result["live_engine"]["status"], "not-qualified") + self.assertIsNone(result["live_engine"]["engine"]) + self.assertEqual(result["public_activation"], "inhibited") + + def test_kernel_gap_is_explicit_and_not_an_injected_pass(self): + results = complete_results() + results["production-exec-mechanism"]["status"] = "partial" + result = e.report_results(results, True) + self.assertTrue(result["private_conformance_passed"]) + self.assertEqual(result["production_exec_mechanism"], "partial") + self.assertFalse(result["mapped_runtime_qualified"]) + results["production-exec-mechanism"]["status"] = "failed" + self.assertFalse(e.report_results(results, True)["collection_ok"]) + + def test_any_private_gap_or_source_change_blocks_collection_success(self): + for name in ("identity", "native-state", "private-orchestration"): + results = complete_results() + results[name]["status"] = "partial" + self.assertFalse(e.report_results(results, True)["collection_ok"]) + result = e.report_results(complete_results(), False) + self.assertFalse(result["collection_ok"]) + self.assertFalse(result["private_conformance_passed"]) + + def test_source_inventory_binds_dependencies_modes_and_new_files(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + names = ["go.mod", "go.sum", "AGENTS.md", ".github/workflows/ci.yml", + "internal/dependency/policy.json", "cmd/reploy/main.go", + "tools/conformance/multiidentity/e_conformance.py", + "tools/conformance/multiidentity/test_e_conformance.py", + "tools/conformance/multiidentity/E_CONFORMANCE.md"] + for name in names: + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(name) + initial = e.source_inventory(root) + dependency = root / "internal/dependency/policy.json" + dependency.write_text("changed policy") + self.assertNotEqual(initial, e.source_inventory(root)) + dependency.write_text("internal/dependency/policy.json") + dependency.chmod(0o700) + self.assertNotEqual(initial, e.source_inventory(root)) + (root / "internal/new.go").write_text("new source") + self.assertIn("internal/new.go", e.source_inventory(root)) + dependency.unlink() + self.assertNotIn("internal/dependency/policy.json", e.source_inventory(root)) + + +if __name__ == "__main__": + unittest.main()