From f42b078d160d8040c50b5b6e7ae24e4e84a25b4b Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:14:53 -0400 Subject: [PATCH 01/12] Add bounded Docker Torch import diagnostic and retain r2 evidence --- .agents/skills/fine-tuning/SKILL.md | 1 + .../fine-tuning/reference/sft-training.md | 21 ++ .claude/skills/fine-tuning/SKILL.md | 1 + .../fine-tuning/reference/sft-training.md | 21 ++ .skills/fine-tuning/SKILL.md | 1 + .skills/fine-tuning/reference/sft-training.md | 21 ++ .../docker-torch-import-r2-diagnosis.md | 68 +++++ scripts/diagnose_docker_torch_import.py | 247 ++++++++++++++++++ .../test_diagnose_docker_torch_import.py | 101 +++++++ 9 files changed, 482 insertions(+) create mode 100644 docs/review/docker-torch-import-r2-diagnosis.md create mode 100644 scripts/diagnose_docker_torch_import.py create mode 100644 tests/scripts/test_diagnose_docker_torch_import.py diff --git a/.agents/skills/fine-tuning/SKILL.md b/.agents/skills/fine-tuning/SKILL.md index 73c91863..4a39a46a 100644 --- a/.agents/skills/fine-tuning/SKILL.md +++ b/.agents/skills/fine-tuning/SKILL.md @@ -57,6 +57,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Check SFT loss masking (tokenizer only) | `python tuner.py doctor sft-mask --dataset-path --model ` | | Inspect native training host candidate | `python tuner.py doctor native-training --json` — read-only platform/Python/package metadata; see `reference/native-local-training.md` | | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | +| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for the bounded CPU-only bare and guarded probes; see `reference/sft-training.md` | | Native MLX prepared SFT candidate | Internal exact-token batch/loss and runtime comparison only; no native launch CLI or API provider yet; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.agents/skills/fine-tuning/reference/sft-training.md b/.agents/skills/fine-tuning/reference/sft-training.md index 3029c594..18e6af1e 100644 --- a/.agents/skills/fine-tuning/reference/sft-training.md +++ b/.agents/skills/fine-tuning/reference/sft-training.md @@ -492,6 +492,27 @@ the consumed claim, roots, container and retained logs; the diagnostics never authorize replay. See `docs/review/offline-sft-worker-diagnostics.md` for the October 20 evidence boundary, CPU regressions and fresh-run prerequisites. +For a fresh, credential-free Docker Torch import reproduction against an exact +retained packaged release, use the checked-in +`scripts/diagnose_docker_torch_import.py` with `--docker`, `--docker-config`, +`--release`, the exact `--expected-release-sha256`, `--daemon-id`, and a +nonexistent private `--output-dir`. It +requires the release's image ID to be present locally, starts one retained +read-only container with at most 2 CPUs, 4 GiB, 120 seconds, no network or GPU, +and no model mount. The bare mode has no source mount. Its result and stdout contain only closed import +categories and identities. Keep the new container and `container.id`; do not +replay the original training claim. `PASS` only says Torch imports in that +reduced environment; a rejection class is a reproduction boundary, not proof +of a dependency or repair. Review the pinned package/runtime source before any +change to profile, image, or runtime policy. +When the bare probe passes, an optional `--source-root` binds the exact retained +staged engine read-only and repeats Torch import after the offline worker's +closure validation, owned-module guard, and trainer environment bootstrap. +Use a fresh output directory for each probe. The guarded result adds the last +closed stage; it still runs no trainer or model code. +The exact r2 CPU evidence and its unresolved runtime difference are in +`docs/review/docker-torch-import-r2-diagnosis.md`. + After an intentional worker source change, check and refresh existing manifests with the checked-in maintainers in dependency order: `regenerate_offline_sft_worker_closure.py`, diff --git a/.claude/skills/fine-tuning/SKILL.md b/.claude/skills/fine-tuning/SKILL.md index 73c91863..4a39a46a 100644 --- a/.claude/skills/fine-tuning/SKILL.md +++ b/.claude/skills/fine-tuning/SKILL.md @@ -57,6 +57,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Check SFT loss masking (tokenizer only) | `python tuner.py doctor sft-mask --dataset-path --model ` | | Inspect native training host candidate | `python tuner.py doctor native-training --json` — read-only platform/Python/package metadata; see `reference/native-local-training.md` | | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | +| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for the bounded CPU-only bare and guarded probes; see `reference/sft-training.md` | | Native MLX prepared SFT candidate | Internal exact-token batch/loss and runtime comparison only; no native launch CLI or API provider yet; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.claude/skills/fine-tuning/reference/sft-training.md b/.claude/skills/fine-tuning/reference/sft-training.md index 3029c594..18e6af1e 100644 --- a/.claude/skills/fine-tuning/reference/sft-training.md +++ b/.claude/skills/fine-tuning/reference/sft-training.md @@ -492,6 +492,27 @@ the consumed claim, roots, container and retained logs; the diagnostics never authorize replay. See `docs/review/offline-sft-worker-diagnostics.md` for the October 20 evidence boundary, CPU regressions and fresh-run prerequisites. +For a fresh, credential-free Docker Torch import reproduction against an exact +retained packaged release, use the checked-in +`scripts/diagnose_docker_torch_import.py` with `--docker`, `--docker-config`, +`--release`, the exact `--expected-release-sha256`, `--daemon-id`, and a +nonexistent private `--output-dir`. It +requires the release's image ID to be present locally, starts one retained +read-only container with at most 2 CPUs, 4 GiB, 120 seconds, no network or GPU, +and no model mount. The bare mode has no source mount. Its result and stdout contain only closed import +categories and identities. Keep the new container and `container.id`; do not +replay the original training claim. `PASS` only says Torch imports in that +reduced environment; a rejection class is a reproduction boundary, not proof +of a dependency or repair. Review the pinned package/runtime source before any +change to profile, image, or runtime policy. +When the bare probe passes, an optional `--source-root` binds the exact retained +staged engine read-only and repeats Torch import after the offline worker's +closure validation, owned-module guard, and trainer environment bootstrap. +Use a fresh output directory for each probe. The guarded result adds the last +closed stage; it still runs no trainer or model code. +The exact r2 CPU evidence and its unresolved runtime difference are in +`docs/review/docker-torch-import-r2-diagnosis.md`. + After an intentional worker source change, check and refresh existing manifests with the checked-in maintainers in dependency order: `regenerate_offline_sft_worker_closure.py`, diff --git a/.skills/fine-tuning/SKILL.md b/.skills/fine-tuning/SKILL.md index 73c91863..4a39a46a 100644 --- a/.skills/fine-tuning/SKILL.md +++ b/.skills/fine-tuning/SKILL.md @@ -57,6 +57,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Check SFT loss masking (tokenizer only) | `python tuner.py doctor sft-mask --dataset-path --model ` | | Inspect native training host candidate | `python tuner.py doctor native-training --json` — read-only platform/Python/package metadata; see `reference/native-local-training.md` | | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | +| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for the bounded CPU-only bare and guarded probes; see `reference/sft-training.md` | | Native MLX prepared SFT candidate | Internal exact-token batch/loss and runtime comparison only; no native launch CLI or API provider yet; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.skills/fine-tuning/reference/sft-training.md b/.skills/fine-tuning/reference/sft-training.md index 3029c594..18e6af1e 100644 --- a/.skills/fine-tuning/reference/sft-training.md +++ b/.skills/fine-tuning/reference/sft-training.md @@ -492,6 +492,27 @@ the consumed claim, roots, container and retained logs; the diagnostics never authorize replay. See `docs/review/offline-sft-worker-diagnostics.md` for the October 20 evidence boundary, CPU regressions and fresh-run prerequisites. +For a fresh, credential-free Docker Torch import reproduction against an exact +retained packaged release, use the checked-in +`scripts/diagnose_docker_torch_import.py` with `--docker`, `--docker-config`, +`--release`, the exact `--expected-release-sha256`, `--daemon-id`, and a +nonexistent private `--output-dir`. It +requires the release's image ID to be present locally, starts one retained +read-only container with at most 2 CPUs, 4 GiB, 120 seconds, no network or GPU, +and no model mount. The bare mode has no source mount. Its result and stdout contain only closed import +categories and identities. Keep the new container and `container.id`; do not +replay the original training claim. `PASS` only says Torch imports in that +reduced environment; a rejection class is a reproduction boundary, not proof +of a dependency or repair. Review the pinned package/runtime source before any +change to profile, image, or runtime policy. +When the bare probe passes, an optional `--source-root` binds the exact retained +staged engine read-only and repeats Torch import after the offline worker's +closure validation, owned-module guard, and trainer environment bootstrap. +Use a fresh output directory for each probe. The guarded result adds the last +closed stage; it still runs no trainer or model code. +The exact r2 CPU evidence and its unresolved runtime difference are in +`docs/review/docker-torch-import-r2-diagnosis.md`. + After an intentional worker source change, check and refresh existing manifests with the checked-in maintainers in dependency order: `regenerate_offline_sft_worker_closure.py`, diff --git a/docs/review/docker-torch-import-r2-diagnosis.md b/docs/review/docker-torch-import-r2-diagnosis.md new file mode 100644 index 00000000..1a4836d5 --- /dev/null +++ b/docs/review/docker-torch-import-r2-diagnosis.md @@ -0,0 +1,68 @@ +# Docker r2 Torch import boundary — CPU diagnosis, 2026-10-20 + +## Retained failure + +The one authorized r2 public start on engine `0d515ae1ae235d0858802228ce4af8dba8fccd03` +and host `85ea6b56af89f6dcb7c0a2c7644e95c511762486` was accepted at +13:47:50Z and failed at 13:47:52Z. Container +`5256d2446d3751e423a6ff535c3ed9a13a2bc5df92765a61db5da700bf7e601d` +remains retained with exit 23. Its 81-byte trainer stderr, SHA-256 +`95d34d74eae9cb55a223eeda8428ee4387ee79095e5c0f39a0ceb60da9f45513`, +contains only `OFFLINE_SFT_WORKER_REJECTED phase=TRAINER category=IMPORT +milestone=TORCH_IMPORT`. This locates the first Torch import boundary in +`Trainers/sft/train_sft.py`; it does not identify an import dependency or a +remedy. The earlier bare marker on container +`07ecde6d5b798abf51eb85595be165039cdc1043295cbd0a5b0d3d526064bfd7` remains historical +and uninterpreted. + +The release file SHA-256 is +`5f65824ceb5e250a58314da0233b9c07f1d63f1b0a39892d9b36af694938bfe6`; +its immutable image ID is +`sha256:e737a0a091474c4892690690e712bc1f932f108856a3d5df60383e1cd9436e1e`. +The retained final runtime capture records CPython 3.12.3 and Torch +`2.11.0+cu128`. Source, image, release and installed wheel were not changed. + +## Targeted CPU experiments + +The checked-in `scripts/diagnose_docker_torch_import.py` binds that release +file hash, daemon ID and exact local image ID. It starts a fresh retained +container with at most 2 CPUs, 4 GiB and 120 seconds; network and GPU devices +are disabled. It has no credentials, model mount or training entrypoint, and +emits only closed categories and identity fields. + +The bare image probe returned `PASS` in container +`33af62c4fdda5d855b3b154c2937912b5266a88a48f2c0757b3664fe42a45917`. +The final guarded probe mounted the r2 staged source read-only, validated the +offline worker closure, installed its owned-module guard, ran +`init_trainer_env()`, and imported Torch successfully. Its retained container +is `2e5b6d06a4cc946541a475da72b7aa7e851b5840f86fd675c2099540b6fc1045`; +`evidence-guarded-final2/result.json` has SHA-256 +`326b59ebf3896a737732661895ef67b3efcf2527d65ce912b02ff20d9437ee33`. +The exact final checked-in helper source has SHA-256 +`c214fc8b9d617fe7171e7eac987d93187169b8a637fac9ef24b4323a70bf6c53`. +These artifacts are under the private Windows WSL root +`/home/profsynapse/.local/share/synaptic-qualification/torch-import-r4-20261020/`. + +Earlier guarded diagnostic containers returned `READ_MANIFEST/OS` because +the probe dropped all capabilities while running as root against mode-0600 +staged files owned by the WSL user. Running the diagnostic as that owner +resolved this probe setup failure. Those containers and reports are retained; +their failures do not reinterpret r2. + +## Interpretation and next boundary + +Torch is importable in the exact image both bare and after the retained worker +closure, guard and trainer bootstrap under the finite CPU diagnostic. The r2 +container requested GPU 0 and used root, default capabilities and a writable +root filesystem; the CPU probes deliberately did not reproduce that full +runtime. The cause of r2's Torch import failure remains unresolved. In +particular, the evidence does not support changing packages, image, recipe, +model, GPU settings or SFT runtime code yet. + +The r2 claim, container and all older failure evidence remain consumed and +preserved. No training start, image build, pull, model load, package install or +HF action occurred during this diagnosis. A further comparison of the +original GPU/runtime environment needs a separately reviewed exact scope; +this diagnostic does not authorize another training attempt. No runtime +closure or Modal lock refresh is needed for the diagnostic script and skill +changes, because no locked runtime member changed. diff --git a/scripts/diagnose_docker_torch_import.py b/scripts/diagnose_docker_torch_import.py new file mode 100644 index 00000000..1065d548 --- /dev/null +++ b/scripts/diagnose_docker_torch_import.py @@ -0,0 +1,247 @@ +"""Bounded, credential-free Torch import diagnosis for an exact packaged image. + +This is a diagnostic, never training or qualification authority. It retains +the new container and writes only closed results to a fresh private directory. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import subprocess +import sys +from pathlib import Path + + +SCHEMA = "synaptic-docker-torch-import-diagnostic/v1" +_SHA = re.compile(r"sha256:[0-9a-f]{64}\Z") +_CID = re.compile(r"[0-9a-f]{64}\Z") +_CATEGORIES = frozenset({"PASS", "MODULE_NOT_FOUND", "IMPORT", "OS", "RUNTIME", "OTHER"}) +_REASONS = frozenset({"DAEMON_ID_INVALID", "INPUT_INVALID", "DOCKER_INVALID", + "DOCKER_CHANGED", "DOCKER_COMMAND_REJECTED", "DIAGNOSTIC_OUTPUT_INVALID", + "RELEASE_INVALID", "RELEASE_HASH_INVALID", "OUTPUT_INVALID", "SOURCE_INVALID", "DAEMON_INVALID", "IMAGE_INVALID", + "CHILD_OUTPUT_INVALID", "CONTAINER_ID_INVALID", "CONTAINER_IMAGE_INVALID", + "IMAGE_CHANGED"}) + +# This fixed program never formats an exception, inspects its attributes, or +# imports an engine/trainer module. The exact packaged interpreter imports +# Torch under the isolated, offline child environment. +_PROBE = """import json,os,sys +os.environ.clear() +os.environ.update({'PATH':os.path.dirname(sys.executable)+':/usr/local/bin:/usr/bin:/bin','HOME':'/tmp','TMPDIR':'/tmp','LC_ALL':'C.UTF-8','PYTHONNOUSERSITE':'1','PYTHONSAFEPATH':'1','PYTHONDONTWRITEBYTECODE':'1','HF_HUB_OFFLINE':'1','TRANSFORMERS_OFFLINE':'1','WANDB_DISABLED':'true','CUDA_VISIBLE_DEVICES':'','NVIDIA_VISIBLE_DEVICES':'void'}) +try: + import torch + category='PASS' +except ModuleNotFoundError: + category='MODULE_NOT_FOUND' +except ImportError: + category='IMPORT' +except OSError: + category='OS' +except RuntimeError: + category='RUNTIME' +except BaseException: + category='OTHER' +print(json.dumps({'schema_version':'synaptic-docker-torch-import-child/v1','category':category},sort_keys=True,separators=(',',':'))) +""" + +_GUARDED_PROBE = """import json,os,sys +from pathlib import Path +os.environ.clear() +os.environ.update({'PATH':os.path.dirname(sys.executable)+':/usr/local/bin:/usr/bin:/bin','HOME':'/tmp','TMPDIR':'/tmp','LC_ALL':'C.UTF-8','PYTHONNOUSERSITE':'1','PYTHONSAFEPATH':'1','PYTHONDONTWRITEBYTECODE':'1','HF_HUB_OFFLINE':'1','TRANSFORMERS_OFFLINE':'1','WANDB_DISABLED':'true','CUDA_VISIBLE_DEVICES':'','NVIDIA_VISIBLE_DEVICES':'void'}) +root=Path('/source/engine') +sys.path.insert(0,str(root)) +stage='IMPORT_WORKER' +try: + from tuner.runtime.offline_sft_worker import load_offline_sft_worker_closure,install_owned_module_guard,verify_loaded_owned_module_origins + stage='READ_MANIFEST' + manifest=Path('/source/control/offline-sft-worker-v1.json') + declared=json.loads(manifest.read_bytes())['closure_digest'] + stage='LOAD_CLOSURE' + closure=load_offline_sft_worker_closure(manifest,expected_digest=declared,engine_root=root) + stage='VERIFY_ORIGINS' + verify_loaded_owned_module_origins(closure,engine_root=root) + stage='INSTALL_GUARD' + install_owned_module_guard(closure,engine_root=root) + sys.path[:0]=[str(root/'Trainers/sft'),str(root)] + stage='BOOTSTRAP_ENV' + from shared.env_bootstrap import init_trainer_env + init_trainer_env() + stage='TORCH_IMPORT' + import torch + category='PASS' +except ModuleNotFoundError: + category='MODULE_NOT_FOUND' +except ImportError: + category='IMPORT' +except OSError: + category='OS' +except RuntimeError: + category='RUNTIME' +except BaseException: + category='OTHER' +print(json.dumps({'schema_version':'synaptic-docker-torch-import-child/v1','category':category,'stage':stage},sort_keys=True,separators=(',',':'))) +""" + + +def _read(path: Path, maximum: int) -> bytes: + if path.is_symlink() or not path.is_file() or path.stat().st_size > maximum: + raise ValueError("INPUT_INVALID") + with path.open("rb") as stream: + raw = stream.read(maximum + 1) + if not raw or len(raw) > maximum: + raise ValueError("INPUT_INVALID") + return raw + + +def _identity(path: Path): + value = path.stat() + return value.st_dev, value.st_ino, value.st_mode, value.st_size, value.st_mtime_ns + + +def _authority(docker: Path, docker_config: Path): + if (not docker.is_absolute() or docker.is_symlink() or not docker.is_file() + or not docker_config.is_absolute() or docker_config.is_symlink() + or not docker_config.is_dir() or any(docker_config.iterdir())): + raise ValueError("DOCKER_INVALID") + return (docker, docker_config, _identity(docker), _identity(docker_config)) + + +def _run(authority, args: tuple[str, ...], *, timeout: int = 120): + docker, config, exe_stat, config_stat = authority + if (_identity(docker) != exe_stat or _identity(config) != config_stat): + raise ValueError("DOCKER_CHANGED") + result = subprocess.run((str(docker), "--config", str(config), + "--host", "unix:///var/run/docker.sock", *args), + env={"PATH": os.defpath, "DOCKER_CONTENT_TRUST": "1"}, + stdin=subprocess.DEVNULL, capture_output=True, timeout=timeout, check=False) + if (_identity(docker) != exe_stat or _identity(config) != config_stat): + raise ValueError("DOCKER_CHANGED") + if result.returncode != 0 or len(result.stdout) > 8192 or len(result.stderr) > 8192: + raise ValueError("DOCKER_COMMAND_REJECTED") + return result + + +def _one_line(raw: bytes) -> str: + value = raw.decode("ascii") + if value != value.strip() and value != value.strip() + "\n": + raise ValueError("DIAGNOSTIC_OUTPUT_INVALID") + if not value.strip() or "\n" in value.strip(): + raise ValueError("DIAGNOSTIC_OUTPUT_INVALID") + return value.strip() + + +def diagnose(*, docker: Path, docker_config: Path, release_path: Path, + daemon_id: str, output_dir: Path, expected_release_sha256: str, + source_root: Path | None = None) -> dict[str, object]: + if not re.fullmatch(r"[0-9a-f-]{36}", daemon_id): + raise ValueError("DAEMON_ID_INVALID") + release_raw = _read(release_path, 128 * 1024) + if (not re.fullmatch(r"[0-9a-f]{64}", expected_release_sha256) + or hashlib.sha256(release_raw).hexdigest() != expected_release_sha256): + raise ValueError("RELEASE_HASH_INVALID") + release = json.loads(release_raw) + if (type(release) is not dict or + json.dumps(release, sort_keys=True, separators=(",", ":"), + ensure_ascii=True).encode("ascii") != release_raw): + raise ValueError("RELEASE_INVALID") + image = "sha256:" + release["image"]["digest"] + python = release["python"]["executable"] + digest = release["manifest_digest"] + if (not _SHA.fullmatch(image) or type(python) is not str or not python.startswith("/") + or type(digest) is not str or not re.fullmatch(r"[0-9a-f]{64}", digest) + or release["image"]["reference"].split("@")[-1] != image): + raise ValueError("RELEASE_INVALID") + if output_dir.exists() or output_dir.is_symlink() or not output_dir.parent.is_dir(): + raise ValueError("OUTPUT_INVALID") + if source_root is not None and (not source_root.is_absolute() or source_root.is_symlink() + or not source_root.is_dir() + or source_root.name != "engine" + or not (source_root.parent / "control/offline-sft-worker-v1.json").is_file()): + raise ValueError("SOURCE_INVALID") + authority = _authority(docker, docker_config) + info = _run(authority, ("info", "--format", "{{.ID}}")) + if info.stderr or _one_line(info.stdout) != daemon_id: + raise ValueError("DAEMON_INVALID") + before = _run(authority, ("image", "inspect", "--format", "{{.Id}}", image)) + if before.stderr or _one_line(before.stdout) != image: + raise ValueError("IMAGE_INVALID") + output_dir.mkdir(mode=0o700) + cidfile = output_dir / "container.id" + mount = (() if source_root is None else + ("--mount", f"type=bind,src={source_root.parent},dst=/source,readonly")) + user = (() if source_root is None else + ("--user", f"{source_root.stat().st_uid}:{source_root.stat().st_gid}")) + command = ( + "run", "--cidfile", str(cidfile), "--pull=never", "--network=none", + "--read-only", "--cap-drop=ALL", "--runtime=runc", + "--security-opt=no-new-privileges", "--pids-limit=64", "--memory=4g", + "--cpus=2", "--tmpfs", "/tmp:rw,nosuid,nodev,size=64m,mode=1777", + *mount, *user, + "--env=NVIDIA_VISIBLE_DEVICES=void", "--env=CUDA_VISIBLE_DEVICES=", + "--workdir=/tmp", "--entrypoint", python, + image, "-I", "-c", _PROBE if source_root is None else _GUARDED_PROBE, + ) + result = _run(authority, command) + if result.stderr: + raise ValueError("CHILD_OUTPUT_INVALID") + child = json.loads(_one_line(result.stdout)) + fields = {"schema_version", "category"} if source_root is None else {"schema_version", "category", "stage"} + if (type(child) is not dict or set(child) != fields + or child["schema_version"] != "synaptic-docker-torch-import-child/v1" + or child["category"] not in _CATEGORIES + or (source_root is not None and child["stage"] not in {"IMPORT_WORKER", "READ_MANIFEST", + "LOAD_CLOSURE", "VERIFY_ORIGINS", "INSTALL_GUARD", "BOOTSTRAP_ENV", "TORCH_IMPORT"})): + raise ValueError("CHILD_OUTPUT_INVALID") + cid = _one_line(_read(cidfile, 80)) + if not _CID.fullmatch(cid): + raise ValueError("CONTAINER_ID_INVALID") + inspected = _run(authority, ("container", "inspect", "--format", "{{.Image}}", cid)) + if inspected.stderr or _one_line(inspected.stdout) != image: + raise ValueError("CONTAINER_IMAGE_INVALID") + after = _run(authority, ("image", "inspect", "--format", "{{.Id}}", image)) + if after.stderr or _one_line(after.stdout) != image: + raise ValueError("IMAGE_CHANGED") + report = {"schema_version": SCHEMA, "category": child["category"], + "container_id": cid, "image_id": image, "daemon_id": daemon_id, + "runtime_release_digest": digest, + "training_executed": False, "gpu_used": False, + "scope": "bare" if source_root is None else "guarded"} + if source_root is not None: + report["stage"] = child["stage"] + with (output_dir / "result.json").open("x", encoding="ascii") as stream: + json.dump(report, stream, sort_keys=True, separators=(",", ":")) + stream.write("\n") + return report + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + for name in ("docker", "docker-config", "release", "output-dir"): + parser.add_argument("--" + name, required=True, type=Path) + parser.add_argument("--daemon-id", required=True) + parser.add_argument("--expected-release-sha256", required=True) + parser.add_argument("--source-root", type=Path) + args = parser.parse_args(argv) + try: + report = diagnose(docker=args.docker, docker_config=args.docker_config, + release_path=args.release, daemon_id=args.daemon_id, + output_dir=args.output_dir, + expected_release_sha256=args.expected_release_sha256, + source_root=args.source_root) + except BaseException as error: + reason = error.args[0] if type(error) is ValueError and len(error.args) == 1 else None + if type(reason) is not str or reason not in _REASONS: + reason = "OTHER" + print(json.dumps({"schema_version": SCHEMA, "status": "REJECTED", + "reason_code": reason}, sort_keys=True, separators=(",", ":")), file=sys.stderr) + return 2 + print(json.dumps(report, sort_keys=True, separators=(",", ":"))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/scripts/test_diagnose_docker_torch_import.py b/tests/scripts/test_diagnose_docker_torch_import.py new file mode 100644 index 00000000..862575cb --- /dev/null +++ b/tests/scripts/test_diagnose_docker_torch_import.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import json +import hashlib +from pathlib import Path + +import pytest + +from scripts import diagnose_docker_torch_import as diagnostic + + +def test_closed_single_line_parser_rejects_extra_output() -> None: + assert diagnostic._one_line(b"abc\n") == "abc" + assert diagnostic._one_line(b"abc") == "abc" + for raw in (b"abc\nsecret\n", b" abc\n", b"\n", b"abc\r\n"): + with pytest.raises(ValueError): + diagnostic._one_line(raw) + + +def test_exact_image_import_probe_is_cpu_only_and_retains_container( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + image = "sha256:" + "a" * 64 + cid = "b" * 64 + daemon = "24805003-b125-469b-8080-9da4daacd364" + release = {"image": {"digest": "a" * 64, "reference": "example/image@" + image}, + "python": {"executable": "/opt/unsloth-venv/bin/python3"}, + "manifest_digest": "c" * 64} + (tmp_path / "release.json").write_text(json.dumps(release, sort_keys=True, separators=(",", ":"))) + release_sha = hashlib.sha256((tmp_path / "release.json").read_bytes()).hexdigest() + monkeypatch.setattr(diagnostic, "_authority", lambda *_: object()) + commands = [] + + def run(_authority, args, *, timeout=120): + commands.append(args) + if args[:2] == ("info", "--format"): + return diagnostic.subprocess.CompletedProcess(args, 0, (daemon + "\n").encode(), b"") + if args[:2] == ("image", "inspect"): + return diagnostic.subprocess.CompletedProcess(args, 0, (image + "\n").encode(), b"") + if args[:2] == ("container", "inspect"): + return diagnostic.subprocess.CompletedProcess(args, 0, (image + "\n").encode(), b"") + assert args[0] == "run" + assert "--pull=never" in args and "--network=none" in args + assert "--read-only" in args and "--cap-drop=ALL" in args + assert "--memory=4g" in args and "--cpus=2" in args + assert "--env=NVIDIA_VISIBLE_DEVICES=void" in args + assert "--env=CUDA_VISIBLE_DEVICES=" in args + assert "--volume" not in args and "--gpus" not in args + assert "-I" in args and "-c" in args + guarded = "--mount" in args + if guarded: + assert "--user" in args + assert "readonly" in args[args.index("--mount") + 1] + target = tmp_path / "guarded" + payload = b'{"category":"PASS","schema_version":"synaptic-docker-torch-import-child/v1","stage":"TORCH_IMPORT"}\n' + else: + target = out + payload = b'{"category":"IMPORT","schema_version":"synaptic-docker-torch-import-child/v1"}\n' + (target / "container.id").write_text(cid + "\n") + return diagnostic.subprocess.CompletedProcess(args, 0, payload, b"") + + monkeypatch.setattr(diagnostic, "_run", run) + out = tmp_path / "fresh" + report = diagnostic.diagnose(docker=Path("/usr/bin/docker"), + docker_config=Path("/private/config"), release_path=tmp_path / "release.json", + daemon_id=daemon, output_dir=out, expected_release_sha256=release_sha) + assert report["category"] == "IMPORT" + assert report["container_id"] == cid + assert json.loads((out / "result.json").read_text()) == report + assert len(commands) == 5 + source = tmp_path / "source" / "engine" + source.mkdir(parents=True) + (source.parent / "control").mkdir() + (source.parent / "control" / "offline-sft-worker-v1.json").write_text("{}") + guarded = diagnostic.diagnose(docker=Path("/usr/bin/docker"), + docker_config=Path("/private/config"), release_path=tmp_path / "release.json", + daemon_id=daemon, output_dir=tmp_path / "guarded", + expected_release_sha256=release_sha, source_root=source) + assert guarded["category"] == "PASS" and guarded["stage"] == "TORCH_IMPORT" + assert guarded["scope"] == "guarded" + with pytest.raises(ValueError, match="OUTPUT_INVALID"): + diagnostic.diagnose(docker=Path("/usr/bin/docker"), + docker_config=Path("/private/config"), release_path=tmp_path / "release.json", + daemon_id=daemon, output_dir=out, expected_release_sha256=release_sha) + + +def test_cli_rejection_never_prints_exception_text(monkeypatch, capsys) -> None: + def fail(**_kwargs): + raise RuntimeError("private exception path /secret and credential") + + monkeypatch.setattr(diagnostic, "diagnose", fail) + code = diagnostic.main(["--docker", "/usr/bin/docker", "--docker-config", "/private/config", + "--release", "/private/release.json", "--output-dir", "/private/output", + "--daemon-id", "24805003-b125-469b-8080-9da4daacd364", + "--expected-release-sha256", "a" * 64]) + assert code == 2 + output = capsys.readouterr() + assert output.out == "" + assert json.loads(output.err) == {"schema_version": diagnostic.SCHEMA, + "status": "REJECTED", "reason_code": "OTHER"} + assert "private" not in output.err and "Traceback" not in output.err From 0f566adb02fe64ff3eb9a7aa307320d9f2328088 Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:34:49 -0400 Subject: [PATCH 02/12] Bound Docker Torch import diagnosis and authenticate r2 source --- .../fine-tuning/reference/sft-training.md | 45 ++- .../fine-tuning/reference/sft-training.md | 45 ++- .skills/fine-tuning/reference/sft-training.md | 45 ++- .../docker-torch-import-r2-diagnosis.md | 92 ++++- scripts/diagnose_docker_torch_import.py | 334 ++++++++++++++++-- .../test_diagnose_docker_torch_import.py | 323 +++++++++++++---- 6 files changed, 715 insertions(+), 169 deletions(-) diff --git a/.agents/skills/fine-tuning/reference/sft-training.md b/.agents/skills/fine-tuning/reference/sft-training.md index 18e6af1e..36543ded 100644 --- a/.agents/skills/fine-tuning/reference/sft-training.md +++ b/.agents/skills/fine-tuning/reference/sft-training.md @@ -492,24 +492,33 @@ the consumed claim, roots, container and retained logs; the diagnostics never authorize replay. See `docs/review/offline-sft-worker-diagnostics.md` for the October 20 evidence boundary, CPU regressions and fresh-run prerequisites. -For a fresh, credential-free Docker Torch import reproduction against an exact -retained packaged release, use the checked-in -`scripts/diagnose_docker_torch_import.py` with `--docker`, `--docker-config`, -`--release`, the exact `--expected-release-sha256`, `--daemon-id`, and a -nonexistent private `--output-dir`. It -requires the release's image ID to be present locally, starts one retained -read-only container with at most 2 CPUs, 4 GiB, 120 seconds, no network or GPU, -and no model mount. The bare mode has no source mount. Its result and stdout contain only closed import -categories and identities. Keep the new container and `container.id`; do not -replay the original training claim. `PASS` only says Torch imports in that -reduced environment; a rejection class is a reproduction boundary, not proof -of a dependency or repair. Review the pinned package/runtime source before any -change to profile, image, or runtime policy. -When the bare probe passes, an optional `--source-root` binds the exact retained -staged engine read-only and repeats Torch import after the offline worker's -closure validation, owned-module guard, and trainer environment bootstrap. -Use a fresh output directory for each probe. The guarded result adds the last -closed stage; it still runs no trainer or model code. +For a fresh, credential-free Docker Torch import reproduction, review the exact +command before effects. The checked-in `scripts/diagnose_docker_torch_import.py` +requires `--docker`, an empty `--docker-config`, `--release`, independently +verified `--expected-release-sha256`, `--daemon-id`, and a nonexistent private +`--output-dir`. Guarded mode additionally requires `--source-root`, +`--source-lock`, independently reviewed `--expected-source-lock-sha256`, +`--expected-manifest-sha256`, and `--expected-closure-digest` from the retained +attempt. It validates every staged worker member before executing mounted code; +the result records those commitments. Run from the reviewed checkout so its +trusted closure verifier is the one checked in with the helper. + +The helper claims a unique name before Docker create, authenticates the exact +new container before start, and retains it. Docker CLI output is capped at 8 KiB +in memory; the container requests JSON log rotation at 1 MiB with one file. +The import execution has a +120-second host deadline. At expiry the helper issues at most one stop request +to the authenticated new container and separately inspects terminal state; +cleanup takes additional bounded time, and a stop acknowledgment alone is not +quiescence. An uncertain create/start/stop remains consumed in `claim.json` and +`events.jsonl`, with no automatic replay or container removal. The probe asks +for at most 2 CPUs, 4 GiB, no network/GPU/model mount, and a read-only root. +Bare mode mounts no source. Guarded mode mounts the independently authenticated +retained source and runs the worker closure check, module guard, trainer path +setup, environment bootstrap and Torch import. Neither runs training. `PASS` +only says Torch imports in the reduced probe; it does not diagnose an earlier +failure. Review the pinned package/runtime source before changing a profile, +image or runtime policy. The exact r2 CPU evidence and its unresolved runtime difference are in `docs/review/docker-torch-import-r2-diagnosis.md`. diff --git a/.claude/skills/fine-tuning/reference/sft-training.md b/.claude/skills/fine-tuning/reference/sft-training.md index 18e6af1e..36543ded 100644 --- a/.claude/skills/fine-tuning/reference/sft-training.md +++ b/.claude/skills/fine-tuning/reference/sft-training.md @@ -492,24 +492,33 @@ the consumed claim, roots, container and retained logs; the diagnostics never authorize replay. See `docs/review/offline-sft-worker-diagnostics.md` for the October 20 evidence boundary, CPU regressions and fresh-run prerequisites. -For a fresh, credential-free Docker Torch import reproduction against an exact -retained packaged release, use the checked-in -`scripts/diagnose_docker_torch_import.py` with `--docker`, `--docker-config`, -`--release`, the exact `--expected-release-sha256`, `--daemon-id`, and a -nonexistent private `--output-dir`. It -requires the release's image ID to be present locally, starts one retained -read-only container with at most 2 CPUs, 4 GiB, 120 seconds, no network or GPU, -and no model mount. The bare mode has no source mount. Its result and stdout contain only closed import -categories and identities. Keep the new container and `container.id`; do not -replay the original training claim. `PASS` only says Torch imports in that -reduced environment; a rejection class is a reproduction boundary, not proof -of a dependency or repair. Review the pinned package/runtime source before any -change to profile, image, or runtime policy. -When the bare probe passes, an optional `--source-root` binds the exact retained -staged engine read-only and repeats Torch import after the offline worker's -closure validation, owned-module guard, and trainer environment bootstrap. -Use a fresh output directory for each probe. The guarded result adds the last -closed stage; it still runs no trainer or model code. +For a fresh, credential-free Docker Torch import reproduction, review the exact +command before effects. The checked-in `scripts/diagnose_docker_torch_import.py` +requires `--docker`, an empty `--docker-config`, `--release`, independently +verified `--expected-release-sha256`, `--daemon-id`, and a nonexistent private +`--output-dir`. Guarded mode additionally requires `--source-root`, +`--source-lock`, independently reviewed `--expected-source-lock-sha256`, +`--expected-manifest-sha256`, and `--expected-closure-digest` from the retained +attempt. It validates every staged worker member before executing mounted code; +the result records those commitments. Run from the reviewed checkout so its +trusted closure verifier is the one checked in with the helper. + +The helper claims a unique name before Docker create, authenticates the exact +new container before start, and retains it. Docker CLI output is capped at 8 KiB +in memory; the container requests JSON log rotation at 1 MiB with one file. +The import execution has a +120-second host deadline. At expiry the helper issues at most one stop request +to the authenticated new container and separately inspects terminal state; +cleanup takes additional bounded time, and a stop acknowledgment alone is not +quiescence. An uncertain create/start/stop remains consumed in `claim.json` and +`events.jsonl`, with no automatic replay or container removal. The probe asks +for at most 2 CPUs, 4 GiB, no network/GPU/model mount, and a read-only root. +Bare mode mounts no source. Guarded mode mounts the independently authenticated +retained source and runs the worker closure check, module guard, trainer path +setup, environment bootstrap and Torch import. Neither runs training. `PASS` +only says Torch imports in the reduced probe; it does not diagnose an earlier +failure. Review the pinned package/runtime source before changing a profile, +image or runtime policy. The exact r2 CPU evidence and its unresolved runtime difference are in `docs/review/docker-torch-import-r2-diagnosis.md`. diff --git a/.skills/fine-tuning/reference/sft-training.md b/.skills/fine-tuning/reference/sft-training.md index 18e6af1e..36543ded 100644 --- a/.skills/fine-tuning/reference/sft-training.md +++ b/.skills/fine-tuning/reference/sft-training.md @@ -492,24 +492,33 @@ the consumed claim, roots, container and retained logs; the diagnostics never authorize replay. See `docs/review/offline-sft-worker-diagnostics.md` for the October 20 evidence boundary, CPU regressions and fresh-run prerequisites. -For a fresh, credential-free Docker Torch import reproduction against an exact -retained packaged release, use the checked-in -`scripts/diagnose_docker_torch_import.py` with `--docker`, `--docker-config`, -`--release`, the exact `--expected-release-sha256`, `--daemon-id`, and a -nonexistent private `--output-dir`. It -requires the release's image ID to be present locally, starts one retained -read-only container with at most 2 CPUs, 4 GiB, 120 seconds, no network or GPU, -and no model mount. The bare mode has no source mount. Its result and stdout contain only closed import -categories and identities. Keep the new container and `container.id`; do not -replay the original training claim. `PASS` only says Torch imports in that -reduced environment; a rejection class is a reproduction boundary, not proof -of a dependency or repair. Review the pinned package/runtime source before any -change to profile, image, or runtime policy. -When the bare probe passes, an optional `--source-root` binds the exact retained -staged engine read-only and repeats Torch import after the offline worker's -closure validation, owned-module guard, and trainer environment bootstrap. -Use a fresh output directory for each probe. The guarded result adds the last -closed stage; it still runs no trainer or model code. +For a fresh, credential-free Docker Torch import reproduction, review the exact +command before effects. The checked-in `scripts/diagnose_docker_torch_import.py` +requires `--docker`, an empty `--docker-config`, `--release`, independently +verified `--expected-release-sha256`, `--daemon-id`, and a nonexistent private +`--output-dir`. Guarded mode additionally requires `--source-root`, +`--source-lock`, independently reviewed `--expected-source-lock-sha256`, +`--expected-manifest-sha256`, and `--expected-closure-digest` from the retained +attempt. It validates every staged worker member before executing mounted code; +the result records those commitments. Run from the reviewed checkout so its +trusted closure verifier is the one checked in with the helper. + +The helper claims a unique name before Docker create, authenticates the exact +new container before start, and retains it. Docker CLI output is capped at 8 KiB +in memory; the container requests JSON log rotation at 1 MiB with one file. +The import execution has a +120-second host deadline. At expiry the helper issues at most one stop request +to the authenticated new container and separately inspects terminal state; +cleanup takes additional bounded time, and a stop acknowledgment alone is not +quiescence. An uncertain create/start/stop remains consumed in `claim.json` and +`events.jsonl`, with no automatic replay or container removal. The probe asks +for at most 2 CPUs, 4 GiB, no network/GPU/model mount, and a read-only root. +Bare mode mounts no source. Guarded mode mounts the independently authenticated +retained source and runs the worker closure check, module guard, trainer path +setup, environment bootstrap and Torch import. Neither runs training. `PASS` +only says Torch imports in the reduced probe; it does not diagnose an earlier +failure. Review the pinned package/runtime source before changing a profile, +image or runtime policy. The exact r2 CPU evidence and its unresolved runtime difference are in `docs/review/docker-torch-import-r2-diagnosis.md`. diff --git a/docs/review/docker-torch-import-r2-diagnosis.md b/docs/review/docker-torch-import-r2-diagnosis.md index 1a4836d5..d13fd8e5 100644 --- a/docs/review/docker-torch-import-r2-diagnosis.md +++ b/docs/review/docker-torch-import-r2-diagnosis.md @@ -24,16 +24,19 @@ The retained final runtime capture records CPython 3.12.3 and Torch ## Targeted CPU experiments -The checked-in `scripts/diagnose_docker_torch_import.py` binds that release -file hash, daemon ID and exact local image ID. It starts a fresh retained -container with at most 2 CPUs, 4 GiB and 120 seconds; network and GPU devices -are disabled. It has no credentials, model mount or training entrypoint, and -emits only closed categories and identity fields. +The r4 version of `scripts/diagnose_docker_torch_import.py` bound that release +file hash, daemon ID and exact local image ID. It requested at most 2 CPUs, +4 GiB, no network or GPU devices, and had no credentials, model mount or +training entrypoint. Its `subprocess.run(timeout=120)` bounded the Docker CLI +call but **did not enforce a 120-second container lifetime**. Its +`capture_output=True` also buffered CLI output without a preallocation limit. +The earlier 120-second container claim was incorrect. The retained r4 results +are observations, not proof of a bounded lifecycle. The bare image probe returned `PASS` in container `33af62c4fdda5d855b3b154c2937912b5266a88a48f2c0757b3664fe42a45917`. -The final guarded probe mounted the r2 staged source read-only, validated the -offline worker closure, installed its owned-module guard, ran +The final guarded probe mounted the r2 staged source read-only, validated a +manifest against the digest in that same mount, installed its owned-module guard, ran `init_trainer_env()`, and imported Torch successfully. Its retained container is `2e5b6d06a4cc946541a475da72b7aa7e851b5840f86fd675c2099540b6fc1045`; `evidence-guarded-final2/result.json` has SHA-256 @@ -42,6 +45,9 @@ The exact final checked-in helper source has SHA-256 `c214fc8b9d617fe7171e7eac987d93187169b8a637fac9ef24b4323a70bf6c53`. These artifacts are under the private Windows WSL root `/home/profsynapse/.local/share/synaptic-qualification/torch-import-r4-20261020/`. +The r4 helper had no independently admitted source or closure commitment, so +this `PASS` cannot establish that it executed the exact r2 closure. The result +remains historical and is not relabeled after the helper repair. Earlier guarded diagnostic containers returned `READ_MANIFEST/OS` because the probe dropped all capabilities while running as root against mode-0600 @@ -59,6 +65,22 @@ runtime. The cause of r2's Torch import failure remains unresolved. In particular, the evidence does not support changing packages, image, recipe, model, GPU settings or SFT runtime code yet. +Read-only inspection of the retained r2 container on 2026-10-20 confirmed +entrypoint `/opt/unsloth-venv/bin/python3`, command prefix +`/source/engine/Trainers/sft/runtime_v1.py --canonical-workload-file +/source/control/workload.json`, user `root`, workdir `/workspace`, runtime +`runc`, writable root, default capabilities, network `none`, and Docker GPU +device request `0`. The parent runtime creates a separate offline worker with +an explicit child environment (`Trainers/sft/runtime_v1.py:1480-1497`) and +executes the trainer through `runpy.run_path`; the trainer prepends its `src` +and engine roots to `sys.path` immediately before `init_trainer_env()` and +`import torch` (`Trainers/sft/train_sft.py:217-227`). The r4 guarded probe +cleared/replaced its environment, disabled CUDA/NVIDIA visibility, dropped +capabilities, used the staged-source owner, and skipped the parent runtime. +Its path setup also lacked the trainer's exact `src` insertion. The repaired +guarded probe mirrors that path order, but it still does not reproduce the +parent's environment, user, capabilities, writable root or GPU request. + The r2 claim, container and all older failure evidence remain consumed and preserved. No training start, image build, pull, model load, package install or HF action occurred during this diagnosis. A further comparison of the @@ -66,3 +88,59 @@ original GPU/runtime environment needs a separately reviewed exact scope; this diagnostic does not authorize another training attempt. No runtime closure or Modal lock refresh is needed for the diagnostic script and skill changes, because no locked runtime member changed. + +## Repaired helper and next review boundary + +The successor helper claims a fresh unique diagnostic name before create, +caps aggregate Docker CLI output at 8 KiB, requests JSON log rotation at +1 MiB with one file, and +separates create, authenticated start, terminal inspection and one owned stop +request. Its 120 seconds are an execution deadline followed by a bounded +cleanup and inspection tail; a Docker stop response alone never proves physical +quiescence. If create/start/stop or identity is uncertain, the claim and events +remain and no automatic replay occurs. For guarded mode, independently supplied +r2 source-lock SHA-256 `d19aecb2d3ca1b7b6fa32a2445c45251e59d8cac2332293c68b73a61cbff4665`, +manifest SHA-256 `3842689403abc5b42e28d58545b88616b81cd8fc738ee5557e97159df65d00ef`, +and closure digest `8465ff2b7cedb352337e7cedc5bb17cc169892c2eedae0b280569f4a0cde5002` +are checked before mounted code executes and included in the result. These +values were read from the retained r2 material on 2026-10-20; live rechecking +before any new effect remains required. No successor container has been +launched. Review the exact command in this branch before a fresh CPU import +diagnostic; a GPU-exposed probe needs separate scope review. + +Proposed guarded CPU command for root review **only**. Run from a clean WSL +checkout of this successor commit after confirming that its helper and trusted +closure verifier match the reviewed source. First create the parent +`/home/profsynapse/.local/share/synaptic-qualification/torch-import-r5-20261020` +as a private mode-0700 directory; the `guarded-cpu-1` child must not exist. + +```sh +python3 -I scripts/diagnose_docker_torch_import.py \ + --docker /usr/bin/docker \ + --docker-config /home/profsynapse/.local/share/synaptic-qualification/docker-sft-stable-image-c/docker-config \ + --release /home/profsynapse/.local/share/synaptic-qualification/docker-sft-stable-image-c/evidence/runtime-release.json \ + --expected-release-sha256 5f65824ceb5e250a58314da0233b9c07f1d63f1b0a39892d9b36af694938bfe6 \ + --daemon-id 24805003-b125-469b-8080-9da4daacd364 \ + --source-root /home/profsynapse/.local/share/synaptic-qualification/docker-public-plan-0d515-20261020/attempt-r2/source/engine \ + --source-lock /home/profsynapse/.local/share/synaptic-qualification/docker-public-plan-0d515-20261020/attempt-r2/material/source-lock.json \ + --expected-source-lock-sha256 d19aecb2d3ca1b7b6fa32a2445c45251e59d8cac2332293c68b73a61cbff4665 \ + --expected-manifest-sha256 3842689403abc5b42e28d58545b88616b81cd8fc738ee5557e97159df65d00ef \ + --expected-closure-digest 8465ff2b7cedb352337e7cedc5bb17cc169892c2eedae0b280569f4a0cde5002 \ + --output-dir /home/profsynapse/.local/share/synaptic-qualification/torch-import-r5-20261020/guarded-cpu-1 +``` + +This repeats the reduced CPU import boundary with repaired ownership and +source binding. It is **not** an exact r2 invocation, does not use GPU, and +does not authorize another SFT start. The original r2 claim remains consumed. + +The next comparison should hold the independently authenticated r2 source, +daemon and image fixed, then compare the offline child environment produced by +`runtime_v1.py` with the CPU diagnostic at the Torch import boundary. In +particular compare Python executable/`-I` invocation, worker closure and +module guard, final `sys.path`, `PATH`, `LD_LIBRARY_PATH`, `HF_HOME`, +`TRANSFORMERS_CACHE`, `TMPDIR`, and CUDA/NVIDIA visibility by key and approved +value or digest without printing secrets. Vary one of user/capabilities, +writable root and GPU visibility at a time in import-only containers under +reviewed bounds. A GPU-visible import-only experiment is a separate reviewed +scope; no trainer, model load or training is proposed. Treat any changed result +as evidence for a narrower hypothesis, not retrospective proof of r2's cause. diff --git a/scripts/diagnose_docker_torch_import.py b/scripts/diagnose_docker_torch_import.py index 1065d548..f8ed0ca1 100644 --- a/scripts/diagnose_docker_torch_import.py +++ b/scripts/diagnose_docker_torch_import.py @@ -11,8 +11,13 @@ import json import os import re +import secrets +import selectors +import signal +import stat import subprocess import sys +import time from pathlib import Path @@ -24,12 +29,19 @@ "DOCKER_CHANGED", "DOCKER_COMMAND_REJECTED", "DIAGNOSTIC_OUTPUT_INVALID", "RELEASE_INVALID", "RELEASE_HASH_INVALID", "OUTPUT_INVALID", "SOURCE_INVALID", "DAEMON_INVALID", "IMAGE_INVALID", "CHILD_OUTPUT_INVALID", "CONTAINER_ID_INVALID", "CONTAINER_IMAGE_INVALID", - "IMAGE_CHANGED"}) + "IMAGE_CHANGED", "SOURCE_IDENTITY_INVALID", "CREATE_UNCERTAIN", "START_UNCERTAIN", + "DEADLINE_UNCERTAIN", "DEADLINE_STOPPED", "STOP_UNCONFIRMED", "CONTAINER_STATE_INVALID"}) +_CLI_BUDGET = 8192 +_RUN_SECONDS = 120 +_INSPECT = "{{json .Id}}\n{{json .Name}}\n{{json .Image}}\n{{json .Config.Labels}}\n{{json .State.Status}}\n{{json .State.ExitCode}}" # This fixed program never formats an exception, inspects its attributes, or # imports an engine/trainer module. The exact packaged interpreter imports # Torch under the isolated, offline child environment. _PROBE = """import json,os,sys +report_fd=os.dup(1) +null_fd=os.open('/dev/null',os.O_WRONLY) +os.dup2(null_fd,1);os.dup2(null_fd,2);os.close(null_fd) os.environ.clear() os.environ.update({'PATH':os.path.dirname(sys.executable)+':/usr/local/bin:/usr/bin:/bin','HOME':'/tmp','TMPDIR':'/tmp','LC_ALL':'C.UTF-8','PYTHONNOUSERSITE':'1','PYTHONSAFEPATH':'1','PYTHONDONTWRITEBYTECODE':'1','HF_HUB_OFFLINE':'1','TRANSFORMERS_OFFLINE':'1','WANDB_DISABLED':'true','CUDA_VISIBLE_DEVICES':'','NVIDIA_VISIBLE_DEVICES':'void'}) try: @@ -45,10 +57,14 @@ category='RUNTIME' except BaseException: category='OTHER' -print(json.dumps({'schema_version':'synaptic-docker-torch-import-child/v1','category':category},sort_keys=True,separators=(',',':'))) +os.write(report_fd,(json.dumps({'schema_version':'synaptic-docker-torch-import-child/v1','category':category},sort_keys=True,separators=(',',':'))+'\\n').encode('ascii')) +os.close(report_fd) """ _GUARDED_PROBE = """import json,os,sys +report_fd=os.dup(1) +null_fd=os.open('/dev/null',os.O_WRONLY) +os.dup2(null_fd,1);os.dup2(null_fd,2);os.close(null_fd) from pathlib import Path os.environ.clear() os.environ.update({'PATH':os.path.dirname(sys.executable)+':/usr/local/bin:/usr/bin:/bin','HOME':'/tmp','TMPDIR':'/tmp','LC_ALL':'C.UTF-8','PYTHONNOUSERSITE':'1','PYTHONSAFEPATH':'1','PYTHONDONTWRITEBYTECODE':'1','HF_HUB_OFFLINE':'1','TRANSFORMERS_OFFLINE':'1','WANDB_DISABLED':'true','CUDA_VISIBLE_DEVICES':'','NVIDIA_VISIBLE_DEVICES':'void'}) @@ -59,14 +75,14 @@ from tuner.runtime.offline_sft_worker import load_offline_sft_worker_closure,install_owned_module_guard,verify_loaded_owned_module_origins stage='READ_MANIFEST' manifest=Path('/source/control/offline-sft-worker-v1.json') - declared=json.loads(manifest.read_bytes())['closure_digest'] + declared='EXPECTED_CLOSURE_DIGEST' stage='LOAD_CLOSURE' closure=load_offline_sft_worker_closure(manifest,expected_digest=declared,engine_root=root) stage='VERIFY_ORIGINS' verify_loaded_owned_module_origins(closure,engine_root=root) stage='INSTALL_GUARD' install_owned_module_guard(closure,engine_root=root) - sys.path[:0]=[str(root/'Trainers/sft'),str(root)] + sys.path[:0]=[str(root),str(root/'Trainers/sft/src'),str(root/'Trainers/sft'),str(root)] stage='BOOTSTRAP_ENV' from shared.env_bootstrap import init_trainer_env init_trainer_env() @@ -83,7 +99,8 @@ category='RUNTIME' except BaseException: category='OTHER' -print(json.dumps({'schema_version':'synaptic-docker-torch-import-child/v1','category':category,'stage':stage},sort_keys=True,separators=(',',':'))) +os.write(report_fd,(json.dumps({'schema_version':'synaptic-docker-torch-import-child/v1','category':category,'stage':stage},sort_keys=True,separators=(',',':'))+'\\n').encode('ascii')) +os.close(report_fd) """ @@ -110,19 +127,53 @@ def _authority(docker: Path, docker_config: Path): return (docker, docker_config, _identity(docker), _identity(docker_config)) -def _run(authority, args: tuple[str, ...], *, timeout: int = 120): +def _run(authority, args: tuple[str, ...], *, timeout: float = 15): + """Bound both CLI lifetime and combined bytes before they enter memory.""" docker, config, exe_stat, config_stat = authority if (_identity(docker) != exe_stat or _identity(config) != config_stat): raise ValueError("DOCKER_CHANGED") - result = subprocess.run((str(docker), "--config", str(config), + process = subprocess.Popen((str(docker), "--config", str(config), "--host", "unix:///var/run/docker.sock", *args), env={"PATH": os.defpath, "DOCKER_CONTENT_TRUST": "1"}, - stdin=subprocess.DEVNULL, capture_output=True, timeout=timeout, check=False) - if (_identity(docker) != exe_stat or _identity(config) != config_stat): - raise ValueError("DOCKER_CHANGED") - if result.returncode != 0 or len(result.stdout) > 8192 or len(result.stderr) > 8192: - raise ValueError("DOCKER_COMMAND_REJECTED") - return result + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + start_new_session=True) + output = {process.stdout: bytearray(), process.stderr: bytearray()} + deadline = time.monotonic() + timeout + try: + with selectors.DefaultSelector() as selector: + for stream in output: + os.set_blocking(stream.fileno(), False) + selector.register(stream, selectors.EVENT_READ) + while selector.get_map() or process.poll() is None: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise ValueError("DOCKER_COMMAND_REJECTED") + for key, _ in selector.select(min(remaining, 0.1)): + chunk = os.read(key.fileobj.fileno(), _CLI_BUDGET + 1) + if not chunk: + selector.unregister(key.fileobj) + elif sum(map(len, output.values())) + len(chunk) > _CLI_BUDGET: + raise ValueError("DOCKER_COMMAND_REJECTED") + else: + output[key.fileobj].extend(chunk) + if process.wait(timeout=max(0.01, deadline - time.monotonic())) != 0: + raise ValueError("DOCKER_COMMAND_REJECTED") + if (_identity(docker) != exe_stat or _identity(config) != config_stat): + raise ValueError("DOCKER_CHANGED") + return subprocess.CompletedProcess(args, 0, bytes(output[process.stdout]), + bytes(output[process.stderr])) + finally: + if process.poll() is None: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + pass + for stream in output: + stream.close() def _one_line(raw: bytes) -> str: @@ -134,9 +185,120 @@ def _one_line(raw: bytes) -> str: return value.strip() +def _append(output_dir: Path, event: dict[str, object]) -> None: + with (output_dir / "events.jsonl").open("ab") as stream: + stream.write(json.dumps(event, sort_keys=True, separators=(",", ":")).encode("ascii") + b"\n") + stream.flush() + os.fsync(stream.fileno()) + + +def _source_identity(source_root: Path, source_lock: Path, expected_lock: str, + expected_manifest: str, expected_closure: str) -> dict[str, str]: + if (not source_root.is_absolute() or source_root.name != "engine" + or source_root.is_symlink() or not source_root.is_dir() + or not source_lock.is_absolute() or not re.fullmatch(r"[0-9a-f]{64}", expected_lock) + or not re.fullmatch(r"[0-9a-f]{64}", expected_manifest) + or not re.fullmatch(r"[0-9a-f]{64}", expected_closure)): + raise ValueError("SOURCE_IDENTITY_INVALID") + lock_raw = _read(source_lock, 128 * 1024) + manifest = source_root.parent / "control/offline-sft-worker-v1.json" + manifest_raw = _read(manifest, 128 * 1024) + if (hashlib.sha256(lock_raw).hexdigest() != expected_lock + or hashlib.sha256(manifest_raw).hexdigest() != expected_manifest): + raise ValueError("SOURCE_IDENTITY_INVALID") + lock = json.loads(lock_raw) + if (lock.get("schema_version") != "synaptic-source-lock/v1" + or lock.get("sources", {}).get("engine", {}).get("commit") != + "0d515ae1ae235d0858802228ce4af8dba8fccd03"): + raise ValueError("SOURCE_IDENTITY_INVALID") + # This trusted, checked-in verifier hashes every mounted worker member. + # It is imported from this checkout, never from the staged source. + checkout = Path(__file__).resolve().parents[1] + sys.path.insert(0, str(checkout)) + from tuner.runtime import offline_sft_worker + if Path(offline_sft_worker.__file__).resolve() != ( + checkout / "tuner/runtime/offline_sft_worker.py").resolve(): + raise ValueError("SOURCE_IDENTITY_INVALID") + try: + offline_sft_worker.load_offline_sft_worker_closure( + manifest, expected_digest=expected_closure, engine_root=source_root) + except Exception: + raise ValueError("SOURCE_IDENTITY_INVALID") from None + return {"source_lock_sha256": expected_lock, "manifest_sha256": expected_manifest, + "closure_digest": expected_closure, + "engine_commit": "0d515ae1ae235d0858802228ce4af8dba8fccd03"} + + +def _path_custody(path: Path) -> tuple[tuple[int, int, int], ...]: + if not path.is_absolute(): + raise ValueError("SOURCE_IDENTITY_INVALID") + observed = [] + for member in (path, *path.parents): + info = member.lstat() + if stat.S_ISLNK(info.st_mode): + raise ValueError("SOURCE_IDENTITY_INVALID") + observed.append((info.st_dev, info.st_ino, info.st_mode)) + return tuple(observed) + + +def _require_daemon(authority, expected: str, *, timeout: float = 5) -> None: + result = _run(authority, ("info", "--format", "{{.ID}}"), timeout=timeout) + if result.stderr or _one_line(result.stdout) != expected: + raise ValueError("DAEMON_INVALID") + + +def _inspect_owned(authority, identifier: str, *, name: str, label: str, image: str, + daemon_id: str | None = None, timeout: float = 15): + if not _CID.fullmatch(identifier): + raise ValueError("CONTAINER_ID_INVALID") + deadline = time.monotonic() + timeout + if daemon_id is not None: + _require_daemon(authority, daemon_id, timeout=max(0.05, min(5, timeout / 2))) + result = _run(authority, ("container", "inspect", "--format", _INSPECT, identifier), + timeout=max(0.05, deadline - time.monotonic())) + if result.stderr: + raise ValueError("CONTAINER_STATE_INVALID") + rows = result.stdout.decode("ascii").splitlines() + if len(rows) != 6: + raise ValueError("CONTAINER_STATE_INVALID") + cid, actual_name, actual_image, labels, status, code = map(json.loads, rows) + if (cid != identifier or actual_name != "/" + name or actual_image != image + or type(labels) is not dict or labels.get("ai.synapticlabs.torch-import.claim") != label + or status not in {"created", "running", "exited", "dead"} or type(code) is not int): + raise ValueError("CONTAINER_STATE_INVALID") + return status, code + + +def _stop_once(authority, identifier: str, *, name: str, label: str, + image: str, daemon_id: str, output_dir: Path) -> bool: + """One stop request; only a separate exact inspection can prove terminal state.""" + try: + status, _ = _inspect_owned(authority, identifier, name=name, label=label, + image=image, daemon_id=daemon_id) + if status in {"exited", "dead"}: + _append(output_dir, {"phase": "TERMINAL_OBSERVED", "container_id": identifier}) + return True + if status != "running": + return False + _append(output_dir, {"phase": "STOP_CLAIMED", "container_id": identifier}) + _run(authority, ("container", "stop", "--timeout", "2", identifier), timeout=10) + status, _ = _inspect_owned(authority, identifier, name=name, label=label, + image=image, daemon_id=daemon_id) + if status in {"exited", "dead"}: + _append(output_dir, {"phase": "TERMINAL_OBSERVED", "container_id": identifier}) + return True + except Exception: + pass + _append(output_dir, {"phase": "STOP_UNCONFIRMED", "container_id": identifier}) + return False + + def diagnose(*, docker: Path, docker_config: Path, release_path: Path, daemon_id: str, output_dir: Path, expected_release_sha256: str, - source_root: Path | None = None) -> dict[str, object]: + source_root: Path | None = None, source_lock: Path | None = None, + expected_source_lock_sha256: str | None = None, + expected_manifest_sha256: str | None = None, + expected_closure_digest: str | None = None) -> dict[str, object]: if not re.fullmatch(r"[0-9a-f-]{36}", daemon_id): raise ValueError("DAEMON_ID_INVALID") release_raw = _read(release_path, 128 * 1024) @@ -155,13 +317,25 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, or type(digest) is not str or not re.fullmatch(r"[0-9a-f]{64}", digest) or release["image"]["reference"].split("@")[-1] != image): raise ValueError("RELEASE_INVALID") - if output_dir.exists() or output_dir.is_symlink() or not output_dir.parent.is_dir(): + parent_info = output_dir.parent.stat() if output_dir.parent.is_dir() else None + if (output_dir.exists() or output_dir.is_symlink() or parent_info is None + or output_dir.parent.is_symlink() or parent_info.st_uid != os.getuid() + or stat.S_IMODE(parent_info.st_mode) != 0o700): raise ValueError("OUTPUT_INVALID") - if source_root is not None and (not source_root.is_absolute() or source_root.is_symlink() - or not source_root.is_dir() - or source_root.name != "engine" - or not (source_root.parent / "control/offline-sft-worker-v1.json").is_file()): + try: + _path_custody(output_dir.parent) + except Exception: + raise ValueError("OUTPUT_INVALID") from None + source_values = (source_lock, expected_source_lock_sha256, + expected_manifest_sha256, expected_closure_digest) + if (source_root is None and any(value is not None for value in source_values)) or ( + source_root is not None and any(value is None for value in source_values)): raise ValueError("SOURCE_INVALID") + source_identity = ({} if source_root is None else _source_identity( + source_root, source_lock, expected_source_lock_sha256, + expected_manifest_sha256, expected_closure_digest)) + custody = (() if source_root is None else tuple(_path_custody(path) for path in + (source_root, source_root.parent / "control/offline-sft-worker-v1.json", source_lock))) authority = _authority(docker, docker_config) info = _run(authority, ("info", "--format", "{{.ID}}")) if info.stderr or _one_line(info.stdout) != daemon_id: @@ -170,22 +344,114 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, if before.stderr or _one_line(before.stdout) != image: raise ValueError("IMAGE_INVALID") output_dir.mkdir(mode=0o700) - cidfile = output_dir / "container.id" + nonce = secrets.token_hex(16) + name = "synaptic-torch-import-" + nonce + probe = _PROBE if source_root is None else _GUARDED_PROBE.replace( + "EXPECTED_CLOSURE_DIGEST", expected_closure_digest) + claim = {"schema_version": SCHEMA, "daemon_id": daemon_id, "image_id": image, + "release_sha256": expected_release_sha256, "source_identity": source_identity, + "name": name, "scope": "bare" if source_root is None else "guarded", + "helper_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "probe_sha256": hashlib.sha256(probe.encode("utf-8")).hexdigest(), + "docker_file_identity": list(authority[2]), + "docker_config_identity": list(authority[3])} + claim_raw = json.dumps(claim, sort_keys=True, separators=(",", ":")).encode("ascii") + label = hashlib.sha256(claim_raw).hexdigest() + with (output_dir / "claim.json").open("xb") as stream: + stream.write(claim_raw + b"\n") + stream.flush() + os.fsync(stream.fileno()) + _append(output_dir, {"phase": "CREATE_CLAIMED", "claim_sha256": label}) mount = (() if source_root is None else ("--mount", f"type=bind,src={source_root.parent},dst=/source,readonly")) user = (() if source_root is None else ("--user", f"{source_root.stat().st_uid}:{source_root.stat().st_gid}")) command = ( - "run", "--cidfile", str(cidfile), "--pull=never", "--network=none", + "container", "create", "--name", name, "--label", "ai.synapticlabs.torch-import.claim=" + label, + "--pull=never", "--network=none", "--log-driver", "json-file", + "--log-opt", "max-size=1m", "--log-opt", "max-file=1", "--read-only", "--cap-drop=ALL", "--runtime=runc", "--security-opt=no-new-privileges", "--pids-limit=64", "--memory=4g", "--cpus=2", "--tmpfs", "/tmp:rw,nosuid,nodev,size=64m,mode=1777", *mount, *user, "--env=NVIDIA_VISIBLE_DEVICES=void", "--env=CUDA_VISIBLE_DEVICES=", "--workdir=/tmp", "--entrypoint", python, - image, "-I", "-c", _PROBE if source_root is None else _GUARDED_PROBE, + image, "-I", "-c", probe, ) - result = _run(authority, command) + if source_root is not None and tuple(_path_custody(path) for path in + (source_root, source_root.parent / "control/offline-sft-worker-v1.json", + source_lock)) != custody: + raise ValueError("SOURCE_IDENTITY_INVALID") + _require_daemon(authority, daemon_id) + try: + created = _run(authority, command) + except Exception: + _append(output_dir, {"phase": "CREATE_UNCERTAIN"}) + raise ValueError("CREATE_UNCERTAIN") from None + if created.stderr: + _append(output_dir, {"phase": "CREATE_UNCERTAIN"}) + raise ValueError("CREATE_UNCERTAIN") + cid = _one_line(created.stdout) + if not _CID.fullmatch(cid): + _append(output_dir, {"phase": "CREATE_UNCERTAIN"}) + raise ValueError("CREATE_UNCERTAIN") + _append(output_dir, {"phase": "CREATED", "container_id": cid}) + if _inspect_owned(authority, cid, name=name, label=label, image=image, + daemon_id=daemon_id)[0] != "created": + raise ValueError("CONTAINER_STATE_INVALID") + # Recheck the exact source bytes immediately before starting mounted code. + if source_root is not None and _source_identity(source_root, source_lock, + expected_source_lock_sha256, expected_manifest_sha256, + expected_closure_digest) != source_identity: + raise ValueError("SOURCE_IDENTITY_INVALID") + if source_root is not None and tuple(_path_custody(path) for path in + (source_root, source_root.parent / "control/offline-sft-worker-v1.json", + source_lock)) != custody: + raise ValueError("SOURCE_IDENTITY_INVALID") + deadline = time.monotonic() + _RUN_SECONDS + _append(output_dir, {"phase": "START_CLAIMED", "container_id": cid}) + cleanup_attempted = False + start_acknowledged = False + try: + started = _run(authority, ("container", "start", cid), timeout=15) + if started.stderr or _one_line(started.stdout) != cid: + raise ValueError("START_UNCERTAIN") + start_acknowledged = True + _append(output_dir, {"phase": "STARTED", "container_id": cid}) + while True: + if time.monotonic() >= deadline: + cleanup_attempted = True + confirmed = _stop_once(authority, cid, name=name, label=label, + image=image, daemon_id=daemon_id, output_dir=output_dir) + raise ValueError("DEADLINE_STOPPED" if confirmed else "DEADLINE_UNCERTAIN") + status, exit_code = _inspect_owned(authority, cid, name=name, label=label, + image=image, daemon_id=daemon_id, + timeout=max(0.1, min(5, deadline - time.monotonic()))) + if status in {"exited", "dead"}: + _append(output_dir, {"phase": "TERMINAL_OBSERVED", "container_id": cid, + "exit_code": exit_code}) + break + if time.monotonic() >= deadline: + cleanup_attempted = True + confirmed = _stop_once(authority, cid, name=name, label=label, + image=image, daemon_id=daemon_id, output_dir=output_dir) + raise ValueError("DEADLINE_STOPPED" if confirmed else "DEADLINE_UNCERTAIN") + time.sleep(min(0.5, max(0, deadline - time.monotonic()))) + except BaseException as error: + if not cleanup_attempted: + cleanup_attempted = True + confirmed = _stop_once(authority, cid, name=name, label=label, + image=image, daemon_id=daemon_id, output_dir=output_dir) + if not confirmed: + raise ValueError("DEADLINE_UNCERTAIN") from None + if isinstance(error, KeyboardInterrupt): + raise + if isinstance(error, ValueError) and error.args and error.args[0] in {"DEADLINE_UNCERTAIN", "DEADLINE_STOPPED"}: + raise + raise ValueError("DEADLINE_UNCERTAIN" if start_acknowledged else "START_UNCERTAIN") from None + if exit_code != 0: + raise ValueError("CHILD_OUTPUT_INVALID") + result = _run(authority, ("container", "logs", cid)) if result.stderr: raise ValueError("CHILD_OUTPUT_INVALID") child = json.loads(_one_line(result.stdout)) @@ -196,12 +462,9 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, or (source_root is not None and child["stage"] not in {"IMPORT_WORKER", "READ_MANIFEST", "LOAD_CLOSURE", "VERIFY_ORIGINS", "INSTALL_GUARD", "BOOTSTRAP_ENV", "TORCH_IMPORT"})): raise ValueError("CHILD_OUTPUT_INVALID") - cid = _one_line(_read(cidfile, 80)) - if not _CID.fullmatch(cid): - raise ValueError("CONTAINER_ID_INVALID") - inspected = _run(authority, ("container", "inspect", "--format", "{{.Image}}", cid)) - if inspected.stderr or _one_line(inspected.stdout) != image: - raise ValueError("CONTAINER_IMAGE_INVALID") + if _inspect_owned(authority, cid, name=name, label=label, image=image, + daemon_id=daemon_id)[0] not in {"exited", "dead"}: + raise ValueError("CONTAINER_STATE_INVALID") after = _run(authority, ("image", "inspect", "--format", "{{.Id}}", image)) if after.stderr or _one_line(after.stdout) != image: raise ValueError("IMAGE_CHANGED") @@ -209,7 +472,9 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, "container_id": cid, "image_id": image, "daemon_id": daemon_id, "runtime_release_digest": digest, "training_executed": False, "gpu_used": False, - "scope": "bare" if source_root is None else "guarded"} + "scope": "bare" if source_root is None else "guarded", + "claim_sha256": label, "source_identity": source_identity, + "terminal_observed": True} if source_root is not None: report["stage"] = child["stage"] with (output_dir / "result.json").open("x", encoding="ascii") as stream: @@ -225,13 +490,20 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--daemon-id", required=True) parser.add_argument("--expected-release-sha256", required=True) parser.add_argument("--source-root", type=Path) + parser.add_argument("--source-lock", type=Path) + parser.add_argument("--expected-source-lock-sha256") + parser.add_argument("--expected-manifest-sha256") + parser.add_argument("--expected-closure-digest") args = parser.parse_args(argv) try: report = diagnose(docker=args.docker, docker_config=args.docker_config, release_path=args.release, daemon_id=args.daemon_id, output_dir=args.output_dir, expected_release_sha256=args.expected_release_sha256, - source_root=args.source_root) + source_root=args.source_root, source_lock=args.source_lock, + expected_source_lock_sha256=args.expected_source_lock_sha256, + expected_manifest_sha256=args.expected_manifest_sha256, + expected_closure_digest=args.expected_closure_digest) except BaseException as error: reason = error.args[0] if type(error) is ValueError and len(error.args) == 1 else None if type(reason) is not str or reason not in _REASONS: diff --git a/tests/scripts/test_diagnose_docker_torch_import.py b/tests/scripts/test_diagnose_docker_torch_import.py index 862575cb..73d153e1 100644 --- a/tests/scripts/test_diagnose_docker_torch_import.py +++ b/tests/scripts/test_diagnose_docker_torch_import.py @@ -1,7 +1,10 @@ from __future__ import annotations -import json import hashlib +import json +import os +import shutil +import sys from pathlib import Path import pytest @@ -9,93 +12,259 @@ from scripts import diagnose_docker_torch_import as diagnostic -def test_closed_single_line_parser_rejects_extra_output() -> None: - assert diagnostic._one_line(b"abc\n") == "abc" - assert diagnostic._one_line(b"abc") == "abc" - for raw in (b"abc\nsecret\n", b" abc\n", b"\n", b"abc\r\n"): - with pytest.raises(ValueError): - diagnostic._one_line(raw) - - -def test_exact_image_import_probe_is_cpu_only_and_retains_container( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, -) -> None: - image = "sha256:" + "a" * 64 - cid = "b" * 64 - daemon = "24805003-b125-469b-8080-9da4daacd364" - release = {"image": {"digest": "a" * 64, "reference": "example/image@" + image}, - "python": {"executable": "/opt/unsloth-venv/bin/python3"}, - "manifest_digest": "c" * 64} - (tmp_path / "release.json").write_text(json.dumps(release, sort_keys=True, separators=(",", ":"))) - release_sha = hashlib.sha256((tmp_path / "release.json").read_bytes()).hexdigest() - monkeypatch.setattr(diagnostic, "_authority", lambda *_: object()) - commands = [] - - def run(_authority, args, *, timeout=120): - commands.append(args) - if args[:2] == ("info", "--format"): - return diagnostic.subprocess.CompletedProcess(args, 0, (daemon + "\n").encode(), b"") - if args[:2] == ("image", "inspect"): - return diagnostic.subprocess.CompletedProcess(args, 0, (image + "\n").encode(), b"") - if args[:2] == ("container", "inspect"): - return diagnostic.subprocess.CompletedProcess(args, 0, (image + "\n").encode(), b"") - assert args[0] == "run" - assert "--pull=never" in args and "--network=none" in args - assert "--read-only" in args and "--cap-drop=ALL" in args - assert "--memory=4g" in args and "--cpus=2" in args - assert "--env=NVIDIA_VISIBLE_DEVICES=void" in args - assert "--env=CUDA_VISIBLE_DEVICES=" in args - assert "--volume" not in args and "--gpus" not in args - assert "-I" in args and "-c" in args - guarded = "--mount" in args - if guarded: - assert "--user" in args - assert "readonly" in args[args.index("--mount") + 1] - target = tmp_path / "guarded" - payload = b'{"category":"PASS","schema_version":"synaptic-docker-torch-import-child/v1","stage":"TORCH_IMPORT"}\n' - else: - target = out - payload = b'{"category":"IMPORT","schema_version":"synaptic-docker-torch-import-child/v1"}\n' - (target / "container.id").write_text(cid + "\n") - return diagnostic.subprocess.CompletedProcess(args, 0, payload, b"") - - monkeypatch.setattr(diagnostic, "_run", run) - out = tmp_path / "fresh" - report = diagnostic.diagnose(docker=Path("/usr/bin/docker"), - docker_config=Path("/private/config"), release_path=tmp_path / "release.json", - daemon_id=daemon, output_dir=out, expected_release_sha256=release_sha) - assert report["category"] == "IMPORT" - assert report["container_id"] == cid - assert json.loads((out / "result.json").read_text()) == report - assert len(commands) == 5 - source = tmp_path / "source" / "engine" - source.mkdir(parents=True) - (source.parent / "control").mkdir() - (source.parent / "control" / "offline-sft-worker-v1.json").write_text("{}") - guarded = diagnostic.diagnose(docker=Path("/usr/bin/docker"), - docker_config=Path("/private/config"), release_path=tmp_path / "release.json", - daemon_id=daemon, output_dir=tmp_path / "guarded", - expected_release_sha256=release_sha, source_root=source) - assert guarded["category"] == "PASS" and guarded["stage"] == "TORCH_IMPORT" - assert guarded["scope"] == "guarded" +DAEMON = "24805003-b125-469b-8080-9da4daacd364" +IMAGE = "sha256:" + "a" * 64 +CID = "b" * 64 +ROOT = Path(__file__).parents[2] + + +def _release(root: Path) -> tuple[Path, str]: + path = root / "release.json" + path.write_text(json.dumps({"image": {"digest": "a" * 64, + "reference": "example/image@" + IMAGE}, "python": {"executable": "/usr/bin/python3"}, + "manifest_digest": "c" * 64}, sort_keys=True, separators=(",", ":"))) + return path, hashlib.sha256(path.read_bytes()).hexdigest() + + +def _fake_docker(root: Path) -> tuple[Path, Path, Path]: + executable = root / "docker" + state = root / "state.json" + state.write_text(json.dumps({"mode": "normal", "status": "absent", "stops": 0})) + config = root / "config" + config.mkdir() + executable.write_text(f"#!{sys.executable}\n" + r'''import json,sys,time +from pathlib import Path +state_path=Path(__file__).with_name('state.json') +s=json.loads(state_path.read_text()) +a=sys.argv[sys.argv.index('--host')+2:] +def save(): state_path.write_text(json.dumps(s)) +if a[0]=='info': print('24805003-b125-469b-8080-9da4daacd364') +elif a[:2]==['image','inspect']: print('sha256:'+'a'*64) +elif a[:2]==['container','create']: + s['status']='created'; s['name']=a[a.index('--name')+1] + s['guarded']='--mount' in a + s['label']=a[a.index('--label')+1].split('=',1)[1]; save() + if s['mode']=='create_hang': time.sleep(10) + if s['mode']=='bad_id': print('invalid'); sys.exit(0) + print('b'*64) +elif a[:2]==['container','start']: + s['status']='running'; save() + if s['mode']=='start_hang': time.sleep(10) + if s['mode']=='start_error': sys.exit(7) + if s['mode']=='start_stdout_flood': sys.stdout.write('x'*20000); sys.exit(0) + if s['mode']=='start_stderr_flood': sys.stderr.write('x'*20000); sys.exit(0) + print('b'*64) +elif a[:2]==['container','inspect']: + name='/' + s.get('name','missing') + if s['mode']=='wrong_identity' or (s['mode']=='wrong_running_identity' and s['status']=='running'): + name='/another-container' + for value in ('b'*64,name,'sha256:'+'a'*64, + {'ai.synapticlabs.torch-import.claim':s.get('label','')}, + s['status'],0): print(json.dumps(value)) +elif a[:2]==['container','stop']: + s['stops']+=1 + if s['mode']!='stop_unconfirmed': s['status']='exited' + save(); print('b'*64) +elif a[:2]==['container','logs']: + if s['mode']=='stdout_flood': sys.stdout.write('x'*20000) + elif s['mode']=='stderr_flood': sys.stderr.write('x'*20000) + elif s['mode']=='both_flood': + sys.stdout.write('x'*5000); sys.stderr.write('x'*5000) + elif s.get('guarded'): print('{"category":"PASS","schema_version":"synaptic-docker-torch-import-child/v1","stage":"TORCH_IMPORT"}') + else: print('{"category":"PASS","schema_version":"synaptic-docker-torch-import-child/v1"}') +else: sys.exit(8) +''') + executable.chmod(0o700) + return executable, config, state + + +def _call(root: Path, executable: Path, config: Path, *, output: str = "out", **extra): + release, sha = _release(root) + return diagnostic.diagnose(docker=executable, docker_config=config, + release_path=release, daemon_id=DAEMON, output_dir=root / output, + expected_release_sha256=sha, **extra) + + +def _state(path: Path, **changes): + value = json.loads(path.read_text()) + value.update(changes) + path.write_text(json.dumps(value)) + return value + + +def _stage_source(root: Path): + source = root / "source" + engine = source / "engine" + document = json.loads((ROOT / "tuner/runtime/manifests/offline-sft-worker-v1.json").read_text()) + for member in document["members"]: + target = engine / member["path"] + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(ROOT / member["path"], target) + (source / "control").mkdir() + manifest = source / "control/offline-sft-worker-v1.json" + shutil.copy2(ROOT / "tuner/runtime/manifests/offline-sft-worker-v1.json", manifest) + lock = root / "source-lock.json" + lock.write_text(json.dumps({"schema_version": "synaptic-source-lock/v1", "sources": { + "engine": {"commit": "0d515ae1ae235d0858802228ce4af8dba8fccd03"}}})) + flags = {"source_root": engine, "source_lock": lock, + "expected_source_lock_sha256": hashlib.sha256(lock.read_bytes()).hexdigest(), + "expected_manifest_sha256": hashlib.sha256(manifest.read_bytes()).hexdigest(), + "expected_closure_digest": document["closure_digest"]} + return flags, document, manifest + + +def test_real_cli_output_budget_and_timeout(tmp_path: Path) -> None: + docker, config, state = _fake_docker(tmp_path) + authority = diagnostic._authority(docker, config) + _state(state, mode="create_hang") + with pytest.raises(ValueError, match="DOCKER_COMMAND_REJECTED"): + diagnostic._run(authority, ("container", "create", "--name", "test-name", + "--label", "ai.synapticlabs.torch-import.claim=hash"), timeout=0.2) + # Killing the client did not stop the fake daemon's created container. + assert json.loads(state.read_text())["status"] == "created" + for mode in ("stdout_flood", "stderr_flood", "both_flood"): + _state(state, mode=mode) + with pytest.raises(ValueError, match="DOCKER_COMMAND_REJECTED"): + diagnostic._run(authority, ("container", "logs", CID), timeout=2) + + +def test_owned_lifecycle_and_closed_result(tmp_path: Path, monkeypatch) -> None: + docker, config, state = _fake_docker(tmp_path) + real = diagnostic._run + monkeypatch.setattr(diagnostic, "_run", lambda authority, args, *, timeout=15: + real(authority, args, timeout=min(timeout, 0.4))) + monkeypatch.setattr(diagnostic, "_RUN_SECONDS", 2) + # The fake daemon advances to terminal independently after start. + original_inspect = diagnostic._inspect_owned + def inspect(*args, **kwargs): + status, code = original_inspect(*args, **kwargs) + if status == "running": + _state(state, status="exited") + return status, code + monkeypatch.setattr(diagnostic, "_inspect_owned", inspect) + report = _call(tmp_path, docker, config) + assert report["category"] == "PASS" and report["terminal_observed"] is True + assert report["container_id"] == CID and report["source_identity"] == {} + assert json.loads((tmp_path / "out/result.json").read_text()) == report + assert json.loads(state.read_text())["stops"] == 0 + + +def test_guarded_happy_path_records_independent_source_identity(tmp_path: Path, monkeypatch) -> None: + flags, document, _manifest = _stage_source(tmp_path) + docker, config, state = _fake_docker(tmp_path) + monkeypatch.setattr(diagnostic, "_RUN_SECONDS", 2) + original_inspect = diagnostic._inspect_owned + def inspect(*args, **kwargs): + status, code = original_inspect(*args, **kwargs) + if status == "running": + _state(state, status="exited") + return status, code + monkeypatch.setattr(diagnostic, "_inspect_owned", inspect) + report = _call(tmp_path, docker, config, **flags) + assert report["scope"] == "guarded" and report["stage"] == "TORCH_IMPORT" + assert report["source_identity"]["closure_digest"] == document["closure_digest"] + assert report["source_identity"]["manifest_sha256"] == flags["expected_manifest_sha256"] + + +@pytest.mark.parametrize("mode,reason,expected_stops", [ + ("start_hang", "START_UNCERTAIN", 1), + ("start_error", "START_UNCERTAIN", 1), + ("start_stdout_flood", "START_UNCERTAIN", 1), + ("start_stderr_flood", "START_UNCERTAIN", 1), + ("normal", "DEADLINE_STOPPED", 1), + ("stop_unconfirmed", "DEADLINE_UNCERTAIN", 1), + ("wrong_identity", "CONTAINER_STATE_INVALID", 0), + ("wrong_running_identity", "DEADLINE_UNCERTAIN", 0), +]) +def test_uncertain_start_and_exact_owned_cleanup(tmp_path: Path, monkeypatch, + mode, reason, expected_stops) -> None: + docker, config, state = _fake_docker(tmp_path) + _state(state, mode=mode) + real = diagnostic._run + monkeypatch.setattr(diagnostic, "_run", lambda authority, args, *, timeout=15: + real(authority, args, timeout=min(timeout, 0.3))) + monkeypatch.setattr(diagnostic, "_RUN_SECONDS", + 1 if mode in {"normal", "stop_unconfirmed"} else 0.1) + with pytest.raises(ValueError, match=reason): + _call(tmp_path, docker, config) + actual = json.loads(state.read_text()) + assert actual["stops"] == expected_stops + if expected_stops and mode != "stop_unconfirmed": + assert actual["status"] == "exited" + assert (tmp_path / "out/claim.json").exists() + assert not (tmp_path / "out/result.json").exists() + + +def test_uncertain_create_does_not_replay_or_start(tmp_path: Path, monkeypatch) -> None: + docker, config, state = _fake_docker(tmp_path) + _state(state, mode="create_hang") + real = diagnostic._run + monkeypatch.setattr(diagnostic, "_run", lambda authority, args, *, timeout=15: + real(authority, args, timeout=min(timeout, 0.2))) + with pytest.raises(ValueError, match="CREATE_UNCERTAIN"): + _call(tmp_path, docker, config) + assert json.loads(state.read_text())["status"] == "created" + assert json.loads(state.read_text())["stops"] == 0 with pytest.raises(ValueError, match="OUTPUT_INVALID"): - diagnostic.diagnose(docker=Path("/usr/bin/docker"), - docker_config=Path("/private/config"), release_path=tmp_path / "release.json", - daemon_id=daemon, output_dir=out, expected_release_sha256=release_sha) + _call(tmp_path, docker, config) + + +def test_invalid_created_id_is_consumed_without_start(tmp_path: Path) -> None: + docker, config, state = _fake_docker(tmp_path) + _state(state, mode="bad_id") + with pytest.raises(ValueError, match="CREATE_UNCERTAIN"): + _call(tmp_path, docker, config) + assert json.loads(state.read_text())["status"] == "created" + assert (tmp_path / "out/claim.json").exists() + + +def test_interrupt_after_start_stops_only_owned_container_once(tmp_path: Path, monkeypatch) -> None: + docker, config, state = _fake_docker(tmp_path) + monkeypatch.setattr(diagnostic, "time", type("Clock", (), { + "monotonic": staticmethod(__import__("time").monotonic), + "sleep": staticmethod(lambda _seconds: (_ for _ in ()).throw(KeyboardInterrupt())) + })()) + with pytest.raises(KeyboardInterrupt): + _call(tmp_path, docker, config) + actual = json.loads(state.read_text()) + assert actual["stops"] == 1 and actual["status"] == "exited" + + +def test_wrong_self_consistent_source_rejected_before_docker(tmp_path: Path, monkeypatch) -> None: + from tuner.runtime.offline_sft_worker import closure_digest + flags, document, manifest = _stage_source(tmp_path) + engine = flags["source_root"] + assert diagnostic._source_identity(engine, flags["source_lock"], + flags["expected_source_lock_sha256"], flags["expected_manifest_sha256"], + flags["expected_closure_digest"])["closure_digest"] == flags["expected_closure_digest"] + # Alter a staged member and rewrite all its internal commitments coherently. + target = engine / "shared/env_bootstrap.py" + target.write_bytes(target.read_bytes() + b"\n# alternate closure\n") + member = next(item for item in document["members"] if item["path"] == "shared/env_bootstrap.py") + member["size_bytes"] = target.stat().st_size + member["sha256"] = hashlib.sha256(target.read_bytes()).hexdigest() + document["payload_bytes"] += len(b"\n# alternate closure\n") + document["closure_digest"] = closure_digest(document) + manifest.write_bytes(json.dumps(document, sort_keys=True, separators=(",", ":")).encode() + b"\n") + from tuner.runtime.offline_sft_worker import load_offline_sft_worker_closure + assert load_offline_sft_worker_closure(manifest, + expected_digest=document["closure_digest"], engine_root=engine).closure_digest == document["closure_digest"] + docker, config, _state_path = _fake_docker(tmp_path) + monkeypatch.setattr(diagnostic, "_run", lambda *args, **kwargs: pytest.fail("Docker touched")) + with pytest.raises(ValueError, match="SOURCE_IDENTITY_INVALID"): + _call(tmp_path, docker, config, **flags) + assert not (tmp_path / "out").exists() def test_cli_rejection_never_prints_exception_text(monkeypatch, capsys) -> None: def fail(**_kwargs): raise RuntimeError("private exception path /secret and credential") - monkeypatch.setattr(diagnostic, "diagnose", fail) code = diagnostic.main(["--docker", "/usr/bin/docker", "--docker-config", "/private/config", "--release", "/private/release.json", "--output-dir", "/private/output", - "--daemon-id", "24805003-b125-469b-8080-9da4daacd364", - "--expected-release-sha256", "a" * 64]) + "--daemon-id", DAEMON, "--expected-release-sha256", "a" * 64]) assert code == 2 output = capsys.readouterr() assert output.out == "" assert json.loads(output.err) == {"schema_version": diagnostic.SCHEMA, "status": "REJECTED", "reason_code": "OTHER"} - assert "private" not in output.err and "Traceback" not in output.err From c48fa8299333d6057bc876899d484386f2059ddc Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:46:27 -0400 Subject: [PATCH 03/12] Preserve owned diagnostic cleanup when evidence writes fail --- .../docker-torch-import-r2-diagnosis.md | 9 ++++ scripts/diagnose_docker_torch_import.py | 30 ++++++++--- .../test_diagnose_docker_torch_import.py | 50 ++++++++++++++++++- 3 files changed, 81 insertions(+), 8 deletions(-) diff --git a/docs/review/docker-torch-import-r2-diagnosis.md b/docs/review/docker-torch-import-r2-diagnosis.md index d13fd8e5..cb29a407 100644 --- a/docs/review/docker-torch-import-r2-diagnosis.md +++ b/docs/review/docker-torch-import-r2-diagnosis.md @@ -108,6 +108,15 @@ before any new effect remains required. No successor container has been launched. Review the exact command in this branch before a fresh CPU import diagnostic; a GPU-exposed probe needs separate scope review. +The cleanup path attempts its single authenticated stop even if evidence +appends fail, then inspects terminal state independently even if the stop CLI +acknowledgement fails. Failed evidence writes retain an uncertain result; an +in-memory observation does not establish durable proof. Source verification +and a read-only container mount do not freeze files on the host. Before this +probe, verify private ownership/permissions of the retained source and its +ancestors, and ensure that no preparation or other writer is using that source. +This probe assumes that private host custody remains in force during execution. + Proposed guarded CPU command for root review **only**. Run from a clean WSL checkout of this successor commit after confirming that its helper and trusted closure verifier match the reviewed source. First create the parent diff --git a/scripts/diagnose_docker_torch_import.py b/scripts/diagnose_docker_torch_import.py index f8ed0ca1..74a96514 100644 --- a/scripts/diagnose_docker_torch_import.py +++ b/scripts/diagnose_docker_torch_import.py @@ -272,24 +272,40 @@ def _inspect_owned(authority, identifier: str, *, name: str, label: str, image: def _stop_once(authority, identifier: str, *, name: str, label: str, image: str, daemon_id: str, output_dir: Path) -> bool: """One stop request; only a separate exact inspection can prove terminal state.""" + evidence_retained = True + + def record(phase: str) -> None: + nonlocal evidence_retained + try: + _append(output_dir, {"phase": phase, "container_id": identifier}) + except Exception: + # Storage failure cannot suppress cleanup of the already claimed + # container. It does prevent claiming durable terminal evidence. + evidence_retained = False + try: status, _ = _inspect_owned(authority, identifier, name=name, label=label, image=image, daemon_id=daemon_id) if status in {"exited", "dead"}: - _append(output_dir, {"phase": "TERMINAL_OBSERVED", "container_id": identifier}) - return True + record("TERMINAL_OBSERVED") + return evidence_retained if status != "running": return False - _append(output_dir, {"phase": "STOP_CLAIMED", "container_id": identifier}) - _run(authority, ("container", "stop", "--timeout", "2", identifier), timeout=10) + record("STOP_CLAIMED") + try: + _run(authority, ("container", "stop", "--timeout", "2", identifier), timeout=10) + except Exception: + # A lost CLI acknowledgement does not establish daemon state. + # Inspect independently without issuing a second stop. + pass status, _ = _inspect_owned(authority, identifier, name=name, label=label, image=image, daemon_id=daemon_id) if status in {"exited", "dead"}: - _append(output_dir, {"phase": "TERMINAL_OBSERVED", "container_id": identifier}) - return True + record("TERMINAL_OBSERVED") + return evidence_retained except Exception: pass - _append(output_dir, {"phase": "STOP_UNCONFIRMED", "container_id": identifier}) + record("STOP_UNCONFIRMED") return False diff --git a/tests/scripts/test_diagnose_docker_torch_import.py b/tests/scripts/test_diagnose_docker_torch_import.py index 73d153e1..da0bc66d 100644 --- a/tests/scripts/test_diagnose_docker_torch_import.py +++ b/tests/scripts/test_diagnose_docker_torch_import.py @@ -2,7 +2,6 @@ import hashlib import json -import os import shutil import sys from pathlib import Path @@ -230,6 +229,55 @@ def test_interrupt_after_start_stops_only_owned_container_once(tmp_path: Path, m assert actual["stops"] == 1 and actual["status"] == "exited" +def test_storage_failure_after_start_still_stops_owned_container(tmp_path: Path, monkeypatch) -> None: + docker, config, state = _fake_docker(tmp_path) + original_append = diagnostic._append + storage_failed = False + + def append(output_dir, event): + nonlocal storage_failed + if event["phase"] == "STARTED": + storage_failed = True + if storage_failed: + raise OSError("simulated full evidence filesystem") + original_append(output_dir, event) + + monkeypatch.setattr(diagnostic, "_append", append) + with pytest.raises(ValueError, match="DEADLINE_UNCERTAIN"): + _call(tmp_path, docker, config) + actual = json.loads(state.read_text()) + assert actual["stops"] == 1 and actual["status"] == "exited" + assert (tmp_path / "out/claim.json").exists() + assert not (tmp_path / "out/result.json").exists() + + +def test_uncertain_stop_acknowledgement_is_independently_inspected(tmp_path: Path, monkeypatch) -> None: + docker, config, state = _fake_docker(tmp_path) + original_run = diagnostic._run + inspected_after_stop = [] + stop_sent = False + + def run(authority, args, *, timeout=15): + nonlocal stop_sent + result = original_run(authority, args, timeout=timeout) + if args[:2] == ("container", "stop"): + stop_sent = True + raise ValueError("DOCKER_COMMAND_REJECTED") + if stop_sent and args[:2] == ("container", "inspect"): + inspected_after_stop.append(args[-1]) + return result + + monkeypatch.setattr(diagnostic, "_run", run) + monkeypatch.setattr(diagnostic, "_RUN_SECONDS", 0.1) + with pytest.raises(ValueError, match="DEADLINE_STOPPED"): + _call(tmp_path, docker, config) + actual = json.loads(state.read_text()) + assert actual["stops"] == 1 and actual["status"] == "exited" + assert inspected_after_stop == [CID] + events = [json.loads(line) for line in (tmp_path / "out/events.jsonl").read_text().splitlines()] + assert events[-1]["phase"] == "TERMINAL_OBSERVED" + + def test_wrong_self_consistent_source_rejected_before_docker(tmp_path: Path, monkeypatch) -> None: from tuner.runtime.offline_sft_worker import closure_digest flags, document, manifest = _stage_source(tmp_path) From 98e4a49ee175102f645e8c497acaff90fd5c4248 Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:05:12 -0400 Subject: [PATCH 04/12] Record r6 Torch context and prepare guarded GPU import comparison --- .agents/skills/fine-tuning/SKILL.md | 2 +- .../fine-tuning/reference/sft-training.md | 16 ++- .claude/skills/fine-tuning/SKILL.md | 2 +- .../fine-tuning/reference/sft-training.md | 16 ++- .skills/fine-tuning/SKILL.md | 2 +- .skills/fine-tuning/reference/sft-training.md | 16 ++- .../docker-torch-import-r2-diagnosis.md | 122 ++++++++++++++++-- scripts/diagnose_docker_torch_import.py | 50 ++++++- .../test_diagnose_docker_torch_import.py | 62 ++++++++- 9 files changed, 263 insertions(+), 25 deletions(-) diff --git a/.agents/skills/fine-tuning/SKILL.md b/.agents/skills/fine-tuning/SKILL.md index 4a39a46a..76e63fde 100644 --- a/.agents/skills/fine-tuning/SKILL.md +++ b/.agents/skills/fine-tuning/SKILL.md @@ -57,7 +57,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Check SFT loss masking (tokenizer only) | `python tuner.py doctor sft-mask --dataset-path --model ` | | Inspect native training host candidate | `python tuner.py doctor native-training --json` — read-only platform/Python/package metadata; see `reference/native-local-training.md` | | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | -| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for the bounded CPU-only bare and guarded probes; see `reference/sft-training.md` | +| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for bounded CPU probes and a separately reviewed guarded device-0 import probe; see `reference/sft-training.md` | | Native MLX prepared SFT candidate | Internal exact-token batch/loss and runtime comparison only; no native launch CLI or API provider yet; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.agents/skills/fine-tuning/reference/sft-training.md b/.agents/skills/fine-tuning/reference/sft-training.md index 36543ded..28dff3fd 100644 --- a/.agents/skills/fine-tuning/reference/sft-training.md +++ b/.agents/skills/fine-tuning/reference/sft-training.md @@ -511,8 +511,9 @@ The import execution has a to the authenticated new container and separately inspects terminal state; cleanup takes additional bounded time, and a stop acknowledgment alone is not quiescence. An uncertain create/start/stop remains consumed in `claim.json` and -`events.jsonl`, with no automatic replay or container removal. The probe asks -for at most 2 CPUs, 4 GiB, no network/GPU/model mount, and a read-only root. +`events.jsonl`, with no automatic replay or container removal. The default +probe asks for at most 2 CPUs, 4 GiB, no network/GPU/model mount, and a +read-only root. Bare mode mounts no source. Guarded mode mounts the independently authenticated retained source and runs the worker closure check, module guard, trainer path setup, environment bootstrap and Torch import. Neither runs training. `PASS` @@ -522,6 +523,17 @@ image or runtime policy. The exact r2 CPU evidence and its unresolved runtime difference are in `docs/review/docker-torch-import-r2-diagnosis.md`. +The guarded-only `--gpu-device 0` option is an import-only comparison that +requires separate exact root review before any live use. It requests only +device 0, verifies Docker retained that device request before start, removes +the CPU probe's visibility suppressors, and records the scope in its claim. +`gpu_used=null` means actual device use was not measured. The same 8-KiB +output budget, 120-second execution deadline, owned cleanup, source +authentication and closed child codes apply. It neither invokes the trainer +nor loads a model; a PASS or failure cannot retrospectively diagnose an +earlier trainer attempt. See the exact proposed command and remaining +environment/user/filesystem gaps in the r2 diagnosis note. + After an intentional worker source change, check and refresh existing manifests with the checked-in maintainers in dependency order: `regenerate_offline_sft_worker_closure.py`, diff --git a/.claude/skills/fine-tuning/SKILL.md b/.claude/skills/fine-tuning/SKILL.md index 4a39a46a..76e63fde 100644 --- a/.claude/skills/fine-tuning/SKILL.md +++ b/.claude/skills/fine-tuning/SKILL.md @@ -57,7 +57,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Check SFT loss masking (tokenizer only) | `python tuner.py doctor sft-mask --dataset-path --model ` | | Inspect native training host candidate | `python tuner.py doctor native-training --json` — read-only platform/Python/package metadata; see `reference/native-local-training.md` | | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | -| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for the bounded CPU-only bare and guarded probes; see `reference/sft-training.md` | +| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for bounded CPU probes and a separately reviewed guarded device-0 import probe; see `reference/sft-training.md` | | Native MLX prepared SFT candidate | Internal exact-token batch/loss and runtime comparison only; no native launch CLI or API provider yet; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.claude/skills/fine-tuning/reference/sft-training.md b/.claude/skills/fine-tuning/reference/sft-training.md index 36543ded..28dff3fd 100644 --- a/.claude/skills/fine-tuning/reference/sft-training.md +++ b/.claude/skills/fine-tuning/reference/sft-training.md @@ -511,8 +511,9 @@ The import execution has a to the authenticated new container and separately inspects terminal state; cleanup takes additional bounded time, and a stop acknowledgment alone is not quiescence. An uncertain create/start/stop remains consumed in `claim.json` and -`events.jsonl`, with no automatic replay or container removal. The probe asks -for at most 2 CPUs, 4 GiB, no network/GPU/model mount, and a read-only root. +`events.jsonl`, with no automatic replay or container removal. The default +probe asks for at most 2 CPUs, 4 GiB, no network/GPU/model mount, and a +read-only root. Bare mode mounts no source. Guarded mode mounts the independently authenticated retained source and runs the worker closure check, module guard, trainer path setup, environment bootstrap and Torch import. Neither runs training. `PASS` @@ -522,6 +523,17 @@ image or runtime policy. The exact r2 CPU evidence and its unresolved runtime difference are in `docs/review/docker-torch-import-r2-diagnosis.md`. +The guarded-only `--gpu-device 0` option is an import-only comparison that +requires separate exact root review before any live use. It requests only +device 0, verifies Docker retained that device request before start, removes +the CPU probe's visibility suppressors, and records the scope in its claim. +`gpu_used=null` means actual device use was not measured. The same 8-KiB +output budget, 120-second execution deadline, owned cleanup, source +authentication and closed child codes apply. It neither invokes the trainer +nor loads a model; a PASS or failure cannot retrospectively diagnose an +earlier trainer attempt. See the exact proposed command and remaining +environment/user/filesystem gaps in the r2 diagnosis note. + After an intentional worker source change, check and refresh existing manifests with the checked-in maintainers in dependency order: `regenerate_offline_sft_worker_closure.py`, diff --git a/.skills/fine-tuning/SKILL.md b/.skills/fine-tuning/SKILL.md index 4a39a46a..76e63fde 100644 --- a/.skills/fine-tuning/SKILL.md +++ b/.skills/fine-tuning/SKILL.md @@ -57,7 +57,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Check SFT loss masking (tokenizer only) | `python tuner.py doctor sft-mask --dataset-path --model ` | | Inspect native training host candidate | `python tuner.py doctor native-training --json` — read-only platform/Python/package metadata; see `reference/native-local-training.md` | | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | -| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for the bounded CPU-only bare and guarded probes; see `reference/sft-training.md` | +| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for bounded CPU probes and a separately reviewed guarded device-0 import probe; see `reference/sft-training.md` | | Native MLX prepared SFT candidate | Internal exact-token batch/loss and runtime comparison only; no native launch CLI or API provider yet; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.skills/fine-tuning/reference/sft-training.md b/.skills/fine-tuning/reference/sft-training.md index 36543ded..28dff3fd 100644 --- a/.skills/fine-tuning/reference/sft-training.md +++ b/.skills/fine-tuning/reference/sft-training.md @@ -511,8 +511,9 @@ The import execution has a to the authenticated new container and separately inspects terminal state; cleanup takes additional bounded time, and a stop acknowledgment alone is not quiescence. An uncertain create/start/stop remains consumed in `claim.json` and -`events.jsonl`, with no automatic replay or container removal. The probe asks -for at most 2 CPUs, 4 GiB, no network/GPU/model mount, and a read-only root. +`events.jsonl`, with no automatic replay or container removal. The default +probe asks for at most 2 CPUs, 4 GiB, no network/GPU/model mount, and a +read-only root. Bare mode mounts no source. Guarded mode mounts the independently authenticated retained source and runs the worker closure check, module guard, trainer path setup, environment bootstrap and Torch import. Neither runs training. `PASS` @@ -522,6 +523,17 @@ image or runtime policy. The exact r2 CPU evidence and its unresolved runtime difference are in `docs/review/docker-torch-import-r2-diagnosis.md`. +The guarded-only `--gpu-device 0` option is an import-only comparison that +requires separate exact root review before any live use. It requests only +device 0, verifies Docker retained that device request before start, removes +the CPU probe's visibility suppressors, and records the scope in its claim. +`gpu_used=null` means actual device use was not measured. The same 8-KiB +output budget, 120-second execution deadline, owned cleanup, source +authentication and closed child codes apply. It neither invokes the trainer +nor loads a model; a PASS or failure cannot retrospectively diagnose an +earlier trainer attempt. See the exact proposed command and remaining +environment/user/filesystem gaps in the r2 diagnosis note. + After an intentional worker source change, check and refresh existing manifests with the checked-in maintainers in dependency order: `regenerate_offline_sft_worker_closure.py`, diff --git a/docs/review/docker-torch-import-r2-diagnosis.md b/docs/review/docker-torch-import-r2-diagnosis.md index cb29a407..afa72b41 100644 --- a/docs/review/docker-torch-import-r2-diagnosis.md +++ b/docs/review/docker-torch-import-r2-diagnosis.md @@ -104,9 +104,8 @@ manifest SHA-256 `3842689403abc5b42e28d58545b88616b81cd8fc738ee5557e97159df65d00 and closure digest `8465ff2b7cedb352337e7cedc5bb17cc169892c2eedae0b280569f4a0cde5002` are checked before mounted code executes and included in the result. These values were read from the retained r2 material on 2026-10-20; live rechecking -before any new effect remains required. No successor container has been -launched. Review the exact command in this branch before a fresh CPU import -diagnostic; a GPU-exposed probe needs separate scope review. +before any new effect remains required. The reviewed successor CPU probe ran +once as r6 below. A GPU-exposed probe still needs separate scope review. The cleanup path attempts its single authenticated stop even if evidence appends fail, then inspects terminal state independently even if the stop CLI @@ -117,11 +116,12 @@ probe, verify private ownership/permissions of the retained source and its ancestors, and ensure that no preparation or other writer is using that source. This probe assumes that private host custody remains in force during execution. -Proposed guarded CPU command for root review **only**. Run from a clean WSL -checkout of this successor commit after confirming that its helper and trusted -closure verifier match the reviewed source. First create the parent +The reviewed r6 guarded CPU command is retained here as the exact executed +invocation. It ran from a clean WSL checkout of +`c48fa8299333d6057bc876899d484386f2059ddc` after confirming that its +helper and trusted closure verifier matched the reviewed source. Its parent `/home/profsynapse/.local/share/synaptic-qualification/torch-import-r5-20261020` -as a private mode-0700 directory; the `guarded-cpu-1` child must not exist. +was private mode 0700; the `guarded-cpu-1` child was absent before the run. ```sh python3 -I scripts/diagnose_docker_torch_import.py \ @@ -138,9 +138,9 @@ python3 -I scripts/diagnose_docker_torch_import.py \ --output-dir /home/profsynapse/.local/share/synaptic-qualification/torch-import-r5-20261020/guarded-cpu-1 ``` -This repeats the reduced CPU import boundary with repaired ownership and -source binding. It is **not** an exact r2 invocation, does not use GPU, and -does not authorize another SFT start. The original r2 claim remains consumed. +This repeated the reduced CPU import boundary with repaired ownership and +source binding. It was **not** an exact r2 invocation, did not expose a GPU, +and did not authorize another SFT start. The original r2 claim remains consumed. The next comparison should hold the independently authenticated r2 source, daemon and image fixed, then compare the offline child environment produced by @@ -153,3 +153,105 @@ writable root and GPU visibility at a time in import-only containers under reviewed bounds. A GPU-visible import-only experiment is a separate reviewed scope; no trainer, model load or training is proposed. Treat any changed result as evidence for a narrower hypothesis, not retrospective proof of r2's cause. + +## r6 result and read-only r2 context comparison + +The one authorized r6 run used the unchanged command above after read-only +preflight and a saved/read-back Nexus admission. The clean WSL checkout was +`c48fa8299333d6057bc876899d484386f2059ddc`; helper SHA-256 was +`290bf1aa8e10ce3cf8a7eb0f1ff8e5a394a7ac726495330eaf0ff24fe9a33ac1` +and trusted closure-verifier SHA-256 was +`2304eeb00667954510c34cc4aacaa818532a301fbeabf04c2fb165d6cd1d70b5`. +The retained r2 manifest, source lock and release matched the independently +approved hashes in the command. No matching preparer or worker process was +observed before start; the source and parent were privately owned. This +custody check does not freeze host files against a later writer. + +The r6 child returned `PASS` at `TORCH_IMPORT`, with `training_executed=false` +and `gpu_used=false`. Container +`8c578b6ec0930e9c7668f237672b9d11f4122524befe601d077504966ff203e0` +is retained and independently inspected as exited, code 0. Its claim label is +`f07f5348f19fc6bd7210929c78a1b7c2b900361da249ac5755ce6eabb8b2330d`; +the daemon and image IDs match r2. It started at +`2026-10-20T14:51:44.979964328Z` and finished at +`2026-10-20T14:51:48.957092702Z`. The claim, events and result under the +private `torch-import-r5-20261020/guarded-cpu-1` directory have SHA-256 +`a62c0bb6a610c2a27a7951fe7e7c95038720b9ed882a6c6044fe1aeffaf6c8dc`, +`0bab6899f60dd05b885e65fa9a4c90ee8f65d5c751357bf4fad8752943a2446d`, +and `c22892913e1c0b3aaff1bfc99b876c80a653000ae66c8f29d3e83023c00a1308`, +respectively. Events show one claimed create, one start and terminal exit 0; +no stop or deletion was needed. The r2/r4 claims and containers remain preserved. + +The retained r2 workload SHA-256 is +`1534308c977c2c1cb2b1d80d9cc5c07e31303db63b336686e1c5ccc2644b1064`; +its Docker runtime declaration SHA-256 is +`947809449ffce1fde2cdb6a928c2cec60444cc3d841d173405080b7f545e4aae`. +The staged r2 `runtime_v1.py`, `train_sft.py` and `offline_sft_worker.py` bytes +match engine commit `0d515ae1ae235d0858802228ce4af8dba8fccd03`. +The pinned release records CPython 3.12.3 and manifest digest +`c5d6c01fb1dd62276b033234982cd14846865500dd07eaed2a2102dea97a1535`. +Read-only inspection of both retained Docker container configs and these +sources gives the comparison below. + +| Boundary | Retained r2 | Guarded r6 CPU | +| --- | --- | --- | +| Image, daemon, source | Pinned image and daemon; staged engine closure `8465ff2b...` | Same image and daemon; independently authenticated same staged closure | +| Host container | Root, `/workspace`, default caps, writable root, GPU device request `0`, 16 GiB | UID/GID 1000, `/tmp`, all caps dropped, read-only root, no device request, 4 GiB | +| Common host settings | 2 CPUs, network `none`, `runc` | 2 CPUs, network `none`, `runc`; 120-second execution deadline plus bounded cleanup tail | +| Python entry | Image Python starts parent `runtime_v1.py`; parent starts separate `-I offline_sft_worker.py -- ...` child | Same image Python runs `-I -c` fixed diagnostic; no parent or worker entrypoint executes | +| Closure and import order | Worker loads closure, verifies origins, installs guard, adds trainer directory and engine root, then `runpy.run_path` enters `train_sft.py`; trainer adds engine and `src`, bootstraps, imports Torch | Probe loads same closure, verifies origins, installs guard, arranges the same leading engine/`src`/trainer/engine path order, bootstraps and imports Torch directly; later path tail and loaded-module history differ | +| Offline child environment | Source builds `child_env` from the planned environment with `clear_inherited=true`, removes `PYTHONPATH`, adds closure/root bindings and overrides offline/HF/cache flags; actual process environment was not separately captured | Probe clears inherited environment and installs fixed import-only values. It lacks r2's closure/root environment bindings and HF/cache paths; expected closure digest is embedded in fixed probe code | +| Visibility inside child | Planned environment has no `CUDA_VISIBLE_DEVICES`, `NVIDIA_VISIBLE_DEVICES` or `LD_LIBRARY_PATH`; parent Docker config has NVIDIA/LD entries, but `env=` prevents child inheritance | Probe sets CUDA visibility empty and NVIDIA visibility `void`; it also omits `LD_LIBRARY_PATH` | + +The r2 planned environment's sorted compact `jq -cS` serialization +(including its trailing LF) has SHA-256 +`e621be06bb27b0c258e8ef8ecfd35eccaa362236f9a194eb122bfe4445a798ed`. +Its `PATH` value SHA-256 is +`ff201961161ffc598d11b7e168870c3fdbe7485141fa414c3f67f2ce89244338`; +the r6 fixed probe `PATH` value SHA-256 is +`05f89650ff263293ba277e3c910f4343e0f9b1a37c6d27e0aab9cb3b772e0896`. +The r2 plan's `PYTHONPATH` equals the engine root but is explicitly removed +from `child_env`. Its declared `HF_HOME` and `TRANSFORMERS_CACHE` equal the +cache root plus their respective subdirectories, matching the child builder's +overrides. These are source-and-plan deductions, not a captured r2 child +environment. No environment value, raw argv or exception text is published. + +The strongest next single-factor import-only comparison is GPU visibility in +the otherwise unchanged guarded diagnostic. This holds the authenticated +image/source, user, capabilities, read-only root, fixed probe and deadline +constant while changing the device request and removing the two explicit +visibility suppressors. Docker documents `--gpus device=0` for exposing only +GPU index 0 in its [GPU access guide](https://docs.docker.com/engine/containers/gpu/). +A changed result would implicate GPU exposure under +these reduced conditions; either result would leave r2's exact parent/child +environment, user/caps and filesystem differences unresolved. The existing +helper now has a fixed guarded-only `--gpu-device 0` option. It requests only +device 0, checks the resulting Docker device request before start, includes +this scope in its claim, and reports `gpu_used=null` because device use is not +measured. It retains the 8-KiB CLI output budget, 120-second execution +deadline, single owned stop and independent terminal inspection. The fixed +child still only installs the source guard, bootstraps and imports Torch; it +does not invoke the trainer, load a model or run training. + +**Proposal for root review; do not execute under the r6 authorization.** A +future reviewed run would require a clean checkout of the new helper commit, +fresh read-only daemon/image/source/custody checks, a new private mode-0700 +`/home/profsynapse/.local/share/synaptic-qualification/torch-import-r7-20261020` +parent, an absent `guarded-gpu0-1` child, and a saved/read-back one-use +admission. Its exact command is: + +```sh +python3 -I scripts/diagnose_docker_torch_import.py \ + --docker /usr/bin/docker \ + --docker-config /home/profsynapse/.local/share/synaptic-qualification/docker-sft-stable-image-c/docker-config \ + --release /home/profsynapse/.local/share/synaptic-qualification/docker-sft-stable-image-c/evidence/runtime-release.json \ + --expected-release-sha256 5f65824ceb5e250a58314da0233b9c07f1d63f1b0a39892d9b36af694938bfe6 \ + --daemon-id 24805003-b125-469b-8080-9da4daacd364 \ + --source-root /home/profsynapse/.local/share/synaptic-qualification/docker-public-plan-0d515-20261020/attempt-r2/source/engine \ + --source-lock /home/profsynapse/.local/share/synaptic-qualification/docker-public-plan-0d515-20261020/attempt-r2/material/source-lock.json \ + --expected-source-lock-sha256 d19aecb2d3ca1b7b6fa32a2445c45251e59d8cac2332293c68b73a61cbff4665 \ + --expected-manifest-sha256 3842689403abc5b42e28d58545b88616b81cd8fc738ee5557e97159df65d00ef \ + --expected-closure-digest 8465ff2b7cedb352337e7cedc5bb17cc169892c2eedae0b280569f4a0cde5002 \ + --gpu-device 0 \ + --output-dir /home/profsynapse/.local/share/synaptic-qualification/torch-import-r7-20261020/guarded-gpu0-1 +``` diff --git a/scripts/diagnose_docker_torch_import.py b/scripts/diagnose_docker_torch_import.py index 74a96514..1ef6a996 100644 --- a/scripts/diagnose_docker_torch_import.py +++ b/scripts/diagnose_docker_torch_import.py @@ -29,10 +29,11 @@ "DOCKER_CHANGED", "DOCKER_COMMAND_REJECTED", "DIAGNOSTIC_OUTPUT_INVALID", "RELEASE_INVALID", "RELEASE_HASH_INVALID", "OUTPUT_INVALID", "SOURCE_INVALID", "DAEMON_INVALID", "IMAGE_INVALID", "CHILD_OUTPUT_INVALID", "CONTAINER_ID_INVALID", "CONTAINER_IMAGE_INVALID", - "IMAGE_CHANGED", "SOURCE_IDENTITY_INVALID", "CREATE_UNCERTAIN", "START_UNCERTAIN", + "IMAGE_CHANGED", "SOURCE_IDENTITY_INVALID", "GPU_SCOPE_INVALID", "CREATE_UNCERTAIN", "START_UNCERTAIN", "DEADLINE_UNCERTAIN", "DEADLINE_STOPPED", "STOP_UNCONFIRMED", "CONTAINER_STATE_INVALID"}) _CLI_BUDGET = 8192 _RUN_SECONDS = 120 +_HIDDEN_GPU_ENV = ",'CUDA_VISIBLE_DEVICES':'','NVIDIA_VISIBLE_DEVICES':'void'" _INSPECT = "{{json .Id}}\n{{json .Name}}\n{{json .Image}}\n{{json .Config.Labels}}\n{{json .State.Status}}\n{{json .State.ExitCode}}" # This fixed program never formats an exception, inspects its attributes, or @@ -269,6 +270,26 @@ def _inspect_owned(authority, identifier: str, *, name: str, label: str, image: return status, code +def _require_gpu_zero(authority, identifier: str) -> None: + """Confirm the fixed device request before any GPU-exposed code starts.""" + result = _run(authority, ("container", "inspect", "--format", + "{{json .HostConfig.DeviceRequests}}", identifier)) + if result.stderr: + raise ValueError("GPU_SCOPE_INVALID") + try: + requests = json.loads(_one_line(result.stdout)) + except Exception: + raise ValueError("GPU_SCOPE_INVALID") from None + if (type(requests) is not list or len(requests) != 1 + or type(requests[0]) is not dict + or requests[0].get("DeviceIDs") != ["0"] + or requests[0].get("Capabilities") != [["gpu"]] + or requests[0].get("Count") != 0 + or requests[0].get("Driver") != "" + or requests[0].get("Options") != {}): + raise ValueError("GPU_SCOPE_INVALID") + + def _stop_once(authority, identifier: str, *, name: str, label: str, image: str, daemon_id: str, output_dir: Path) -> bool: """One stop request; only a separate exact inspection can prove terminal state.""" @@ -314,7 +335,10 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, source_root: Path | None = None, source_lock: Path | None = None, expected_source_lock_sha256: str | None = None, expected_manifest_sha256: str | None = None, - expected_closure_digest: str | None = None) -> dict[str, object]: + expected_closure_digest: str | None = None, + gpu_device: str | None = None) -> dict[str, object]: + if gpu_device not in (None, "0") or (gpu_device is not None and source_root is None): + raise ValueError("GPU_SCOPE_INVALID") if not re.fullmatch(r"[0-9a-f-]{36}", daemon_id): raise ValueError("DAEMON_ID_INVALID") release_raw = _read(release_path, 128 * 1024) @@ -364,6 +388,12 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, name = "synaptic-torch-import-" + nonce probe = _PROBE if source_root is None else _GUARDED_PROBE.replace( "EXPECTED_CLOSURE_DIGEST", expected_closure_digest) + if gpu_device is not None: + if probe.count(_HIDDEN_GPU_ENV) != 1: + raise ValueError("GPU_SCOPE_INVALID") + # The r2 isolated child did not inherit either visibility variable. + # The host device request is separately inspected before start. + probe = probe.replace(_HIDDEN_GPU_ENV, "") claim = {"schema_version": SCHEMA, "daemon_id": daemon_id, "image_id": image, "release_sha256": expected_release_sha256, "source_identity": source_identity, "name": name, "scope": "bare" if source_root is None else "guarded", @@ -371,6 +401,8 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, "probe_sha256": hashlib.sha256(probe.encode("utf-8")).hexdigest(), "docker_file_identity": list(authority[2]), "docker_config_identity": list(authority[3])} + if gpu_device is not None: + claim["gpu_device"] = gpu_device claim_raw = json.dumps(claim, sort_keys=True, separators=(",", ":")).encode("ascii") label = hashlib.sha256(claim_raw).hexdigest() with (output_dir / "claim.json").open("xb") as stream: @@ -382,6 +414,8 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, ("--mount", f"type=bind,src={source_root.parent},dst=/source,readonly")) user = (() if source_root is None else ("--user", f"{source_root.stat().st_uid}:{source_root.stat().st_gid}")) + visibility = (("--env=NVIDIA_VISIBLE_DEVICES=void", "--env=CUDA_VISIBLE_DEVICES=") + if gpu_device is None else ("--gpus", "device=0")) command = ( "container", "create", "--name", name, "--label", "ai.synapticlabs.torch-import.claim=" + label, "--pull=never", "--network=none", "--log-driver", "json-file", @@ -390,7 +424,7 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, "--security-opt=no-new-privileges", "--pids-limit=64", "--memory=4g", "--cpus=2", "--tmpfs", "/tmp:rw,nosuid,nodev,size=64m,mode=1777", *mount, *user, - "--env=NVIDIA_VISIBLE_DEVICES=void", "--env=CUDA_VISIBLE_DEVICES=", + *visibility, "--workdir=/tmp", "--entrypoint", python, image, "-I", "-c", probe, ) @@ -415,6 +449,8 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, if _inspect_owned(authority, cid, name=name, label=label, image=image, daemon_id=daemon_id)[0] != "created": raise ValueError("CONTAINER_STATE_INVALID") + if gpu_device is not None: + _require_gpu_zero(authority, cid) # Recheck the exact source bytes immediately before starting mounted code. if source_root is not None and _source_identity(source_root, source_lock, expected_source_lock_sha256, expected_manifest_sha256, @@ -487,12 +523,14 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, report = {"schema_version": SCHEMA, "category": child["category"], "container_id": cid, "image_id": image, "daemon_id": daemon_id, "runtime_release_digest": digest, - "training_executed": False, "gpu_used": False, + "training_executed": False, "gpu_used": None if gpu_device is not None else False, "scope": "bare" if source_root is None else "guarded", "claim_sha256": label, "source_identity": source_identity, "terminal_observed": True} if source_root is not None: report["stage"] = child["stage"] + if gpu_device is not None: + report["gpu_device"] = gpu_device with (output_dir / "result.json").open("x", encoding="ascii") as stream: json.dump(report, stream, sort_keys=True, separators=(",", ":")) stream.write("\n") @@ -510,6 +548,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--expected-source-lock-sha256") parser.add_argument("--expected-manifest-sha256") parser.add_argument("--expected-closure-digest") + parser.add_argument("--gpu-device") args = parser.parse_args(argv) try: report = diagnose(docker=args.docker, docker_config=args.docker_config, @@ -519,7 +558,8 @@ def main(argv: list[str] | None = None) -> int: source_root=args.source_root, source_lock=args.source_lock, expected_source_lock_sha256=args.expected_source_lock_sha256, expected_manifest_sha256=args.expected_manifest_sha256, - expected_closure_digest=args.expected_closure_digest) + expected_closure_digest=args.expected_closure_digest, + gpu_device=args.gpu_device) except BaseException as error: reason = error.args[0] if type(error) is ValueError and len(error.args) == 1 else None if type(reason) is not str or reason not in _REASONS: diff --git a/tests/scripts/test_diagnose_docker_torch_import.py b/tests/scripts/test_diagnose_docker_torch_import.py index da0bc66d..a65d890a 100644 --- a/tests/scripts/test_diagnose_docker_torch_import.py +++ b/tests/scripts/test_diagnose_docker_torch_import.py @@ -42,6 +42,8 @@ def save(): state_path.write_text(json.dumps(s)) elif a[:2]==['container','create']: s['status']='created'; s['name']=a[a.index('--name')+1] s['guarded']='--mount' in a + s['gpu_requested']='--gpus' in a and a[a.index('--gpus')+1]=='device=0' if '--gpus' in a else False + s['gpu_probe_unhidden']=s['gpu_requested'] and "'NVIDIA_VISIBLE_DEVICES':'void'" not in a[-1] and "'CUDA_VISIBLE_DEVICES':''" not in a[-1] s['label']=a[a.index('--label')+1].split('=',1)[1]; save() if s['mode']=='create_hang': time.sleep(10) if s['mode']=='bad_id': print('invalid'); sys.exit(0) @@ -54,6 +56,10 @@ def save(): state_path.write_text(json.dumps(s)) if s['mode']=='start_stderr_flood': sys.stderr.write('x'*20000); sys.exit(0) print('b'*64) elif a[:2]==['container','inspect']: + if 'DeviceRequests' in a[a.index('--format')+1]: + if s['mode']=='wrong_gpu_request': print('null') + else: print(json.dumps([{'Capabilities':[['gpu']],'Count':0,'DeviceIDs':['0'],'Driver':'','Options':{}}] if s.get('gpu_requested') else None)) + sys.exit(0) name='/' + s.get('name','missing') if s['mode']=='wrong_identity' or (s['mode']=='wrong_running_identity' and s['status']=='running'): name='/another-container' @@ -165,6 +171,60 @@ def inspect(*args, **kwargs): assert report["source_identity"]["manifest_sha256"] == flags["expected_manifest_sha256"] +def test_gpu_visibility_is_fixed_guarded_import_only(tmp_path: Path, monkeypatch) -> None: + flags, _, _ = _stage_source(tmp_path) + docker, config, state = _fake_docker(tmp_path) + monkeypatch.setattr(diagnostic, "_RUN_SECONDS", 2) + original_inspect = diagnostic._inspect_owned + def inspect(*args, **kwargs): + status, code = original_inspect(*args, **kwargs) + if status == "running": + _state(state, status="exited") + return status, code + monkeypatch.setattr(diagnostic, "_inspect_owned", inspect) + report = _call(tmp_path, docker, config, gpu_device="0", **flags) + claim = json.loads((tmp_path / "out/claim.json").read_text()) + actual = json.loads(state.read_text()) + assert claim["gpu_device"] == "0" + assert actual["gpu_requested"] is True and actual["gpu_probe_unhidden"] is True + assert actual["stops"] == 0 + assert report["category"] == "PASS" and report["stage"] == "TORCH_IMPORT" + assert report["gpu_device"] == "0" and report["gpu_used"] is None + assert report["training_executed"] is False + + +def test_gpu_scope_rejects_before_effect_or_start(tmp_path: Path) -> None: + docker, config, state = _fake_docker(tmp_path) + with pytest.raises(ValueError, match="GPU_SCOPE_INVALID"): + _call(tmp_path, docker, config, gpu_device="0") + assert json.loads(state.read_text())["status"] == "absent" + assert not (tmp_path / "out").exists() + flags, _, _ = _stage_source(tmp_path) + with pytest.raises(ValueError, match="GPU_SCOPE_INVALID"): + _call(tmp_path, docker, config, gpu_device="1", **flags) + assert json.loads(state.read_text())["status"] == "absent" + _state(state, mode="wrong_gpu_request") + with pytest.raises(ValueError, match="GPU_SCOPE_INVALID"): + _call(tmp_path, docker, config, gpu_device="0", **flags) + assert json.loads(state.read_text())["status"] == "created" + assert json.loads(state.read_text())["stops"] == 0 + assert (tmp_path / "out/claim.json").exists() + assert not (tmp_path / "out/result.json").exists() + + +def test_invalid_gpu_cli_value_is_closed(tmp_path: Path, capsys) -> None: + exit_code = diagnostic.main([ + "--docker", str(tmp_path / "docker"), "--docker-config", str(tmp_path / "config"), + "--release", str(tmp_path / "release"), "--output-dir", str(tmp_path / "out"), + "--daemon-id", DAEMON, "--expected-release-sha256", "a" * 64, + "--gpu-device", "private-value", + ]) + captured = capsys.readouterr() + assert exit_code == 2 and captured.out == "" + assert json.loads(captured.err) == {"schema_version": diagnostic.SCHEMA, + "status": "REJECTED", "reason_code": "GPU_SCOPE_INVALID"} + + @pytest.mark.parametrize("mode,reason,expected_stops", [ ("start_hang", "START_UNCERTAIN", 1), ("start_error", "START_UNCERTAIN", 1), @@ -268,7 +328,7 @@ def run(authority, args, *, timeout=15): return result monkeypatch.setattr(diagnostic, "_run", run) - monkeypatch.setattr(diagnostic, "_RUN_SECONDS", 0.1) + monkeypatch.setattr(diagnostic, "_RUN_SECONDS", 1) with pytest.raises(ValueError, match="DEADLINE_STOPPED"): _call(tmp_path, docker, config) actual = json.loads(state.read_text()) From 24691802150605149b18bf9e0ea36f31a1a9b7e5 Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:12:05 -0400 Subject: [PATCH 05/12] Record single guarded GPU import result and limits --- .../docker-torch-import-r2-diagnosis.md | 46 +++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/docs/review/docker-torch-import-r2-diagnosis.md b/docs/review/docker-torch-import-r2-diagnosis.md index afa72b41..f27d4f59 100644 --- a/docs/review/docker-torch-import-r2-diagnosis.md +++ b/docs/review/docker-torch-import-r2-diagnosis.md @@ -233,12 +233,12 @@ deadline, single owned stop and independent terminal inspection. The fixed child still only installs the source guard, bootstraps and imports Torch; it does not invoke the trainer, load a model or run training. -**Proposal for root review; do not execute under the r6 authorization.** A -future reviewed run would require a clean checkout of the new helper commit, +Root separately reviewed and authorized the following command for one r8 +import-only run. It required a clean checkout of `98e4a49ee175102f645e8c497acaff90fd5c4248`, fresh read-only daemon/image/source/custody checks, a new private mode-0700 `/home/profsynapse/.local/share/synaptic-qualification/torch-import-r7-20261020` parent, an absent `guarded-gpu0-1` child, and a saved/read-back one-use -admission. Its exact command is: +admission. This exact command ran once and is consumed: ```sh python3 -I scripts/diagnose_docker_torch_import.py \ @@ -255,3 +255,43 @@ python3 -I scripts/diagnose_docker_torch_import.py \ --gpu-device 0 \ --output-dir /home/profsynapse/.local/share/synaptic-qualification/torch-import-r7-20261020/guarded-gpu0-1 ``` + +## r8 single GPU-visible import result + +Before the single r8 start, a fresh clean WSL checkout and the reviewed helper +SHA-256 `95849f03b1133302cef1bc7ceec1e20674897877152c9b35b64af2b86cf718c3` +were authenticated. The trusted closure verifier SHA-256 was +`2304eeb00667954510c34cc4aacaa818532a301fbeabf04c2fb165d6cd1d70b5`. +Release, image, daemon, retained source lock, manifest and closure identities +matched those above. The source and ancestors passed private-custody checks, +no matching writer was observed, and the private mode-0700 output parent had +no `guarded-gpu0-1` child. Windows and WSL GPU queries both identified device +0 as `GPU-ac4398bb-fc15-d722-b39c-2790b4b5f9cc` (RTX 3090) and returned +no active compute-process row. The one-use Nexus admission was saved and read +back before Docker create. + +The closed r8 result is `PASS` at `TORCH_IMPORT`, `gpu_device="0"`, +`gpu_used=null` (device use not measured), and `training_executed=false`. +Container `d41a2c26ee089777b7583e6100e67c5a899eedc7ff2c72b9beddae22cfe669c2` +is retained. Independent Docker inspection found the exact claim label +`e076d416d5d8ea04d97ae245d989db7d1d41d4ef74efe036c3aec723dff57dd2`, +image ID, one device request with `DeviceIDs=["0"]` and `Capabilities=[["gpu"]]`, +2 CPUs, 4 GiB, network `none`, read-only root, UID/GID 1000, all capabilities +dropped, and terminal `exited` with code 0. It ran from +`2026-10-20T15:11:08.249437944Z` to `2026-10-20T15:11:11.665541884Z`. +Events record one claimed create, one start and terminal exit 0; no stop or +deletion occurred. The retained `claim.json`, `events.jsonl` and `result.json` +under `torch-import-r7-20261020/guarded-gpu0-1` have SHA-256 +`a7a85de97044ed8b2381f20be1c5e0846fc9ddb1b0217bde6b1da7b3d95ae89c`, +`05b15a0b21e2dc653572f5f4f51f12e5b98371a65fe216a5b76969efecdc158d`, +and `439d5a19c1b838840ee8cd49085a95b68acfa50e8182e3dceca43b52722794f1`, +respectively. The r2/r4/r6 claims and containers remain unchanged. + +Both guarded r6 CPU and r8 device-0 imports passed in the reduced diagnostic. +Adding device-0 exposure and removing the two visibility suppressors did not +reproduce r2's closed import rejection under those conditions. It does not +recreate the r2 parent/isolated-child environment, Python entry, root/default +capabilities, writable filesystem or 16-GiB limit, and cannot establish or +rule out an interaction with them. The cause of r2 remains unproven. This +consumed diagnostic grants no replay, broader GPU comparison or training +authority. From bdfea4ecd3be7b3f4ccb56bf695f22a8b72457fc Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:29:12 -0400 Subject: [PATCH 06/12] Prepare hash-bound offline worker Torch import boundary --- .agents/skills/fine-tuning/SKILL.md | 2 +- .../fine-tuning/reference/sft-training.md | 14 + .claude/skills/fine-tuning/SKILL.md | 2 +- .../fine-tuning/reference/sft-training.md | 14 + .skills/fine-tuning/SKILL.md | 2 +- .skills/fine-tuning/reference/sft-training.md | 14 + .../docker-torch-import-r2-diagnosis.md | 82 +++++ scripts/diagnose_docker_torch_import.py | 340 +++++++++++++++++- .../test_diagnose_docker_torch_import.py | 116 ++++++ 9 files changed, 570 insertions(+), 16 deletions(-) diff --git a/.agents/skills/fine-tuning/SKILL.md b/.agents/skills/fine-tuning/SKILL.md index 76e63fde..f68017ba 100644 --- a/.agents/skills/fine-tuning/SKILL.md +++ b/.agents/skills/fine-tuning/SKILL.md @@ -57,7 +57,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Check SFT loss masking (tokenizer only) | `python tuner.py doctor sft-mask --dataset-path --model ` | | Inspect native training host candidate | `python tuner.py doctor native-training --json` — read-only platform/Python/package metadata; see `reference/native-local-training.md` | | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | -| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for bounded CPU probes and a separately reviewed guarded device-0 import probe; see `reference/sft-training.md` | +| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for bounded guarded probes and the separately reviewed hash-bound worker-context prefix; see `reference/sft-training.md` | | Native MLX prepared SFT candidate | Internal exact-token batch/loss and runtime comparison only; no native launch CLI or API provider yet; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.agents/skills/fine-tuning/reference/sft-training.md b/.agents/skills/fine-tuning/reference/sft-training.md index 28dff3fd..d2927bd3 100644 --- a/.agents/skills/fine-tuning/reference/sft-training.md +++ b/.agents/skills/fine-tuning/reference/sft-training.md @@ -534,6 +534,20 @@ nor loads a model; a PASS or failure cannot retrospectively diagnose an earlier trainer attempt. See the exact proposed command and remaining environment/user/filesystem gaps in the r2 diagnosis note. +The optional `--worker-context` mode requires independently reviewed +`--expected-workload-sha256`, `--expected-runtime-sha256`, and +`--expected-trainer-sha256` alongside guarded source commitments. It +reconstructs the pinned workload's isolated child environment, launches a +separate `-I` child, loads the authenticated worker closure and guard, and +executes only the authenticated trainer prefix through `import torch`. +The prefix excludes the `UNSLOTH_IMPORT` marker and all later trainer/model +code. It records closed environment, path, file and cwd digests and keeps the +existing claim, output budget, deadline and owned cleanup. Its writable +`/artifacts` is a small private tmpfs with no model weights. Review the exact +workload/source hashes, device scope, host user, caps and filesystem before +each fresh use. A reconstructed child environment is a source-and-plan +deduction; it is not a historical capture of the failed child. + After an intentional worker source change, check and refresh existing manifests with the checked-in maintainers in dependency order: `regenerate_offline_sft_worker_closure.py`, diff --git a/.claude/skills/fine-tuning/SKILL.md b/.claude/skills/fine-tuning/SKILL.md index 76e63fde..f68017ba 100644 --- a/.claude/skills/fine-tuning/SKILL.md +++ b/.claude/skills/fine-tuning/SKILL.md @@ -57,7 +57,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Check SFT loss masking (tokenizer only) | `python tuner.py doctor sft-mask --dataset-path --model ` | | Inspect native training host candidate | `python tuner.py doctor native-training --json` — read-only platform/Python/package metadata; see `reference/native-local-training.md` | | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | -| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for bounded CPU probes and a separately reviewed guarded device-0 import probe; see `reference/sft-training.md` | +| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for bounded guarded probes and the separately reviewed hash-bound worker-context prefix; see `reference/sft-training.md` | | Native MLX prepared SFT candidate | Internal exact-token batch/loss and runtime comparison only; no native launch CLI or API provider yet; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.claude/skills/fine-tuning/reference/sft-training.md b/.claude/skills/fine-tuning/reference/sft-training.md index 28dff3fd..d2927bd3 100644 --- a/.claude/skills/fine-tuning/reference/sft-training.md +++ b/.claude/skills/fine-tuning/reference/sft-training.md @@ -534,6 +534,20 @@ nor loads a model; a PASS or failure cannot retrospectively diagnose an earlier trainer attempt. See the exact proposed command and remaining environment/user/filesystem gaps in the r2 diagnosis note. +The optional `--worker-context` mode requires independently reviewed +`--expected-workload-sha256`, `--expected-runtime-sha256`, and +`--expected-trainer-sha256` alongside guarded source commitments. It +reconstructs the pinned workload's isolated child environment, launches a +separate `-I` child, loads the authenticated worker closure and guard, and +executes only the authenticated trainer prefix through `import torch`. +The prefix excludes the `UNSLOTH_IMPORT` marker and all later trainer/model +code. It records closed environment, path, file and cwd digests and keeps the +existing claim, output budget, deadline and owned cleanup. Its writable +`/artifacts` is a small private tmpfs with no model weights. Review the exact +workload/source hashes, device scope, host user, caps and filesystem before +each fresh use. A reconstructed child environment is a source-and-plan +deduction; it is not a historical capture of the failed child. + After an intentional worker source change, check and refresh existing manifests with the checked-in maintainers in dependency order: `regenerate_offline_sft_worker_closure.py`, diff --git a/.skills/fine-tuning/SKILL.md b/.skills/fine-tuning/SKILL.md index 76e63fde..f68017ba 100644 --- a/.skills/fine-tuning/SKILL.md +++ b/.skills/fine-tuning/SKILL.md @@ -57,7 +57,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Check SFT loss masking (tokenizer only) | `python tuner.py doctor sft-mask --dataset-path --model ` | | Inspect native training host candidate | `python tuner.py doctor native-training --json` — read-only platform/Python/package metadata; see `reference/native-local-training.md` | | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | -| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for bounded CPU probes and a separately reviewed guarded device-0 import probe; see `reference/sft-training.md` | +| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for bounded guarded probes and the separately reviewed hash-bound worker-context prefix; see `reference/sft-training.md` | | Native MLX prepared SFT candidate | Internal exact-token batch/loss and runtime comparison only; no native launch CLI or API provider yet; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.skills/fine-tuning/reference/sft-training.md b/.skills/fine-tuning/reference/sft-training.md index 28dff3fd..d2927bd3 100644 --- a/.skills/fine-tuning/reference/sft-training.md +++ b/.skills/fine-tuning/reference/sft-training.md @@ -534,6 +534,20 @@ nor loads a model; a PASS or failure cannot retrospectively diagnose an earlier trainer attempt. See the exact proposed command and remaining environment/user/filesystem gaps in the r2 diagnosis note. +The optional `--worker-context` mode requires independently reviewed +`--expected-workload-sha256`, `--expected-runtime-sha256`, and +`--expected-trainer-sha256` alongside guarded source commitments. It +reconstructs the pinned workload's isolated child environment, launches a +separate `-I` child, loads the authenticated worker closure and guard, and +executes only the authenticated trainer prefix through `import torch`. +The prefix excludes the `UNSLOTH_IMPORT` marker and all later trainer/model +code. It records closed environment, path, file and cwd digests and keeps the +existing claim, output budget, deadline and owned cleanup. Its writable +`/artifacts` is a small private tmpfs with no model weights. Review the exact +workload/source hashes, device scope, host user, caps and filesystem before +each fresh use. A reconstructed child environment is a source-and-plan +deduction; it is not a historical capture of the failed child. + After an intentional worker source change, check and refresh existing manifests with the checked-in maintainers in dependency order: `regenerate_offline_sft_worker_closure.py`, diff --git a/docs/review/docker-torch-import-r2-diagnosis.md b/docs/review/docker-torch-import-r2-diagnosis.md index f27d4f59..3404408f 100644 --- a/docs/review/docker-torch-import-r2-diagnosis.md +++ b/docs/review/docker-torch-import-r2-diagnosis.md @@ -295,3 +295,85 @@ capabilities, writable filesystem or 16-GiB limit, and cannot establish or rule out an interaction with them. The cause of r2 remains unproven. This consumed diagnostic grants no replay, broader GPU comparison or training authority. + +## r9 prepared worker import boundary — no live execution + +The r8 result leaves the original isolated child invocation and trainer +preamble untested. The next proposed **single** import-only comparison uses +the retained r2 workload, source lock, closure and image. This is preparation +for root review; no new container, device request or probe was started in r9. +It supersedes the earlier idea of varying incidental host settings one at a +time. + +The retained r2 workload is SHA-256 +`1534308c977c2c1cb2b1d80d9cc5c07e31303db63b336686e1c5ccc2644b1064`. +Its pinned `runtime_v1.py` and `train_sft.py` are SHA-256 +`460121aaee3eb83764f67d9307cfb57a3b25c117611d987577b9631b6956a1b3` +and `75c2ab22fbc5dd80b8ddd5e6d0da816d1451cda0cd620c34717336a39a8a7644`. +The helper verifies all three against independently supplied expected hashes +before Docker create and again before start, in addition to the existing +release, daemon, image, source lock, manifest and closure checks. The workload +file is included in source custody checks. The source mount remains read-only; +private host custody and no concurrent writer remain necessary during a run. + +`--worker-context` derives the child environment from the pinned workload +using the exact `runtime_v1.py:1479-1497` transform: start from its declared +variables, remove `PYTHONPATH`, bind the closure and runtime roots, then +override the workload fingerprint, isolated Python flags, HF/cache/offline +values and model-snapshot **path**. It starts a separate `-I` interpreter +with that replacement environment and cwd `/artifacts/run/tmp`. The +model-snapshot path is a string only; no model weights are mounted or loaded. +The child evaluates the authenticated worker module without its `__main__` +entry, loads the closure, verifies origins, installs the owned-module guard +and inserts the worker's trainer-directory/engine path pair. It then compiles +the authenticated `train_sft.py` prefix ending with `import torch`, with +the real trainer `__file__`, `__main__` name and phase callback. The +`UNSLOTH_IMPORT` marker and every later statement are excluded from the +compiled prefix. Marker order, exact Torch/Unsloth adjacency, source hashes +and the last AST statement fail closed on drift. The child returns only +closed category/stage and SHA-256 digests of its preamble environment, cwd, +trainer file path and `sys.path`; the helper checks the environment digest +against its own reconstruction. No raw environment, argv, path, exception or +traceback enters the report. + +The proposed host stays within 2 CPUs, 4 GiB and a 120-second execution +deadline plus bounded owned cleanup. It keeps r8's UID/GID of the staged +source owner, dropped capabilities, read-only root, network `none`, and +device-0 request; it adds only a 64-MiB `/artifacts` tmpfs for private +runtime roots and no model mount. The separately launched child's cwd +matches r2; the outer container still uses `/tmp`. This is closer to r2 +at the child import boundary, but root/default capabilities, 16-GiB memory, +writable root, actual model cache, and the full parent admission/preparation +remain different. The original child environment was not captured, so this +is a source-and-plan reconstruction rather than a historical byte capture. +Even a changed import outcome would narrow a hypothesis, not prove r2's +cause or authorize training. + +The exact **unexecuted** proposal for root review is: + +```sh +python3 -I scripts/diagnose_docker_torch_import.py \ + --docker /usr/bin/docker \ + --docker-config /home/profsynapse/.local/share/synaptic-qualification/docker-sft-stable-image-c/docker-config \ + --release /home/profsynapse/.local/share/synaptic-qualification/docker-sft-stable-image-c/evidence/runtime-release.json \ + --expected-release-sha256 5f65824ceb5e250a58314da0233b9c07f1d63f1b0a39892d9b36af694938bfe6 \ + --daemon-id 24805003-b125-469b-8080-9da4daacd364 \ + --source-root /home/profsynapse/.local/share/synaptic-qualification/docker-public-plan-0d515-20261020/attempt-r2/source/engine \ + --source-lock /home/profsynapse/.local/share/synaptic-qualification/docker-public-plan-0d515-20261020/attempt-r2/material/source-lock.json \ + --expected-source-lock-sha256 d19aecb2d3ca1b7b6fa32a2445c45251e59d8cac2332293c68b73a61cbff4665 \ + --expected-manifest-sha256 3842689403abc5b42e28d58545b88616b81cd8fc738ee5557e97159df65d00ef \ + --expected-closure-digest 8465ff2b7cedb352337e7cedc5bb17cc169892c2eedae0b280569f4a0cde5002 \ + --worker-context \ + --expected-workload-sha256 1534308c977c2c1cb2b1d80d9cc5c07e31303db63b336686e1c5ccc2644b1064 \ + --expected-runtime-sha256 460121aaee3eb83764f67d9307cfb57a3b25c117611d987577b9631b6956a1b3 \ + --expected-trainer-sha256 75c2ab22fbc5dd80b8ddd5e6d0da816d1451cda0cd620c34717336a39a8a7644 \ + --gpu-device 0 \ + --output-dir /home/profsynapse/.local/share/synaptic-qualification/torch-import-r9-20261020/worker-context-gpu0-1 +``` + +The output parent must first be created privately at mode 0700 and the named +child must be absent. Root review must confirm a clean pushed helper/verifier, +the exact remote identities and source custody, no active device-0 compute +job, the proposed UID/caps/tmpfs/readonly filesystem and bounds, then save +and read back a fresh one-use admission before any start. A collision or drift +rejects without renaming or retrying; r2/r4/r6/r8 claims remain consumed. diff --git a/scripts/diagnose_docker_torch_import.py b/scripts/diagnose_docker_torch_import.py index 1ef6a996..d1901da4 100644 --- a/scripts/diagnose_docker_torch_import.py +++ b/scripts/diagnose_docker_torch_import.py @@ -7,6 +7,8 @@ from __future__ import annotations import argparse +import ast +import base64 import hashlib import json import os @@ -34,6 +36,17 @@ _CLI_BUDGET = 8192 _RUN_SECONDS = 120 _HIDDEN_GPU_ENV = ",'CUDA_VISIBLE_DEVICES':'','NVIDIA_VISIBLE_DEVICES':'void'" +_R2_ROOTS = {"engine": "/source/engine", "project": "/source/project", + "artifacts": "/artifacts/run/artifacts", "state": "/artifacts/run/state", + "tracking": "/artifacts/run/tracking", "cache": "/artifacts/run/cache", + "tmp": "/artifacts/run/tmp"} +_R2_ENV_KEYS = frozenset({"PATH", "PYTHONPATH", "PYTHONNOUSERSITE", + "PYTHONSAFEPATH", "HF_HOME", "HF_HUB_OFFLINE", "TRANSFORMERS_CACHE", + "TRANSFORMERS_OFFLINE", "WANDB_DISABLED", "SYNAPTIC_ENGINE_ROOT", + "SYNAPTIC_PROJECT_ROOT", "SYNAPTIC_ARTIFACT_ROOT", "SYNAPTIC_STATE_ROOT", + "SYNAPTIC_TRACKING_ROOT", "SYNAPTIC_CACHE_ROOT", "SYNAPTIC_TMP_ROOT"}) +_ROOT_ENV = {name: "SYNAPTIC_" + name.upper() + "_ROOT" for name in _R2_ROOTS} +_ROOT_ENV["artifacts"] = "SYNAPTIC_ARTIFACT_ROOT" _INSPECT = "{{json .Id}}\n{{json .Name}}\n{{json .Image}}\n{{json .Config.Labels}}\n{{json .State.Status}}\n{{json .State.ExitCode}}" # This fixed program never formats an exception, inspects its attributes, or @@ -104,6 +117,149 @@ os.close(report_fd) """ +# A second -I interpreter receives the reconstructed offline child environment. +# Only the authenticated trainer prefix is compiled. Its final statement is +# import torch; the UNSLOTH_IMPORT marker and every later byte are excluded. +_WORKER_PREFIX_CHILD = """import hashlib,json,os,sys +report_fd=os.dup(1) +null_fd=os.open('/dev/null',os.O_WRONLY) +os.dup2(null_fd,1);os.dup2(null_fd,2);os.close(null_fd) +from pathlib import Path +stage='WORKER_IMPORT' +category='OTHER' +context={} +try: + if sys.flags.isolated!=1 or sys.flags.no_user_site!=1 or not sys.flags.safe_path: + raise ValueError() + if os.environ.get('PYTHONNOUSERSITE')!='1' or os.environ.get('PYTHONSAFEPATH')!='1': + raise ValueError() + root=Path('/source/engine') + import runpy + worker=runpy.run_path(str(root/'tuner/runtime/offline_sft_worker.py'),run_name='__worker_diagnostic__') + stage='WORKER_CLOSURE' + closure=worker['load_offline_sft_worker_environment'](os.environ,engine_root=root) + stage='WORKER_GUARD' + worker['verify_loaded_owned_module_origins'](closure,engine_root=root) + worker['install_owned_module_guard'](closure,engine_root=root) + trainer=root/'Trainers/sft/train_sft.py' + sys.path[:0]=[str(trainer.parent),str(root)] + stage='TRAINER_PREFIX' + with trainer.open('rb') as stream: + source=stream.read(1024*1024+1) + if len(source)>1024*1024: + raise ValueError() + if hashlib.sha256(source).hexdigest()!='EXPECTED_TRAINER_SHA256': + raise ValueError() + lines=source.decode('utf-8').splitlines(keepends=True) + markers=('_mark_packaged_runtime_phase("BOOTSTRAP_ENV")','_mark_packaged_runtime_phase("TORCH_IMPORT")','_mark_packaged_runtime_phase("UNSLOTH_IMPORT")') + positions=[next((i for i,line in enumerate(lines) if line.strip()==marker),-1) for marker in markers] + if positions[0]<0 or not positions[0]1024*1024: + raise ValueError() + if hashlib.sha256(raw).hexdigest()!='EXPECTED_WORKLOAD_SHA256': + raise ValueError() + workload=json.loads(raw) + runtime=workload['execution_source']['runtime'] + roots=runtime['roots'] + if roots!=EXPECTED_ROOTS or runtime['environment']['clear_inherited'] is not True: + raise ValueError() + planned=runtime['environment']['variables'] + if set(planned)!=set(EXPECTED_ENV_KEYS) or any(type(v) is not str for v in planned.values()): + raise ValueError() + environment=dict(planned) + environment.pop('PYTHONPATH') + environment.update({'SYNAPTIC_WORKER_CLOSURE_MANIFEST':'/source/control/offline-sft-worker-v1.json','SYNAPTIC_WORKER_CLOSURE_DIGEST':'EXPECTED_CLOSURE_DIGEST'}) + environment.update({key:value for key,value in ROOT_BINDINGS.items()}) + model=workload['configuration']['document']['model'] + ref=model['ref']; revision=model['revision'] + if not isinstance(ref,str) or not isinstance(revision,str) or not __import__('re').fullmatch('[0-9a-f]{40}(?:[0-9a-f]{24})?',revision) or any(not __import__('re').fullmatch('[A-Za-z0-9][A-Za-z0-9._-]*',part) for part in ref.split('/')) or len(ref.split('/')) not in (1,2): + raise ValueError() + snapshot=roots['cache']+'/model/models--'+'--'.join(ref.split('/'))+'/snapshots/'+revision + environment.update({'SYNAPTIC_WORKLOAD_FINGERPRINT':hashlib.sha256(b'synaptic-training-workload/v1\\0'+raw).hexdigest(),'PYTHONNOUSERSITE':'1','PYTHONSAFEPATH':'1','HF_HOME':roots['cache']+'/huggingface','TRANSFORMERS_CACHE':roots['cache']+'/transformers','HF_HUB_OFFLINE':'1','TRANSFORMERS_OFFLINE':'1','SYNAPTIC_MODEL_SNAPSHOT':snapshot,'WANDB_DISABLED':'true'}) + os.umask(0o077) + Path('/artifacts/run').mkdir(mode=0o700) + for name in ('artifacts','state','tracking','cache','tmp'): + Path(roots[name]).mkdir(mode=0o700) + for name in ('huggingface','transformers'): + (Path(roots['cache'])/name).mkdir(mode=0o700,exist_ok=True) + stage='ISOLATED_CHILD' + child=base64.b64decode('CHILD_SOURCE_B64') + process=subprocess.Popen([sys.executable,'-I','-c',child.decode('utf-8')],cwd=roots['tmp'],env=environment,stdin=subprocess.DEVNULL,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL) + payload=process.stdout.read(4097) + if len(payload)>4096: + process.kill();process.wait() + raise ValueError() + if process.wait()!=0: + raise ValueError() + result=json.loads(payload) + if set(result)!={'schema_version','category','stage','context','boundary'} or result['schema_version']!='synaptic-docker-torch-import-child/v1' or result['boundary']!='BEFORE_UNSLOTH' or result['category'] not in ('PASS','MODULE_NOT_FOUND','IMPORT','OS','RUNTIME','OTHER') or result['stage'] not in ('WORKER_IMPORT','WORKER_CLOSURE','WORKER_GUARD','TRAINER_PREFIX','BOOTSTRAP_ENV','TORCH_IMPORT') or not isinstance(result['context'],dict): + raise ValueError() + if result['category']=='PASS': + if result['stage']!='TORCH_IMPORT' or set(result['context'])!={'environment_sha256','python_isolated','cwd_sha256','file_sha256','sys_path_sha256'} or result['context']['python_isolated'] is not True or any(not isinstance(value,str) or not __import__('re').fullmatch('[0-9a-f]{64}',value) for key,value in result['context'].items() if key.endswith('_sha256')): + raise ValueError() + elif result['context']!={}: + raise ValueError() + category=result['category'];stage=result['stage'];context=result['context'] +except BaseException: + context={} +os.write(report_fd,(json.dumps({'schema_version':'synaptic-docker-torch-import-child/v1','category':category,'stage':stage,'context':context,'boundary':'BEFORE_UNSLOTH'},sort_keys=True,separators=(',',':'))+'\\n').encode('ascii')) +os.close(report_fd) +""" + def _read(path: Path, maximum: int) -> bytes: if path.is_symlink() or not path.is_file() or path.stat().st_size > maximum: @@ -230,6 +386,111 @@ def _source_identity(source_root: Path, source_lock: Path, expected_lock: str, "engine_commit": "0d515ae1ae235d0858802228ce4af8dba8fccd03"} +def _trainer_prefix_boundary(raw: bytes) -> int: + """Validate the fixed marker sequence and return the first excluded line.""" + lines = raw.decode("utf-8").splitlines(keepends=True) + markers = tuple(f'_mark_packaged_runtime_phase("{name}")' for name in + ("BOOTSTRAP_ENV", "TORCH_IMPORT", "UNSLOTH_IMPORT")) + positions = [] + for marker in markers: + matches = [i for i, line in enumerate(lines) if line.strip() == marker] + if len(matches) != 1: + raise ValueError("SOURCE_IDENTITY_INVALID") + positions.append(matches[0]) + if (not positions[0] < positions[1] < positions[2] + or lines[positions[1] + 1].strip() != "import torch # noqa: E402" + or lines[positions[2] + 1].strip() != + "from unsloth import is_bfloat16_supported # noqa: E402"): + raise ValueError("SOURCE_IDENTITY_INVALID") + prefix = "".join(lines[:positions[2]]) + tree = ast.parse(prefix) + if (not isinstance(tree.body[-1], ast.Import) + or [alias.name for alias in tree.body[-1].names] != ["torch"] + or any(isinstance(node, ast.ImportFrom) and node.module is not None and + node.module.split(".")[0] in {"unsloth", "transformers", "trl"} + for node in ast.walk(tree)) + or any(isinstance(node, ast.Import) and + any(alias.name.split(".")[0] in {"unsloth", "transformers", "trl"} + for alias in node.names) for node in ast.walk(tree))): + raise ValueError("SOURCE_IDENTITY_INVALID") + return positions[2] + + +def _worker_context_identity(source_root: Path, expected_workload: str, + expected_runtime: str, expected_trainer: str, + expected_closure: str) -> dict[str, str]: + if any(not re.fullmatch(r"[0-9a-f]{64}", value) for value in + (expected_workload, expected_runtime, expected_trainer, expected_closure)): + raise ValueError("SOURCE_IDENTITY_INVALID") + workload_raw = _read(source_root.parent / "control/workload.json", 1024 * 1024) + runtime_raw = _read(source_root / "Trainers/sft/runtime_v1.py", 1024 * 1024) + trainer_raw = _read(source_root / "Trainers/sft/train_sft.py", 1024 * 1024) + if (hashlib.sha256(workload_raw).hexdigest() != expected_workload + or hashlib.sha256(runtime_raw).hexdigest() != expected_runtime + or hashlib.sha256(trainer_raw).hexdigest() != expected_trainer): + raise ValueError("SOURCE_IDENTITY_INVALID") + workload = json.loads(workload_raw) + runtime = workload["execution_source"]["runtime"] + planned = runtime["environment"]["variables"] + if (runtime["roots"] != _R2_ROOTS + or runtime["environment"]["clear_inherited"] is not True + or set(planned) != _R2_ENV_KEYS + or any(type(value) is not str for value in planned.values()) + or any(planned[_ROOT_ENV[name]] != path for name, path in _R2_ROOTS.items()) + or planned["PYTHONPATH"] != _R2_ROOTS["engine"] + or planned["HF_HUB_OFFLINE"] != "1" + or planned["TRANSFORMERS_OFFLINE"] != "1" + or runtime["interpreter"]["executable"] != "/opt/unsloth-venv/bin/python3"): + raise ValueError("SOURCE_IDENTITY_INVALID") + _trainer_prefix_boundary(trainer_raw) + model = workload["configuration"]["document"]["model"] + ref, revision = model["ref"], model["revision"] + if (type(ref) is not str or type(revision) is not str + or len(ref.split("/")) not in (1, 2) + or any(not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", part) + for part in ref.split("/")) + or not re.fullmatch(r"[0-9a-f]{40}(?:[0-9a-f]{24})?", revision)): + raise ValueError("SOURCE_IDENTITY_INVALID") + child_env = dict(planned) + child_env.pop("PYTHONPATH") + child_env.update({"SYNAPTIC_WORKER_CLOSURE_MANIFEST": + "/source/control/offline-sft-worker-v1.json"}) + child_env["SYNAPTIC_WORKER_CLOSURE_DIGEST"] = expected_closure + child_env.update({_ROOT_ENV[name]: value for name, value in _R2_ROOTS.items()}) + child_env.update({"SYNAPTIC_WORKLOAD_FINGERPRINT": hashlib.sha256( + b"synaptic-training-workload/v1\0" + workload_raw).hexdigest(), + "PYTHONNOUSERSITE": "1", "PYTHONSAFEPATH": "1", + "HF_HOME": _R2_ROOTS["cache"] + "/huggingface", + "TRANSFORMERS_CACHE": _R2_ROOTS["cache"] + "/transformers", + "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", + "SYNAPTIC_MODEL_SNAPSHOT": _R2_ROOTS["cache"] + "/model/models--" + + "--".join(ref.split("/")) + "/snapshots/" + revision, + "WANDB_DISABLED": "true"}) + return {"workload_sha256": expected_workload, + "runtime_sha256": expected_runtime, + "trainer_sha256": expected_trainer, + "child_env_sha256": hashlib.sha256(json.dumps( + child_env, sort_keys=True, separators=(",", ":")).encode()).hexdigest()} + + +def _worker_probe(identity: dict[str, str], closure: str) -> str: + child = _WORKER_PREFIX_CHILD.replace("EXPECTED_TRAINER_SHA256", + identity["trainer_sha256"]) + probe = _WORKER_CONTEXT_PROBE + replacements = {"EXPECTED_WORKLOAD_SHA256": identity["workload_sha256"], + "EXPECTED_CLOSURE_DIGEST": closure, + "EXPECTED_ROOTS": repr(_R2_ROOTS), + "EXPECTED_ENV_KEYS": repr(sorted(_R2_ENV_KEYS)), + "ROOT_BINDINGS": repr({_ROOT_ENV[name]: value + for name, value in _R2_ROOTS.items()}), + "CHILD_SOURCE_B64": base64.b64encode(child.encode()).decode()} + for key, value in replacements.items(): + if probe.count(key) != 1: + raise ValueError("SOURCE_IDENTITY_INVALID") + probe = probe.replace(key, value) + return probe + + def _path_custody(path: Path) -> tuple[tuple[int, int, int], ...]: if not path.is_absolute(): raise ValueError("SOURCE_IDENTITY_INVALID") @@ -336,7 +597,15 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, expected_source_lock_sha256: str | None = None, expected_manifest_sha256: str | None = None, expected_closure_digest: str | None = None, - gpu_device: str | None = None) -> dict[str, object]: + gpu_device: str | None = None, worker_context: bool = False, + expected_workload_sha256: str | None = None, + expected_runtime_sha256: str | None = None, + expected_trainer_sha256: str | None = None) -> dict[str, object]: + worker_values = (expected_workload_sha256, expected_runtime_sha256, + expected_trainer_sha256) + if ((worker_context and (source_root is None or any(value is None for value in worker_values))) + or (not worker_context and any(value is not None for value in worker_values))): + raise ValueError("SOURCE_IDENTITY_INVALID") if gpu_device not in (None, "0") or (gpu_device is not None and source_root is None): raise ValueError("GPU_SCOPE_INVALID") if not re.fullmatch(r"[0-9a-f-]{36}", daemon_id): @@ -374,8 +643,11 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, source_identity = ({} if source_root is None else _source_identity( source_root, source_lock, expected_source_lock_sha256, expected_manifest_sha256, expected_closure_digest)) + context_identity = ({} if not worker_context else _worker_context_identity( + source_root, *worker_values, expected_closure_digest)) custody = (() if source_root is None else tuple(_path_custody(path) for path in - (source_root, source_root.parent / "control/offline-sft-worker-v1.json", source_lock))) + (source_root, source_root.parent / "control/offline-sft-worker-v1.json", source_lock, + *((source_root.parent / "control/workload.json",) if worker_context else ())))) authority = _authority(docker, docker_config) info = _run(authority, ("info", "--format", "{{.ID}}")) if info.stderr or _one_line(info.stdout) != daemon_id: @@ -386,17 +658,21 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, output_dir.mkdir(mode=0o700) nonce = secrets.token_hex(16) name = "synaptic-torch-import-" + nonce - probe = _PROBE if source_root is None else _GUARDED_PROBE.replace( - "EXPECTED_CLOSURE_DIGEST", expected_closure_digest) + probe = (_worker_probe(context_identity, expected_closure_digest) + if worker_context else (_PROBE if source_root is None else + _GUARDED_PROBE.replace("EXPECTED_CLOSURE_DIGEST", expected_closure_digest))) if gpu_device is not None: - if probe.count(_HIDDEN_GPU_ENV) != 1: + if not worker_context and probe.count(_HIDDEN_GPU_ENV) != 1: raise ValueError("GPU_SCOPE_INVALID") # The r2 isolated child did not inherit either visibility variable. # The host device request is separately inspected before start. - probe = probe.replace(_HIDDEN_GPU_ENV, "") + if not worker_context: + probe = probe.replace(_HIDDEN_GPU_ENV, "") claim = {"schema_version": SCHEMA, "daemon_id": daemon_id, "image_id": image, "release_sha256": expected_release_sha256, "source_identity": source_identity, - "name": name, "scope": "bare" if source_root is None else "guarded", + "context_identity": context_identity, + "name": name, "scope": "worker_context" if worker_context else + ("bare" if source_root is None else "guarded"), "helper_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), "probe_sha256": hashlib.sha256(probe.encode("utf-8")).hexdigest(), "docker_file_identity": list(authority[2]), @@ -416,6 +692,8 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, ("--user", f"{source_root.stat().st_uid}:{source_root.stat().st_gid}")) visibility = (("--env=NVIDIA_VISIBLE_DEVICES=void", "--env=CUDA_VISIBLE_DEVICES=") if gpu_device is None else ("--gpus", "device=0")) + artifact_tmpfs = (("--tmpfs", "/artifacts:rw,nosuid,nodev,size=64m,mode=1777") + if worker_context else ()) command = ( "container", "create", "--name", name, "--label", "ai.synapticlabs.torch-import.claim=" + label, "--pull=never", "--network=none", "--log-driver", "json-file", @@ -423,6 +701,7 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, "--read-only", "--cap-drop=ALL", "--runtime=runc", "--security-opt=no-new-privileges", "--pids-limit=64", "--memory=4g", "--cpus=2", "--tmpfs", "/tmp:rw,nosuid,nodev,size=64m,mode=1777", + *artifact_tmpfs, *mount, *user, *visibility, "--workdir=/tmp", "--entrypoint", python, @@ -430,7 +709,8 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, ) if source_root is not None and tuple(_path_custody(path) for path in (source_root, source_root.parent / "control/offline-sft-worker-v1.json", - source_lock)) != custody: + source_lock, *((source_root.parent / "control/workload.json",) + if worker_context else ()))) != custody: raise ValueError("SOURCE_IDENTITY_INVALID") _require_daemon(authority, daemon_id) try: @@ -456,9 +736,13 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, expected_source_lock_sha256, expected_manifest_sha256, expected_closure_digest) != source_identity: raise ValueError("SOURCE_IDENTITY_INVALID") + if worker_context and _worker_context_identity( + source_root, *worker_values, expected_closure_digest) != context_identity: + raise ValueError("SOURCE_IDENTITY_INVALID") if source_root is not None and tuple(_path_custody(path) for path in (source_root, source_root.parent / "control/offline-sft-worker-v1.json", - source_lock)) != custody: + source_lock, *((source_root.parent / "control/workload.json",) + if worker_context else ()))) != custody: raise ValueError("SOURCE_IDENTITY_INVALID") deadline = time.monotonic() + _RUN_SECONDS _append(output_dir, {"phase": "START_CLAIMED", "container_id": cid}) @@ -507,13 +791,31 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, if result.stderr: raise ValueError("CHILD_OUTPUT_INVALID") child = json.loads(_one_line(result.stdout)) - fields = {"schema_version", "category"} if source_root is None else {"schema_version", "category", "stage"} + fields = ({"schema_version", "category", "stage", "context", "boundary"} + if worker_context else ({"schema_version", "category"} if source_root is None + else {"schema_version", "category", "stage"})) if (type(child) is not dict or set(child) != fields or child["schema_version"] != "synaptic-docker-torch-import-child/v1" or child["category"] not in _CATEGORIES - or (source_root is not None and child["stage"] not in {"IMPORT_WORKER", "READ_MANIFEST", + or (worker_context and (child["boundary"] != "BEFORE_UNSLOTH" + or child["stage"] not in {"READ_WORKLOAD", "ISOLATED_CHILD", + "WORKER_IMPORT", "WORKER_CLOSURE", "WORKER_GUARD", + "TRAINER_PREFIX", "BOOTSTRAP_ENV", "TORCH_IMPORT"} + or type(child["context"]) is not dict + or (child["category"] == "PASS" and + (child["stage"] != "TORCH_IMPORT" or + set(child["context"]) != {"environment_sha256", "python_isolated", + "cwd_sha256", "file_sha256", "sys_path_sha256"} + or child["context"]["python_isolated"] is not True)) + or (child["category"] != "PASS" and child["context"] != {}) + or any(type(value) is not str or not re.fullmatch(r"[0-9a-f]{64}", value) + for key, value in child["context"].items() if key.endswith("_sha256")))) + or (source_root is not None and not worker_context and child["stage"] not in {"IMPORT_WORKER", "READ_MANIFEST", "LOAD_CLOSURE", "VERIFY_ORIGINS", "INSTALL_GUARD", "BOOTSTRAP_ENV", "TORCH_IMPORT"})): raise ValueError("CHILD_OUTPUT_INVALID") + if (worker_context and child["category"] == "PASS" + and child["context"]["environment_sha256"] != context_identity["child_env_sha256"]): + raise ValueError("CHILD_OUTPUT_INVALID") if _inspect_owned(authority, cid, name=name, label=label, image=image, daemon_id=daemon_id)[0] not in {"exited", "dead"}: raise ValueError("CONTAINER_STATE_INVALID") @@ -524,11 +826,16 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, "container_id": cid, "image_id": image, "daemon_id": daemon_id, "runtime_release_digest": digest, "training_executed": False, "gpu_used": None if gpu_device is not None else False, - "scope": "bare" if source_root is None else "guarded", + "scope": "worker_context" if worker_context else + ("bare" if source_root is None else "guarded"), "claim_sha256": label, "source_identity": source_identity, + "context_identity": context_identity, "terminal_observed": True} if source_root is not None: report["stage"] = child["stage"] + if worker_context: + report["context"] = child["context"] + report["boundary"] = child["boundary"] if gpu_device is not None: report["gpu_device"] = gpu_device with (output_dir / "result.json").open("x", encoding="ascii") as stream: @@ -549,6 +856,10 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--expected-manifest-sha256") parser.add_argument("--expected-closure-digest") parser.add_argument("--gpu-device") + parser.add_argument("--worker-context", action="store_true") + parser.add_argument("--expected-workload-sha256") + parser.add_argument("--expected-runtime-sha256") + parser.add_argument("--expected-trainer-sha256") args = parser.parse_args(argv) try: report = diagnose(docker=args.docker, docker_config=args.docker_config, @@ -559,7 +870,10 @@ def main(argv: list[str] | None = None) -> int: expected_source_lock_sha256=args.expected_source_lock_sha256, expected_manifest_sha256=args.expected_manifest_sha256, expected_closure_digest=args.expected_closure_digest, - gpu_device=args.gpu_device) + gpu_device=args.gpu_device, worker_context=args.worker_context, + expected_workload_sha256=args.expected_workload_sha256, + expected_runtime_sha256=args.expected_runtime_sha256, + expected_trainer_sha256=args.expected_trainer_sha256) except BaseException as error: reason = error.args[0] if type(error) is ValueError and len(error.args) == 1 else None if type(reason) is not str or reason not in _REASONS: diff --git a/tests/scripts/test_diagnose_docker_torch_import.py b/tests/scripts/test_diagnose_docker_torch_import.py index a65d890a..bdbd05a7 100644 --- a/tests/scripts/test_diagnose_docker_torch_import.py +++ b/tests/scripts/test_diagnose_docker_torch_import.py @@ -4,6 +4,7 @@ import json import shutil import sys +import ast from pathlib import Path import pytest @@ -118,6 +119,121 @@ def _stage_source(root: Path): return flags, document, manifest +def _stage_worker_context(root: Path): + flags, _document, _manifest = _stage_source(root) + engine = flags["source_root"] + variables = {name: "1" for name in diagnostic._R2_ENV_KEYS} + variables.update({diagnostic._ROOT_ENV[name]: value + for name, value in diagnostic._R2_ROOTS.items()}) + variables.update({"PYTHONPATH": "/source/engine", + "PATH": "/opt/unsloth-venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "HF_HOME": "/artifacts/run/cache/huggingface", + "TRANSFORMERS_CACHE": "/artifacts/run/cache/transformers", + "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1"}) + workload = {"execution_source": {"runtime": { + "roots": diagnostic._R2_ROOTS, + "environment": {"clear_inherited": True, "variables": variables}, + "interpreter": {"executable": "/opt/unsloth-venv/bin/python3"}}}, + "configuration": {"document": {"model": { + "ref": "Example/Model", "revision": "a" * 40}}}} + path = engine.parent / "control/workload.json" + path.write_text(json.dumps(workload, sort_keys=True, separators=(",", ":"))) + flags.update({"worker_context": True, + "expected_workload_sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "expected_runtime_sha256": hashlib.sha256( + (engine / "Trainers/sft/runtime_v1.py").read_bytes()).hexdigest(), + "expected_trainer_sha256": hashlib.sha256( + (engine / "Trainers/sft/train_sft.py").read_bytes()).hexdigest()}) + return flags, path + + +def test_worker_prefix_excludes_unsloth_and_all_trainer_calls() -> None: + raw = (ROOT / "Trainers/sft/train_sft.py").read_bytes() + boundary = diagnostic._trainer_prefix_boundary(raw) + prefix = "".join(raw.decode().splitlines(keepends=True)[:boundary]) + tree = ast.parse(prefix) + assert isinstance(tree.body[-1], ast.Import) + assert [alias.name for alias in tree.body[-1].names] == ["torch"] + assert "UNSLOTH_IMPORT" not in prefix + assert "from unsloth import" not in prefix + assert "TRAIN_CALL" not in prefix + assert "MODEL_SNAPSHOT" not in prefix + assert "if __name__" not in prefix + for old, new in (('"TORCH_IMPORT"', '"UNSLOTH_IMPORT"'), + ('import torch # noqa: E402', 'from unsloth import x')): + with pytest.raises(ValueError, match="SOURCE_IDENTITY_INVALID"): + diagnostic._trainer_prefix_boundary(raw.replace(old.encode(), new.encode(), 1)) + + +def test_worker_context_source_and_workload_drift_reject_before_effect(tmp_path: Path, + monkeypatch) -> None: + flags, workload = _stage_worker_context(tmp_path) + docker, config, state = _fake_docker(tmp_path) + monkeypatch.setattr(diagnostic, "_run", lambda *args, **kwargs: pytest.fail("Docker touched")) + workload.write_bytes(workload.read_bytes() + b" ") + with pytest.raises(ValueError, match="SOURCE_IDENTITY_INVALID"): + _call(tmp_path, docker, config, **flags) + assert json.loads(state.read_text())["status"] == "absent" + assert not (tmp_path / "out").exists() + + +def test_worker_context_program_is_isolated_and_excludes_trainer_main(tmp_path: Path) -> None: + flags, _workload = _stage_worker_context(tmp_path) + identity = diagnostic._worker_context_identity( + flags["source_root"], flags["expected_workload_sha256"], + flags["expected_runtime_sha256"], flags["expected_trainer_sha256"], + flags["expected_closure_digest"]) + probe = diagnostic._worker_probe(identity, flags["expected_closure_digest"]) + ast.parse(probe) + ast.parse(diagnostic._WORKER_PREFIX_CHILD.replace( + "EXPECTED_TRAINER_SHA256", flags["expected_trainer_sha256"])) + assert "EXPECTED_" not in probe + assert "CHILD_SOURCE_B64" not in probe + assert "process=subprocess.Popen([sys.executable,'-I','-c'" in probe + assert "runpy.run_path(str(root/'tuner/runtime/offline_sft_worker.py')" in diagnostic._WORKER_PREFIX_CHILD + assert "runpy.run_path(str(trainer)" not in diagnostic._WORKER_PREFIX_CHILD + assert "exec(compile(tree,str(trainer),'exec'),namespace)" in diagnostic._WORKER_PREFIX_CHILD + assert "lines[:positions[2]]" in diagnostic._WORKER_PREFIX_CHILD + assert identity["child_env_sha256"] != hashlib.sha256(b"").hexdigest() + + +def test_worker_context_requires_independent_r2_hashes_before_docker(tmp_path: Path, + monkeypatch) -> None: + flags, _workload = _stage_worker_context(tmp_path) + flags.pop("expected_workload_sha256") + docker, config, state = _fake_docker(tmp_path) + monkeypatch.setattr(diagnostic, "_run", lambda *args, **kwargs: pytest.fail("Docker touched")) + with pytest.raises(ValueError, match="SOURCE_IDENTITY_INVALID"): + _call(tmp_path, docker, config, **flags) + assert json.loads(state.read_text())["status"] == "absent" + assert not (tmp_path / "out").exists() + + +def test_worker_context_rechecks_drift_after_create_without_start(tmp_path: Path, + monkeypatch) -> None: + flags, workload = _stage_worker_context(tmp_path) + docker, config, state = _fake_docker(tmp_path) + original = diagnostic._inspect_owned + changed = False + + def inspect(*args, **kwargs): + nonlocal changed + result = original(*args, **kwargs) + if not changed and result[0] == "created": + workload.write_bytes(workload.read_bytes() + b" ") + changed = True + return result + + monkeypatch.setattr(diagnostic, "_inspect_owned", inspect) + with pytest.raises(ValueError, match="SOURCE_IDENTITY_INVALID"): + _call(tmp_path, docker, config, **flags) + assert json.loads(state.read_text())["status"] == "created" + assert (tmp_path / "out/claim.json").exists() + assert not (tmp_path / "out/result.json").exists() + with pytest.raises(ValueError, match="OUTPUT_INVALID"): + _call(tmp_path, docker, config, **flags) + + def test_real_cli_output_budget_and_timeout(tmp_path: Path) -> None: docker, config, state = _fake_docker(tmp_path) authority = diagnostic._authority(docker, config) From 4b56cdcdb6c4d5f35d5e7af2a03e322f39c50622 Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:52:08 -0400 Subject: [PATCH 07/12] Distinguish supplied and observed worker startup environment --- .../fine-tuning/reference/sft-training.md | 8 +++ .../fine-tuning/reference/sft-training.md | 8 +++ .skills/fine-tuning/reference/sft-training.md | 8 +++ .../docker-torch-import-r2-diagnosis.md | 38 +++++++++++ scripts/diagnose_docker_torch_import.py | 64 +++++++++++++++---- .../test_diagnose_docker_torch_import.py | 52 ++++++++++++++- 6 files changed, 166 insertions(+), 12 deletions(-) diff --git a/.agents/skills/fine-tuning/reference/sft-training.md b/.agents/skills/fine-tuning/reference/sft-training.md index d2927bd3..cd6d69a6 100644 --- a/.agents/skills/fine-tuning/reference/sft-training.md +++ b/.agents/skills/fine-tuning/reference/sft-training.md @@ -534,6 +534,14 @@ nor loads a model; a PASS or failure cannot retrospectively diagnose an earlier trainer attempt. See the exact proposed command and remaining environment/user/filesystem gaps in the r2 diagnosis note. +For the worker-context child, keep supplied environment bytes distinct from +the environment observed after isolated Python startup. Require every +supplied key unchanged; the only reviewed addition is +`LC_CTYPE=C.UTF-8`, recorded as a closed delta with separate digests. +Any other addition or change rejects. Do not reset the observed environment +or reinterpret it as a historical capture of the failed r2 child. See the r10 +repair and exact unexecuted command in the diagnosis note. + The optional `--worker-context` mode requires independently reviewed `--expected-workload-sha256`, `--expected-runtime-sha256`, and `--expected-trainer-sha256` alongside guarded source commitments. It diff --git a/.claude/skills/fine-tuning/reference/sft-training.md b/.claude/skills/fine-tuning/reference/sft-training.md index d2927bd3..cd6d69a6 100644 --- a/.claude/skills/fine-tuning/reference/sft-training.md +++ b/.claude/skills/fine-tuning/reference/sft-training.md @@ -534,6 +534,14 @@ nor loads a model; a PASS or failure cannot retrospectively diagnose an earlier trainer attempt. See the exact proposed command and remaining environment/user/filesystem gaps in the r2 diagnosis note. +For the worker-context child, keep supplied environment bytes distinct from +the environment observed after isolated Python startup. Require every +supplied key unchanged; the only reviewed addition is +`LC_CTYPE=C.UTF-8`, recorded as a closed delta with separate digests. +Any other addition or change rejects. Do not reset the observed environment +or reinterpret it as a historical capture of the failed r2 child. See the r10 +repair and exact unexecuted command in the diagnosis note. + The optional `--worker-context` mode requires independently reviewed `--expected-workload-sha256`, `--expected-runtime-sha256`, and `--expected-trainer-sha256` alongside guarded source commitments. It diff --git a/.skills/fine-tuning/reference/sft-training.md b/.skills/fine-tuning/reference/sft-training.md index d2927bd3..cd6d69a6 100644 --- a/.skills/fine-tuning/reference/sft-training.md +++ b/.skills/fine-tuning/reference/sft-training.md @@ -534,6 +534,14 @@ nor loads a model; a PASS or failure cannot retrospectively diagnose an earlier trainer attempt. See the exact proposed command and remaining environment/user/filesystem gaps in the r2 diagnosis note. +For the worker-context child, keep supplied environment bytes distinct from +the environment observed after isolated Python startup. Require every +supplied key unchanged; the only reviewed addition is +`LC_CTYPE=C.UTF-8`, recorded as a closed delta with separate digests. +Any other addition or change rejects. Do not reset the observed environment +or reinterpret it as a historical capture of the failed r2 child. See the r10 +repair and exact unexecuted command in the diagnosis note. + The optional `--worker-context` mode requires independently reviewed `--expected-workload-sha256`, `--expected-runtime-sha256`, and `--expected-trainer-sha256` alongside guarded source commitments. It diff --git a/docs/review/docker-torch-import-r2-diagnosis.md b/docs/review/docker-torch-import-r2-diagnosis.md index 3404408f..a7e2ddf3 100644 --- a/docs/review/docker-torch-import-r2-diagnosis.md +++ b/docs/review/docker-torch-import-r2-diagnosis.md @@ -377,3 +377,41 @@ the exact remote identities and source custody, no active device-0 compute job, the proposed UID/caps/tmpfs/readonly filesystem and bounds, then save and read back a fresh one-use admission before any start. A collision or drift rejects without renaming or retrying; r2/r4/r6/r8 claims remain consumed. + +## r10 startup-environment evidence repair — no live execution + +Independent review found that the r9 helper compared a digest of the +environment supplied to the isolated child with a digest observed *after* +CPython startup. The retained r2 child plan supplies no locale keys. CPython +may add `LC_CTYPE=C.UTF-8` when coercing the default C locale; its +[3.12 command-line documentation](https://docs.python.org/3.12/using/cmdline.html#envvar-PYTHONCOERCECLOCALE) +describes the insertion, and the +[v3.12.3 preconfiguration source](https://github.com/python/cpython/blob/v3.12.3/Python/preconfig.c) +contains that coercion path. A real local isolated CPython 3.12.15 subprocess +with the fixed no-locale input added exactly this key/value. The pinned image's +CPython 3.12.3 behavior has **not** been observed by executing this proposal. + +The repaired helper passes the exact supplied mapping to the child through a +bounded private stdin pipe as well as `env=`. Before mounted worker code, the +child checks the supplied digest against its independently bound expectation. +At the existing preamble observation point it requires every supplied key +and value unchanged. It accepts either no added keys or exactly +`LC_CTYPE=C.UTF-8`; any other addition or changed supplied value rejects. +It reports separate `environment_supplied_sha256` and +`environment_observed_sha256` plus the closed delta `NONE` or +`LC_CTYPE_C_UTF8`. The host verifies both digests against its own +reconstruction, including the exact allowed locale variant. It never +prints environment values, resets them, or claims to have captured r2's +historical child. The owned lifecycle, 120-second execution deadline, +8-KiB Docker CLI output limit, source commitments and stop-before-Unsloth +boundary are unchanged. + +The proposed command in the r9 section above is unchanged and remains +**unexecuted**. For root delta review, the repaired helper SHA-256 is +`c7543b50c4aa41b91ed8293474b4f1719c69342550515a8d367b06dd3f1fd229`; +the trusted closure verifier remains +`2304eeb00667954510c34cc4aacaa818532a301fbeabf04c2fb165d6cd1d70b5`. +Root must review the pushed commit and exact command, repeat the current +identity/custody/device preflight, and save/read a fresh one-use admission +before any container start. No r10 container, GPU request, ML import or +training effect occurred. diff --git a/scripts/diagnose_docker_torch_import.py b/scripts/diagnose_docker_torch_import.py index d1901da4..45f87109 100644 --- a/scripts/diagnose_docker_torch_import.py +++ b/scripts/diagnose_docker_torch_import.py @@ -125,10 +125,33 @@ null_fd=os.open('/dev/null',os.O_WRONLY) os.dup2(null_fd,1);os.dup2(null_fd,2);os.close(null_fd) from pathlib import Path +def _environment_evidence(supplied,observed): + if type(supplied) is not dict or type(observed) is not dict or 'LC_CTYPE' in supplied: + raise ValueError() + if any(type(key) is not str or type(value) is not str for key,value in supplied.items()): + raise ValueError() + if any(observed.get(key)!=value for key,value in supplied.items()): + raise ValueError() + additions={key:value for key,value in observed.items() if key not in supplied} + if additions=={}: + delta='NONE' + elif additions=={'LC_CTYPE':'C.UTF-8'}: + delta='LC_CTYPE_C_UTF8' + else: + raise ValueError() + return {'environment_supplied_sha256':hashlib.sha256(json.dumps(supplied,sort_keys=True,separators=(',',':')).encode()).hexdigest(), + 'environment_observed_sha256':hashlib.sha256(json.dumps(observed,sort_keys=True,separators=(',',':')).encode()).hexdigest(), + 'startup_env_delta':delta} stage='WORKER_IMPORT' category='OTHER' context={} try: + supplied_raw=sys.stdin.buffer.read(8193) + if not supplied_raw or len(supplied_raw)>8192: + raise ValueError() + supplied_env=json.loads(supplied_raw) + if _environment_evidence(supplied_env,dict(os.environ))['environment_supplied_sha256']!='EXPECTED_SUPPLIED_ENV_SHA256': + raise ValueError() if sys.flags.isolated!=1 or sys.flags.no_user_site!=1 or not sys.flags.safe_path: raise ValueError() if os.environ.get('PYTHONNOUSERSITE')!='1' or os.environ.get('PYTHONSAFEPATH')!='1': @@ -162,9 +185,9 @@ tree=ast.parse(prefix,filename=str(trainer)) if not isinstance(tree.body[-1],ast.Import) or [alias.name for alias in tree.body[-1].names]!=['torch']: raise ValueError() - if any(isinstance(node,(ast.Import,ast.ImportFrom)) and any(alias.name.split('.')[0] in ('unsloth','transformers','trl') for alias in node.names) for node in ast.walk(tree)): + if any(isinstance(node,ast.Import) and any(alias.name.split('.')[0] in ('unsloth','transformers','trl') for alias in node.names) for node in ast.walk(tree)) or any(isinstance(node,ast.ImportFrom) and node.module is not None and node.module.split('.')[0] in ('unsloth','transformers','trl') for node in ast.walk(tree)): raise ValueError() - context['environment_sha256']=hashlib.sha256(json.dumps(dict(os.environ),sort_keys=True,separators=(',',':')).encode()).hexdigest() + context.update(_environment_evidence(supplied_env,dict(os.environ))) context['python_isolated']=True context['cwd_sha256']=hashlib.sha256(os.getcwd().encode()).hexdigest() context['file_sha256']=hashlib.sha256(str(trainer).encode()).hexdigest() @@ -238,7 +261,12 @@ def mark(value): (Path(roots['cache'])/name).mkdir(mode=0o700,exist_ok=True) stage='ISOLATED_CHILD' child=base64.b64decode('CHILD_SOURCE_B64') - process=subprocess.Popen([sys.executable,'-I','-c',child.decode('utf-8')],cwd=roots['tmp'],env=environment,stdin=subprocess.DEVNULL,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL) + supplied_raw=json.dumps(environment,sort_keys=True,separators=(',',':')).encode() + if len(supplied_raw)>8192: + raise ValueError() + process=subprocess.Popen([sys.executable,'-I','-c',child.decode('utf-8')],cwd=roots['tmp'],env=environment,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL) + process.stdin.write(supplied_raw) + process.stdin.close() payload=process.stdout.read(4097) if len(payload)>4096: process.kill();process.wait() @@ -249,7 +277,7 @@ def mark(value): if set(result)!={'schema_version','category','stage','context','boundary'} or result['schema_version']!='synaptic-docker-torch-import-child/v1' or result['boundary']!='BEFORE_UNSLOTH' or result['category'] not in ('PASS','MODULE_NOT_FOUND','IMPORT','OS','RUNTIME','OTHER') or result['stage'] not in ('WORKER_IMPORT','WORKER_CLOSURE','WORKER_GUARD','TRAINER_PREFIX','BOOTSTRAP_ENV','TORCH_IMPORT') or not isinstance(result['context'],dict): raise ValueError() if result['category']=='PASS': - if result['stage']!='TORCH_IMPORT' or set(result['context'])!={'environment_sha256','python_isolated','cwd_sha256','file_sha256','sys_path_sha256'} or result['context']['python_isolated'] is not True or any(not isinstance(value,str) or not __import__('re').fullmatch('[0-9a-f]{64}',value) for key,value in result['context'].items() if key.endswith('_sha256')): + if result['stage']!='TORCH_IMPORT' or set(result['context'])!={'environment_supplied_sha256','environment_observed_sha256','startup_env_delta','python_isolated','cwd_sha256','file_sha256','sys_path_sha256'} or result['context']['python_isolated'] is not True or result['context']['startup_env_delta'] not in ('NONE','LC_CTYPE_C_UTF8') or any(not isinstance(value,str) or not __import__('re').fullmatch('[0-9a-f]{64}',value) for key,value in result['context'].items() if key.endswith('_sha256')): raise ValueError() elif result['context']!={}: raise ValueError() @@ -466,16 +494,22 @@ def _worker_context_identity(source_root: Path, expected_workload: str, "SYNAPTIC_MODEL_SNAPSHOT": _R2_ROOTS["cache"] + "/model/models--" + "--".join(ref.split("/")) + "/snapshots/" + revision, "WANDB_DISABLED": "true"}) + child_env_lc_ctype = dict(child_env) + child_env_lc_ctype["LC_CTYPE"] = "C.UTF-8" return {"workload_sha256": expected_workload, "runtime_sha256": expected_runtime, "trainer_sha256": expected_trainer, "child_env_sha256": hashlib.sha256(json.dumps( - child_env, sort_keys=True, separators=(",", ":")).encode()).hexdigest()} + child_env, sort_keys=True, separators=(",", ":")).encode()).hexdigest(), + "child_env_lc_ctype_sha256": hashlib.sha256(json.dumps( + child_env_lc_ctype, sort_keys=True, separators=(",", ":")).encode()).hexdigest()} def _worker_probe(identity: dict[str, str], closure: str) -> str: child = _WORKER_PREFIX_CHILD.replace("EXPECTED_TRAINER_SHA256", identity["trainer_sha256"]) + child = child.replace("EXPECTED_SUPPLIED_ENV_SHA256", + identity["child_env_sha256"]) probe = _WORKER_CONTEXT_PROBE replacements = {"EXPECTED_WORKLOAD_SHA256": identity["workload_sha256"], "EXPECTED_CLOSURE_DIGEST": closure, @@ -804,18 +838,26 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, or type(child["context"]) is not dict or (child["category"] == "PASS" and (child["stage"] != "TORCH_IMPORT" or - set(child["context"]) != {"environment_sha256", "python_isolated", - "cwd_sha256", "file_sha256", "sys_path_sha256"} - or child["context"]["python_isolated"] is not True)) + set(child["context"]) != {"environment_supplied_sha256", + "environment_observed_sha256", "startup_env_delta", + "python_isolated", "cwd_sha256", "file_sha256", "sys_path_sha256"} + or child["context"]["python_isolated"] is not True + or child["context"]["startup_env_delta"] not in + {"NONE", "LC_CTYPE_C_UTF8"})) or (child["category"] != "PASS" and child["context"] != {}) or any(type(value) is not str or not re.fullmatch(r"[0-9a-f]{64}", value) for key, value in child["context"].items() if key.endswith("_sha256")))) or (source_root is not None and not worker_context and child["stage"] not in {"IMPORT_WORKER", "READ_MANIFEST", "LOAD_CLOSURE", "VERIFY_ORIGINS", "INSTALL_GUARD", "BOOTSTRAP_ENV", "TORCH_IMPORT"})): raise ValueError("CHILD_OUTPUT_INVALID") - if (worker_context and child["category"] == "PASS" - and child["context"]["environment_sha256"] != context_identity["child_env_sha256"]): - raise ValueError("CHILD_OUTPUT_INVALID") + if worker_context and child["category"] == "PASS": + context = child["context"] + expected_observed = (context_identity["child_env_sha256"] + if context["startup_env_delta"] == "NONE" + else context_identity["child_env_lc_ctype_sha256"]) + if (context["environment_supplied_sha256"] != context_identity["child_env_sha256"] + or context["environment_observed_sha256"] != expected_observed): + raise ValueError("CHILD_OUTPUT_INVALID") if _inspect_owned(authority, cid, name=name, label=label, image=image, daemon_id=daemon_id)[0] not in {"exited", "dead"}: raise ValueError("CONTAINER_STATE_INVALID") diff --git a/tests/scripts/test_diagnose_docker_torch_import.py b/tests/scripts/test_diagnose_docker_torch_import.py index bdbd05a7..ac8cae41 100644 --- a/tests/scripts/test_diagnose_docker_torch_import.py +++ b/tests/scripts/test_diagnose_docker_torch_import.py @@ -3,6 +3,7 @@ import hashlib import json import shutil +import subprocess import sys import ast from pathlib import Path @@ -186,7 +187,8 @@ def test_worker_context_program_is_isolated_and_excludes_trainer_main(tmp_path: probe = diagnostic._worker_probe(identity, flags["expected_closure_digest"]) ast.parse(probe) ast.parse(diagnostic._WORKER_PREFIX_CHILD.replace( - "EXPECTED_TRAINER_SHA256", flags["expected_trainer_sha256"])) + "EXPECTED_TRAINER_SHA256", flags["expected_trainer_sha256"]).replace( + "EXPECTED_SUPPLIED_ENV_SHA256", identity["child_env_sha256"])) assert "EXPECTED_" not in probe assert "CHILD_SOURCE_B64" not in probe assert "process=subprocess.Popen([sys.executable,'-I','-c'" in probe @@ -195,6 +197,54 @@ def test_worker_context_program_is_isolated_and_excludes_trainer_main(tmp_path: assert "exec(compile(tree,str(trainer),'exec'),namespace)" in diagnostic._WORKER_PREFIX_CHILD assert "lines[:positions[2]]" in diagnostic._WORKER_PREFIX_CHILD assert identity["child_env_sha256"] != hashlib.sha256(b"").hexdigest() + assert identity["child_env_lc_ctype_sha256"] != identity["child_env_sha256"] + + +@pytest.mark.parametrize("mutation,expected", [ + ("none", "LC_CTYPE_C_UTF8"), + ("no_coercion", "NONE"), + ("extra", "REJECTED"), + ("changed", "REJECTED"), + ("locale_value", "REJECTED"), +]) +def test_real_isolated_python_startup_environment_evidence(mutation: str, + expected: str) -> None: + """Run the exact child classifier after a real -I interpreter startup.""" + source = diagnostic._WORKER_PREFIX_CHILD + tree = ast.parse(source) + function = next(node for node in tree.body + if isinstance(node, ast.FunctionDef) and + node.name == "_environment_evidence") + lines = source.splitlines(keepends=True) + classifier = "".join(lines[function.lineno - 1:function.end_lineno]) + program = ("import hashlib,json,os,sys\n" + classifier + + "\nsupplied=json.loads(sys.stdin.buffer.read(8193))\n" + + ("os.environ['UNEXPECTED_DIAGNOSTIC_KEY']='x'\n" if mutation == "extra" else "") + + ("os.environ['PATH']='/tampered'\n" if mutation == "changed" else "") + + ("os.environ['LC_CTYPE']='C.utf8'\n" if mutation == "locale_value" else "") + + "try:\n evidence=_environment_evidence(supplied,dict(os.environ))\n" + "except BaseException:\n print(json.dumps({'status':'REJECTED'}))\n" + "else:\n print(json.dumps({'status':'PASS','delta':evidence['startup_env_delta']," + "'supplied_sha256':evidence['environment_supplied_sha256']," + "'observed_sha256':evidence['environment_observed_sha256']}))\n") + supplied = {"PATH": "/usr/bin:/bin", "PYTHONNOUSERSITE": "1", + "PYTHONSAFEPATH": "1"} + if mutation == "no_coercion": + supplied["LC_ALL"] = "C" + result = subprocess.run([sys.executable, "-I", "-c", program], + input=json.dumps(supplied, sort_keys=True, separators=(",", ":")), + text=True, capture_output=True, env=supplied, timeout=5, check=True) + assert result.stderr == "" + report = json.loads(result.stdout) + if expected == "REJECTED": + assert report == {"status": "REJECTED"} + else: + assert report["status"] == "PASS" + assert report["delta"] == expected + expected_sha = hashlib.sha256(json.dumps( + supplied, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + assert report["supplied_sha256"] == expected_sha + assert (report["observed_sha256"] == expected_sha) == (expected == "NONE") def test_worker_context_requires_independent_r2_hashes_before_docker(tmp_path: Path, From 60ba62245813edd1b80e14ef2bccd9f325544752 Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:00:24 -0400 Subject: [PATCH 08/12] Document bounded r11 worker-context Torch import result --- .../docker-torch-import-r2-diagnosis.md | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/docs/review/docker-torch-import-r2-diagnosis.md b/docs/review/docker-torch-import-r2-diagnosis.md index a7e2ddf3..a32427c2 100644 --- a/docs/review/docker-torch-import-r2-diagnosis.md +++ b/docs/review/docker-torch-import-r2-diagnosis.md @@ -415,3 +415,68 @@ Root must review the pushed commit and exact command, repeat the current identity/custody/device preflight, and save/read a fresh one-use admission before any container start. No r10 container, GPU request, ML import or training effect occurred. + +## r11 single worker-context import result + +Root accepted the r10 repair and authorized exactly one run of the unchanged +r9 command above. Before Docker effects, the fresh private mode-0700 remote +parent `torch-import-r9-20261020` had no `worker-context-gpu0-1` child. +Its clean detached checkout matched pushed +`4b56cdcdb6c4d5f35d5e7af2a03e322f39c50622`; helper SHA-256 was +`c7543b50c4aa41b91ed8293474b4f1719c69342550515a8d367b06dd3f1fd229` +and trusted closure verifier SHA-256 was +`2304eeb00667954510c34cc4aacaa818532a301fbeabf04c2fb165d6cd1d70b5`. +The exact release, source lock, manifest, closure, workload, runtime and trainer +hashes in the command matched read-only remote validation. The explicit +Docker daemon and image IDs matched. Source files and ancestors had private +owner/root custody, no symlink or group/world write; a process/descriptor +scan found no matching active preparer or open writer. This does not +atomically freeze host files, so private custody remains a scope assumption. +Windows and WSL reported no active compute-process rows for device 0 +(`GPU-ac4398bb-fc15-d722-b39c-2790b4b5f9cc`, RTX 3090). The exact +command and preflight were saved and read back in the Nexus one-use state +`Docker Torch r11 worker-context one-use admission — 2026-10-20` before +start. An immediate checkout/source/output/GPU recheck passed. + +The single helper invocation returned closed `PASS` at `TORCH_IMPORT` with +`boundary=BEFORE_UNSLOTH`, `scope=worker_context`, +`training_executed=false` and `gpu_used=null` (actual device use was not +measured). The supplied child environment SHA-256 was +`1bb0e06d9708dd6fd637f59eaf134dcbe8165f43e120a0b94daad6c79a63699c`. +The observed preamble environment SHA-256 was +`a1eeabbf01741b374ca4e410d62119cf8d882b97d33da962257ec7570bdcb08c`, +with the only accepted startup delta `LC_CTYPE_C_UTF8`. The reported +cwd, trainer `__file__` and `sys.path` digests were respectively +`55846bb91af8c8ccb1548b812885d0ca9d413863279b9030c6a6a1b2327d00a6`, +`4863bf5f38bf9e9cf9b2c8d44473326e77b6aa2c55d9fe039fd20092bb68ad9a`, +and `fbb8a8e665063c6fa0c114cb5edc272b4ecdbbab8c521dac9653a2803c1c0a0d`. +No raw environment, argv or exception text was published. + +Container +`d829fc00a2f9a48437bae1ccfd562d7846b97228b4c526759928fe98c9a618cd` +is retained with exact claim label +`db3afc2a9434fadb65ecaf38d97f26b7e1f7039ea66265554ab5615071209bc4`. +Independent Docker inspection confirmed image/daemon ownership, the single +GPU device-0 request, 2 CPUs, 4 GiB, 64 pids, network `none`, read-only root, +all capabilities dropped, UID/GID 1000, a read-only `/source` mount, private +64-MiB `/artifacts` tmpfs, and no model mount. It started at +`2026-10-20T15:58:15.445432531Z` and finished at +`2026-10-20T15:58:19.681177983Z`, independently observed `exited` with +code 0. Events contain one `CREATE_CLAIMED`, `CREATED`, +`START_CLAIMED`, `STARTED` and `TERMINAL_OBSERVED`; no stop or deletion +was needed. The private `worker-context-gpu0-1` evidence has SHA-256: + +| File | SHA-256 | +| --- | --- | +| `claim.json` | `e65eed000fa69e18d387d18484b0055eff1cf4a9dbb349d645e311a25b5fe99a` | +| `events.jsonl` | `e5a7460c94e94b90f1c3142ed505e34c5bf2c8854447c26ad74b0b840b9bddb1` | +| `result.json` | `1d4e4a9d2e2d9f43a2250b6634c63c270852e0b2babd027d61779f30306e054d` | + +Read-only inspection still found the earlier r2/r4/r6/r8 containers retained +in their previous terminal states. The r11 claim is consumed; no second probe +or training followed. This PASS establishes Torch import through the +authenticated, source-derived child environment and trainer prefix under the +specified reduced host context. It is not a capture of r2's historical child +and leaves root/default capabilities, 16-GiB memory, writable root, actual +model cache and full parent preparation different. The r2 failure cause +remains unproven. From 14f44a3ef389c9c04225b058626bb3cfbba61d45 Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:20:30 -0400 Subject: [PATCH 09/12] Prepare bounded worker-entry Torch import comparison --- .agents/skills/fine-tuning/SKILL.md | 2 +- .../fine-tuning/reference/sft-training.md | 19 ++ .claude/skills/fine-tuning/SKILL.md | 2 +- .../fine-tuning/reference/sft-training.md | 19 ++ .skills/fine-tuning/SKILL.md | 2 +- .skills/fine-tuning/reference/sft-training.md | 19 ++ .../docker-torch-import-r2-diagnosis.md | 87 ++++++ scripts/diagnose_docker_torch_import.py | 262 ++++++++++++++++-- .../test_diagnose_docker_torch_import.py | 178 ++++++++++++ 9 files changed, 564 insertions(+), 26 deletions(-) diff --git a/.agents/skills/fine-tuning/SKILL.md b/.agents/skills/fine-tuning/SKILL.md index f68017ba..1172feb9 100644 --- a/.agents/skills/fine-tuning/SKILL.md +++ b/.agents/skills/fine-tuning/SKILL.md @@ -57,7 +57,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Check SFT loss masking (tokenizer only) | `python tuner.py doctor sft-mask --dataset-path --model ` | | Inspect native training host candidate | `python tuner.py doctor native-training --json` — read-only platform/Python/package metadata; see `reference/native-local-training.md` | | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | -| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for bounded guarded probes and the separately reviewed hash-bound worker-context prefix; see `reference/sft-training.md` | +| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for bounded guarded, worker-context prefix, and worker-entry probes; each live command needs exact review; see `reference/sft-training.md` | | Native MLX prepared SFT candidate | Internal exact-token batch/loss and runtime comparison only; no native launch CLI or API provider yet; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.agents/skills/fine-tuning/reference/sft-training.md b/.agents/skills/fine-tuning/reference/sft-training.md index cd6d69a6..dfddf7ba 100644 --- a/.agents/skills/fine-tuning/reference/sft-training.md +++ b/.agents/skills/fine-tuning/reference/sft-training.md @@ -556,6 +556,25 @@ workload/source hashes, device scope, host user, caps and filesystem before each fresh use. A reconstructed child environment is a source-and-plan deduction; it is not a historical capture of the failed child. +`--worker-context --worker-entry` selects a separate, still import-only +diagnostic. It reconstructs the pinned runtime's trainer argument vector, +including a one-byte sealed stand-in for the prepared dataset descriptor, +then invokes the authenticated worker's real `main()` and nested +`runpy.run_path` trainer entry. A source-bound line trace raises at the +`UNSLOTH_IMPORT` marker after Torch import, while a narrow import-call and +module-finder barrier prevents Unsloth entry even if tracing is disabled or +that module name was cached. A PASS +requires the worker's closed stop label and real trainer `__main__` module +binding. The worker still performs its own argument, closure and import-guard +checks. It records closed environment digests before and after Torch import, +separate from the supplied and interpreter-startup digests. The diagnostic +starts through `-I -c` to install that boundary, so it +does not reproduce the original direct `-I` script startup. The sealed data +byte, reduced filesystem and absent model weights also differ from the failed +attempt. Review and admit a fresh one-use command before any live execution; +do not infer the historical failure cause from either result. See the r12 +proposal in the diagnosis note. + After an intentional worker source change, check and refresh existing manifests with the checked-in maintainers in dependency order: `regenerate_offline_sft_worker_closure.py`, diff --git a/.claude/skills/fine-tuning/SKILL.md b/.claude/skills/fine-tuning/SKILL.md index f68017ba..1172feb9 100644 --- a/.claude/skills/fine-tuning/SKILL.md +++ b/.claude/skills/fine-tuning/SKILL.md @@ -57,7 +57,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Check SFT loss masking (tokenizer only) | `python tuner.py doctor sft-mask --dataset-path --model ` | | Inspect native training host candidate | `python tuner.py doctor native-training --json` — read-only platform/Python/package metadata; see `reference/native-local-training.md` | | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | -| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for bounded guarded probes and the separately reviewed hash-bound worker-context prefix; see `reference/sft-training.md` | +| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for bounded guarded, worker-context prefix, and worker-entry probes; each live command needs exact review; see `reference/sft-training.md` | | Native MLX prepared SFT candidate | Internal exact-token batch/loss and runtime comparison only; no native launch CLI or API provider yet; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.claude/skills/fine-tuning/reference/sft-training.md b/.claude/skills/fine-tuning/reference/sft-training.md index cd6d69a6..dfddf7ba 100644 --- a/.claude/skills/fine-tuning/reference/sft-training.md +++ b/.claude/skills/fine-tuning/reference/sft-training.md @@ -556,6 +556,25 @@ workload/source hashes, device scope, host user, caps and filesystem before each fresh use. A reconstructed child environment is a source-and-plan deduction; it is not a historical capture of the failed child. +`--worker-context --worker-entry` selects a separate, still import-only +diagnostic. It reconstructs the pinned runtime's trainer argument vector, +including a one-byte sealed stand-in for the prepared dataset descriptor, +then invokes the authenticated worker's real `main()` and nested +`runpy.run_path` trainer entry. A source-bound line trace raises at the +`UNSLOTH_IMPORT` marker after Torch import, while a narrow import-call and +module-finder barrier prevents Unsloth entry even if tracing is disabled or +that module name was cached. A PASS +requires the worker's closed stop label and real trainer `__main__` module +binding. The worker still performs its own argument, closure and import-guard +checks. It records closed environment digests before and after Torch import, +separate from the supplied and interpreter-startup digests. The diagnostic +starts through `-I -c` to install that boundary, so it +does not reproduce the original direct `-I` script startup. The sealed data +byte, reduced filesystem and absent model weights also differ from the failed +attempt. Review and admit a fresh one-use command before any live execution; +do not infer the historical failure cause from either result. See the r12 +proposal in the diagnosis note. + After an intentional worker source change, check and refresh existing manifests with the checked-in maintainers in dependency order: `regenerate_offline_sft_worker_closure.py`, diff --git a/.skills/fine-tuning/SKILL.md b/.skills/fine-tuning/SKILL.md index f68017ba..1172feb9 100644 --- a/.skills/fine-tuning/SKILL.md +++ b/.skills/fine-tuning/SKILL.md @@ -57,7 +57,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Check SFT loss masking (tokenizer only) | `python tuner.py doctor sft-mask --dataset-path --model ` | | Inspect native training host candidate | `python tuner.py doctor native-training --json` — read-only platform/Python/package metadata; see `reference/native-local-training.md` | | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | -| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for bounded guarded probes and the separately reviewed hash-bound worker-context prefix; see `reference/sft-training.md` | +| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for bounded guarded, worker-context prefix, and worker-entry probes; each live command needs exact review; see `reference/sft-training.md` | | Native MLX prepared SFT candidate | Internal exact-token batch/loss and runtime comparison only; no native launch CLI or API provider yet; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.skills/fine-tuning/reference/sft-training.md b/.skills/fine-tuning/reference/sft-training.md index cd6d69a6..dfddf7ba 100644 --- a/.skills/fine-tuning/reference/sft-training.md +++ b/.skills/fine-tuning/reference/sft-training.md @@ -556,6 +556,25 @@ workload/source hashes, device scope, host user, caps and filesystem before each fresh use. A reconstructed child environment is a source-and-plan deduction; it is not a historical capture of the failed child. +`--worker-context --worker-entry` selects a separate, still import-only +diagnostic. It reconstructs the pinned runtime's trainer argument vector, +including a one-byte sealed stand-in for the prepared dataset descriptor, +then invokes the authenticated worker's real `main()` and nested +`runpy.run_path` trainer entry. A source-bound line trace raises at the +`UNSLOTH_IMPORT` marker after Torch import, while a narrow import-call and +module-finder barrier prevents Unsloth entry even if tracing is disabled or +that module name was cached. A PASS +requires the worker's closed stop label and real trainer `__main__` module +binding. The worker still performs its own argument, closure and import-guard +checks. It records closed environment digests before and after Torch import, +separate from the supplied and interpreter-startup digests. The diagnostic +starts through `-I -c` to install that boundary, so it +does not reproduce the original direct `-I` script startup. The sealed data +byte, reduced filesystem and absent model weights also differ from the failed +attempt. Review and admit a fresh one-use command before any live execution; +do not infer the historical failure cause from either result. See the r12 +proposal in the diagnosis note. + After an intentional worker source change, check and refresh existing manifests with the checked-in maintainers in dependency order: `regenerate_offline_sft_worker_closure.py`, diff --git a/docs/review/docker-torch-import-r2-diagnosis.md b/docs/review/docker-torch-import-r2-diagnosis.md index a32427c2..cf6785a8 100644 --- a/docs/review/docker-torch-import-r2-diagnosis.md +++ b/docs/review/docker-torch-import-r2-diagnosis.md @@ -480,3 +480,90 @@ specified reduced host context. It is not a capture of r2's historical child and leaves root/default capabilities, 16-GiB memory, writable root, actual model cache and full parent preparation different. The r2 failure cause remains unproven. + +## r12 worker-entry comparison and proposed import-only probe — unexecuted + +Read-only inspection of the retained r2 workload and containers on +2026-10-20 found a prepared dataset reference; the r2 parent source therefore +passes one sealed prepared-dataset descriptor to its isolated worker. The +retained workload remains SHA-256 +`1534308c977c2c1cb2b1d80d9cc5c07e31303db63b336686e1c5ccc2644b1064`. +Its planned environment is bound by that workload, but no historical child +environment or complete child argv was captured. Source inspection establishes +how `runtime_v1.py` builds them, not their actual r2 process bytes. +Read-only execution of the hash-pinned runtime's argument appender against +the hash-pinned workload produced 34 appended tokens with SHA-256 +`7d2de6641ff52cd26c4515b2c0f65bed94b9eb11ee8ae02b0910596b26fd5937`; +this is a source/plan reconstruction, not a retained child argv digest. + +| Boundary | Retained r2 source/container | Consumed r11 probe | Proposed r12 worker entry | +| --- | --- | --- | --- | +| Isolated launch | Parent calls `/opt/unsloth-venv/bin/python3 -I` on the worker script with trainer arguments | Separate `-I -c` child; evaluates worker under a diagnostic name, skipping its main and argument validation | Separate `-I -c` child enters the authenticated worker with `runpy.run_path(..., run_name="__main__")`; worker runs its real admission and nested trainer `runpy.run_path` | +| Trainer module | Worker sets full trainer argv and `runpy` temporarily binds trainer as `sys.modules['__main__']` | Compiled AST prefix executes in an unattached dictionary with only the trainer filename in argv | Trace checks trainer filename, reconstructed argv and actual `sys.modules['__main__']` namespace, then stops at the authenticated `UNSLOTH_IMPORT` marker | +| Environment | Parent replaces inherited environment after removing `PYTHONPATH`; r2 process bytes were not retained | Source/plan reconstruction, supplied through a private stdin pipe | Same source/plan reconstruction, with separate supplied and observed digests; stdin is `/dev/null` | +| Descriptors and stdio | Prepared input gives one sealed `pass_fds` descriptor; stdin `/dev/null`, stdout/stderr exclusive regular files | No prepared descriptor; stdout pipe and stderr `/dev/null` | One sealed one-byte **synthetic** descriptor and `pass_fds`; stdout/stderr private exclusive files under the 64-MiB artifacts tmpfs | +| Container and data | Root, default capabilities, writable root, 16 GiB; `/source` read-only, `/artifacts` writable and populated with model/cache/prepared state; device 0 | Source-owner UID/GID, caps dropped, read-only root, 4 GiB, private artifacts tmpfs, no model or dataset bytes; device 0 | Keeps those reduced r11 host settings and device 0; no model or historical dataset bytes mounted | + +The new `--worker-context --worker-entry` mode uses the pinned runtime source's +`_append_sft_arguments` and retained workload fields to reconstruct the +trainer flags and values, while substituting a sealed one-byte descriptor for +the historical prepared data. The worker's own argument validator, closure +check, origin verification and import guard execute. The complete authenticated +trainer source runs through `import torch`; a line trace raises at the unique +`UNSLOTH_IMPORT` marker, before `from unsloth ...`. A narrow import-call and +module-finder barrier prevents Unsloth entry if tracing is disabled, even if +the module name was cached. +The worker's closed `phase=TRAINER category=OTHER milestone=TORCH_IMPORT` stop +label, real trainer `__main__` binding and no barrier activation are required +for `PASS`. It records environment digests immediately before Torch import and +after it returns, alongside the separate supplied and interpreter-startup +digests; no environment values are published. Any worker failure reports only +fixed stage/category labels. The +helper retains source, release, image and daemon checks, claim-before-create, +8-KiB Docker CLI output budget, 120-second execution deadline and exactly-once +owned cleanup. It does not call trainer main, load weights or train. + +The wrapper and trace/import barrier are diagnostic additions that can affect +timing or a transitive Unsloth import; r2 used direct +`-I` worker script startup. The synthetic dataset content, reduced host +privileges/filesystem and absent model cache remain differences. A PASS or +failure would test this narrower worker-entry hypothesis under those reduced +conditions and could not retroactively identify r2's cause. No container, +Docker start/exec, device request or ML import occurred while preparing r12. + +The **unexecuted** proposed single command, requiring root review and a new +one-use admission, is: + +```sh +python3 -I scripts/diagnose_docker_torch_import.py \ + --docker /usr/bin/docker \ + --docker-config /home/profsynapse/.local/share/synaptic-qualification/docker-sft-stable-image-c/docker-config \ + --release /home/profsynapse/.local/share/synaptic-qualification/docker-sft-stable-image-c/evidence/runtime-release.json \ + --expected-release-sha256 5f65824ceb5e250a58314da0233b9c07f1d63f1b0a39892d9b36af694938bfe6 \ + --daemon-id 24805003-b125-469b-8080-9da4daacd364 \ + --source-root /home/profsynapse/.local/share/synaptic-qualification/docker-public-plan-0d515-20261020/attempt-r2/source/engine \ + --source-lock /home/profsynapse/.local/share/synaptic-qualification/docker-public-plan-0d515-20261020/attempt-r2/material/source-lock.json \ + --expected-source-lock-sha256 d19aecb2d3ca1b7b6fa32a2445c45251e59d8cac2332293c68b73a61cbff4665 \ + --expected-manifest-sha256 3842689403abc5b42e28d58545b88616b81cd8fc738ee5557e97159df65d00ef \ + --expected-closure-digest 8465ff2b7cedb352337e7cedc5bb17cc169892c2eedae0b280569f4a0cde5002 \ + --worker-context --worker-entry \ + --expected-workload-sha256 1534308c977c2c1cb2b1d80d9cc5c07e31303db63b336686e1c5ccc2644b1064 \ + --expected-runtime-sha256 460121aaee3eb83764f67d9307cfb57a3b25c117611d987577b9631b6956a1b3 \ + --expected-trainer-sha256 75c2ab22fbc5dd80b8ddd5e6d0da816d1451cda0cd620c34717336a39a8a7644 \ + --gpu-device 0 \ + --output-dir /home/profsynapse/.local/share/synaptic-qualification/torch-import-r12-20261020/worker-entry-gpu0-1 +``` + +Before any effect, review the clean pushed checkout and helper SHA-256 +`67224904f0124d6defb38003a9da23406848957ec2a5b2935d5c971781485daa`, +trusted verifier SHA-256 +`2304eeb00667954510c34cc4aacaa818532a301fbeabf04c2fb165d6cd1d70b5`, +all pins above, source/ancestor private custody without an active writer, +unchanged daemon/image, idle GPU 0, and the exact host settings and code delta. +Create only the fresh parent at mode 0700 and require the named child absent; +collision or drift rejects without another name. Save and read back a specific +one-use admission before start. The extra authority sought is exactly one +fresh bounded GPU-0 import-only diagnostic container, including the synthetic +sealed descriptor and real worker/runpy entry. This preparation grants none +of that live authority; all r2/r4/r6/r8/r11 claims and containers remain +consumed and preserved. diff --git a/scripts/diagnose_docker_torch_import.py b/scripts/diagnose_docker_torch_import.py index 45f87109..07808bb8 100644 --- a/scripts/diagnose_docker_torch_import.py +++ b/scripts/diagnose_docker_torch_import.py @@ -220,6 +220,143 @@ def mark(value): os.close(report_fd) """ +# The optional entry probe enters the authenticated worker's real main and +# runpy trainer path. A line trace stops on the exact UNSLOTH_IMPORT marker; +# the independent import finder prevents any Unsloth module body from running +# if a future interpreter/library disables tracing. Neither hook replaces the +# worker's argument validation, closure guard or runpy.run_path call. +_WORKER_ENTRY_CHILD = """import builtins,hashlib,json,os,re,runpy,sys +from pathlib import Path +root=Path('DIAGNOSTIC_ENGINE_ROOT') +worker=root/'tuner/runtime/offline_sft_worker.py' +trainer=root/'Trainers/sft/train_sft.py' +stage='ENTRY_ADMISSION' +category='OTHER' +context={} +reached=False +blocked=False +class _StopAtMarker(BaseException): + pass +class _BlockedImport(BaseException): + pass +class _UnslothBarrier: + def find_spec(self,fullname,path=None,target=None): + global blocked + if fullname=='unsloth' or fullname.startswith('unsloth.'): + blocked=True + raise _BlockedImport() + return None +original_import=builtins.__import__ +def guarded_import(name,globals=None,locals=None,fromlist=(),level=0): + global blocked + if name=='unsloth' or name.startswith('unsloth.'): + blocked=True + raise _BlockedImport() + return original_import(name,globals,locals,fromlist,level) +def _digest(value): + return hashlib.sha256(json.dumps(value,sort_keys=True,separators=(',',':')).encode()).hexdigest() +try: + observed=dict(os.environ) + observed_sha=_digest(observed) + if observed_sha=='EXPECTED_SUPPLIED_ENV_SHA256': + delta='NONE' + elif observed_sha=='EXPECTED_LC_ENV_SHA256' and observed.get('LC_CTYPE')=='C.UTF-8': + delta='LC_CTYPE_C_UTF8' + else: + raise ValueError() + if sys.flags.isolated!=1 or sys.flags.no_user_site!=1 or not sys.flags.safe_path or os.environ.get('PYTHONNOUSERSITE')!='1' or os.environ.get('PYTHONSAFEPATH')!='1': + raise ValueError() + if any(name=='unsloth' or name.startswith('unsloth.') for name in sys.modules): + raise ValueError() + with worker.open('rb') as stream: + worker_raw=stream.read(1024*1024+1) + if len(worker_raw)>1024*1024 or hashlib.sha256(worker_raw).hexdigest()!='EXPECTED_WORKER_SHA256': + raise ValueError() + with trainer.open('rb') as stream: + source=stream.read(1024*1024+1) + if len(source)>1024*1024 or hashlib.sha256(source).hexdigest()!='EXPECTED_TRAINER_SHA256': + raise ValueError() + lines=source.decode('utf-8').splitlines(keepends=True) + markers=('_mark_packaged_runtime_phase("BOOTSTRAP_ENV")','_mark_packaged_runtime_phase("TORCH_IMPORT")','_mark_packaged_runtime_phase("UNSLOTH_IMPORT")') + positions=[next((i for i,line in enumerate(lines) if line.strip()==marker),-1) for marker in markers] + if positions[0]<0 or not positions[0]=2 and closed[-1]==b'' else b'' + match=re.fullmatch(rb'OFFLINE_SFT_WORKER_REJECTED phase=(BOOTSTRAP|CLOSURE|BOOTSTRAP_ORIGIN|ARGUMENTS|IMPORT_GUARD|TRAINER|POSTCHECK) category=(REJECTION|IMPORT|OS|VALUE|SYSTEM_EXIT|OTHER)(?: milestone=(BOOTSTRAP_ENV|TORCH_IMPORT|UNSLOTH_IMPORT|TRAINER_IMPORT|TRAIN_CALL))?',label) + if blocked or match is None: + raise ValueError() + if reached and context['runpy_main_bound'] and match.groups()==(b'TRAINER',b'OTHER',b'TORCH_IMPORT'): + category='PASS' + else: + context={} + phase,kind,milestone=match.groups() + stage=({'BOOTSTRAP':'WORKER_MAIN','CLOSURE':'WORKER_CLOSURE','BOOTSTRAP_ORIGIN':'WORKER_ORIGIN','ARGUMENTS':'WORKER_ARGUMENTS','IMPORT_GUARD':'WORKER_GUARD','TRAINER':'TORCH_IMPORT' if milestone==b'TORCH_IMPORT' else 'TRAINER_PREFIX','POSTCHECK':'WORKER_POSTCHECK'})[phase.decode('ascii')] + category=kind.decode('ascii') if kind in (b'IMPORT',b'OS') else 'OTHER' +except ModuleNotFoundError: + category='MODULE_NOT_FOUND' +except ImportError: + category='IMPORT' +except OSError: + category='OS' +except RuntimeError: + category='RUNTIME' +except BaseException: + category='OTHER' +if category!='PASS': + context={} +os.write(1,(json.dumps({'schema_version':'synaptic-docker-torch-import-child/v1','category':category,'stage':stage,'context':context,'boundary':'BEFORE_UNSLOTH'},sort_keys=True,separators=(',',':'))+'\\n').encode('ascii')) +""" + _WORKER_CONTEXT_PROBE = """import base64,hashlib,json,os,sys,subprocess report_fd=os.dup(1) null_fd=os.open('/dev/null',os.O_WRONLY) @@ -261,23 +398,73 @@ def mark(value): (Path(roots['cache'])/name).mkdir(mode=0o700,exist_ok=True) stage='ISOLATED_CHILD' child=base64.b64decode('CHILD_SOURCE_B64') - supplied_raw=json.dumps(environment,sort_keys=True,separators=(',',':')).encode() - if len(supplied_raw)>8192: - raise ValueError() - process=subprocess.Popen([sys.executable,'-I','-c',child.decode('utf-8')],cwd=roots['tmp'],env=environment,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL) - process.stdin.write(supplied_raw) - process.stdin.close() - payload=process.stdout.read(4097) - if len(payload)>4096: - process.kill();process.wait() - raise ValueError() - if process.wait()!=0: - raise ValueError() + if ENTRY_MODE: + import fcntl,runpy + runtime_path=Path('/source/engine/Trainers/sft/runtime_v1.py') + with runtime_path.open('rb') as stream: + runtime_raw=stream.read(1024*1024+1) + if len(runtime_raw)>1024*1024 or hashlib.sha256(runtime_raw).hexdigest()!='EXPECTED_RUNTIME_SHA256': + raise ValueError() + runtime_module=runpy.run_path(str(runtime_path),run_name='__diagnostic_runtime__') + config=workload['configuration']['document'];dataset=config['dataset'] + if not dataset['ref'].startswith('prepared://sha256/'): + raise ValueError() + descriptor=os.memfd_create('synaptic-diagnostic-input',os.MFD_CLOEXEC|os.MFD_ALLOW_SEALING) + try: + os.write(descriptor,b'0') + seals=fcntl.F_SEAL_SEAL|fcntl.F_SEAL_SHRINK|fcntl.F_SEAL_GROW|fcntl.F_SEAL_WRITE + fcntl.fcntl(descriptor,fcntl.F_ADD_SEALS,seals) + arguments=['--','--model-name',ref,'--model-revision',revision,'--anonymous-model', + '--model-cache-dir',roots['cache']+'/model','--model-snapshot',snapshot, + '--local-file','/proc/self/fd/'+str(descriptor), + '--output-root',roots['state']+'/runtime-v1-trainer/output', + '--run-timestamp','runtime-v1','--no-dashboard','--quiet', + '--runtime-v1-workload-fingerprint',environment['SYNAPTIC_WORKLOAD_FINGERPRINT'], + '--runtime-v1-configuration-revision',workload['configuration']['revision'], + '--runtime-v1-tokenizer-revision',model['tokenizer_revision'], + '--runtime-v1-dataset-revision',dataset['revision'], + '--runtime-v1-dataset-digest',dataset['content_digest']] + runtime_module['_append_sft_arguments'](arguments,config['sft'],model, + dataset_schema=dataset['format'] if dataset['format']=='synaptic-conversation-sft-row/v1' else None) + trainer_argv=['/source/engine/Trainers/sft/train_sft.py',*arguments[1:]] + argv_sha=hashlib.sha256(json.dumps(trainer_argv,sort_keys=True,separators=(',',':')).encode()).hexdigest() + token='EXPECTED'+'_'+'TRAINER_ARGV_SHA256' + child_text=child.decode('utf-8').replace(token,argv_sha) + if token in child_text: + raise ValueError() + stdout_path=Path(roots['tracking'])/'worker-entry.stdout' + stderr_path=Path(roots['tracking'])/'worker-entry.stderr' + with stdout_path.open('xb') as stdout,stderr_path.open('xb') as stderr: + process=subprocess.Popen([sys.executable,'-I','-c',child_text,*arguments],cwd=roots['tmp'],env=environment,stdin=subprocess.DEVNULL,stdout=stdout,stderr=stderr,pass_fds=(descriptor,)) + if process.wait()!=0: + raise ValueError() + with stdout_path.open('rb') as stream: + payload=stream.read(4097) + if len(payload)>4096: + raise ValueError() + finally: + os.close(descriptor) + else: + supplied_raw=json.dumps(environment,sort_keys=True,separators=(',',':')).encode() + if len(supplied_raw)>8192: + raise ValueError() + process=subprocess.Popen([sys.executable,'-I','-c',child.decode('utf-8')],cwd=roots['tmp'],env=environment,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL) + process.stdin.write(supplied_raw) + process.stdin.close() + payload=process.stdout.read(4097) + if len(payload)>4096: + process.kill();process.wait() + raise ValueError() + if process.wait()!=0: + raise ValueError() result=json.loads(payload) - if set(result)!={'schema_version','category','stage','context','boundary'} or result['schema_version']!='synaptic-docker-torch-import-child/v1' or result['boundary']!='BEFORE_UNSLOTH' or result['category'] not in ('PASS','MODULE_NOT_FOUND','IMPORT','OS','RUNTIME','OTHER') or result['stage'] not in ('WORKER_IMPORT','WORKER_CLOSURE','WORKER_GUARD','TRAINER_PREFIX','BOOTSTRAP_ENV','TORCH_IMPORT') or not isinstance(result['context'],dict): + if set(result)!={'schema_version','category','stage','context','boundary'} or result['schema_version']!='synaptic-docker-torch-import-child/v1' or result['boundary']!='BEFORE_UNSLOTH' or result['category'] not in ('PASS','MODULE_NOT_FOUND','IMPORT','OS','RUNTIME','OTHER') or result['stage'] not in ('WORKER_IMPORT','WORKER_CLOSURE','WORKER_GUARD','TRAINER_PREFIX','BOOTSTRAP_ENV','TORCH_IMPORT','ENTRY_ADMISSION','WORKER_MAIN','WORKER_ORIGIN','WORKER_ARGUMENTS','WORKER_POSTCHECK') or not isinstance(result['context'],dict): raise ValueError() if result['category']=='PASS': - if result['stage']!='TORCH_IMPORT' or set(result['context'])!={'environment_supplied_sha256','environment_observed_sha256','startup_env_delta','python_isolated','cwd_sha256','file_sha256','sys_path_sha256'} or result['context']['python_isolated'] is not True or result['context']['startup_env_delta'] not in ('NONE','LC_CTYPE_C_UTF8') or any(not isinstance(value,str) or not __import__('re').fullmatch('[0-9a-f]{64}',value) for key,value in result['context'].items() if key.endswith('_sha256')): + expected_context={'environment_supplied_sha256','environment_observed_sha256','startup_env_delta','python_isolated','cwd_sha256','file_sha256','sys_path_sha256'} + if ENTRY_MODE: + expected_context.update(('argv_sha256','runpy_main_bound','environment_before_torch_sha256','environment_after_torch_sha256')) + if result['stage']!='TORCH_IMPORT' or set(result['context'])!=expected_context or result['context']['python_isolated'] is not True or result['context']['startup_env_delta'] not in ('NONE','LC_CTYPE_C_UTF8') or (ENTRY_MODE and (result['context']['runpy_main_bound'] is not True or result['context']['argv_sha256']!=argv_sha)) or any(not isinstance(value,str) or not __import__('re').fullmatch('[0-9a-f]{64}',value) for key,value in result['context'].items() if key.endswith('_sha256')): raise ValueError() elif result['context']!={}: raise ValueError() @@ -453,6 +640,7 @@ def _worker_context_identity(source_root: Path, expected_workload: str, workload_raw = _read(source_root.parent / "control/workload.json", 1024 * 1024) runtime_raw = _read(source_root / "Trainers/sft/runtime_v1.py", 1024 * 1024) trainer_raw = _read(source_root / "Trainers/sft/train_sft.py", 1024 * 1024) + worker_raw = _read(source_root / "tuner/runtime/offline_sft_worker.py", 1024 * 1024) if (hashlib.sha256(workload_raw).hexdigest() != expected_workload or hashlib.sha256(runtime_raw).hexdigest() != expected_runtime or hashlib.sha256(trainer_raw).hexdigest() != expected_trainer): @@ -499,20 +687,29 @@ def _worker_context_identity(source_root: Path, expected_workload: str, return {"workload_sha256": expected_workload, "runtime_sha256": expected_runtime, "trainer_sha256": expected_trainer, + "worker_sha256": hashlib.sha256(worker_raw).hexdigest(), "child_env_sha256": hashlib.sha256(json.dumps( child_env, sort_keys=True, separators=(",", ":")).encode()).hexdigest(), "child_env_lc_ctype_sha256": hashlib.sha256(json.dumps( child_env_lc_ctype, sort_keys=True, separators=(",", ":")).encode()).hexdigest()} -def _worker_probe(identity: dict[str, str], closure: str) -> str: - child = _WORKER_PREFIX_CHILD.replace("EXPECTED_TRAINER_SHA256", - identity["trainer_sha256"]) +def _worker_probe(identity: dict[str, str], closure: str, *, entry_mode: bool = False) -> str: + child = (_WORKER_ENTRY_CHILD if entry_mode else _WORKER_PREFIX_CHILD) + child = child.replace("EXPECTED_TRAINER_SHA256", identity["trainer_sha256"]) child = child.replace("EXPECTED_SUPPLIED_ENV_SHA256", identity["child_env_sha256"]) + if entry_mode: + child = child.replace("EXPECTED_LC_ENV_SHA256", + identity["child_env_lc_ctype_sha256"]) + child = child.replace("EXPECTED_WORKER_SHA256", identity["worker_sha256"]) + child = child.replace("DIAGNOSTIC_ENGINE_ROOT", "/source/engine") + child = child.replace("DIAGNOSTIC_STDERR_PATH", + "/artifacts/run/tracking/worker-entry.stderr") probe = _WORKER_CONTEXT_PROBE replacements = {"EXPECTED_WORKLOAD_SHA256": identity["workload_sha256"], "EXPECTED_CLOSURE_DIGEST": closure, + "EXPECTED_RUNTIME_SHA256": identity["runtime_sha256"], "EXPECTED_ROOTS": repr(_R2_ROOTS), "EXPECTED_ENV_KEYS": repr(sorted(_R2_ENV_KEYS)), "ROOT_BINDINGS": repr({_ROOT_ENV[name]: value @@ -522,6 +719,7 @@ def _worker_probe(identity: dict[str, str], closure: str) -> str: if probe.count(key) != 1: raise ValueError("SOURCE_IDENTITY_INVALID") probe = probe.replace(key, value) + probe = probe.replace("ENTRY_MODE", "True" if entry_mode else "False") return probe @@ -632,12 +830,13 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, expected_manifest_sha256: str | None = None, expected_closure_digest: str | None = None, gpu_device: str | None = None, worker_context: bool = False, + worker_entry: bool = False, expected_workload_sha256: str | None = None, expected_runtime_sha256: str | None = None, expected_trainer_sha256: str | None = None) -> dict[str, object]: worker_values = (expected_workload_sha256, expected_runtime_sha256, expected_trainer_sha256) - if ((worker_context and (source_root is None or any(value is None for value in worker_values))) + if (worker_entry and not worker_context) or ((worker_context and (source_root is None or any(value is None for value in worker_values))) or (not worker_context and any(value is not None for value in worker_values))): raise ValueError("SOURCE_IDENTITY_INVALID") if gpu_device not in (None, "0") or (gpu_device is not None and source_root is None): @@ -679,6 +878,15 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, expected_manifest_sha256, expected_closure_digest)) context_identity = ({} if not worker_context else _worker_context_identity( source_root, *worker_values, expected_closure_digest)) + if worker_entry: + try: + workload = json.loads(_read(source_root.parent / "control/workload.json", 1024 * 1024)) + dataset_ref = workload["configuration"]["document"]["dataset"]["ref"] + except (KeyError, TypeError, ValueError): + raise ValueError("SOURCE_IDENTITY_INVALID") from None + if (type(dataset_ref) is not str or + not re.fullmatch(r"prepared://sha256/[0-9a-f]{64}", dataset_ref)): + raise ValueError("SOURCE_IDENTITY_INVALID") custody = (() if source_root is None else tuple(_path_custody(path) for path in (source_root, source_root.parent / "control/offline-sft-worker-v1.json", source_lock, *((source_root.parent / "control/workload.json",) if worker_context else ())))) @@ -692,7 +900,7 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, output_dir.mkdir(mode=0o700) nonce = secrets.token_hex(16) name = "synaptic-torch-import-" + nonce - probe = (_worker_probe(context_identity, expected_closure_digest) + probe = (_worker_probe(context_identity, expected_closure_digest, entry_mode=worker_entry) if worker_context else (_PROBE if source_root is None else _GUARDED_PROBE.replace("EXPECTED_CLOSURE_DIGEST", expected_closure_digest))) if gpu_device is not None: @@ -705,7 +913,7 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, claim = {"schema_version": SCHEMA, "daemon_id": daemon_id, "image_id": image, "release_sha256": expected_release_sha256, "source_identity": source_identity, "context_identity": context_identity, - "name": name, "scope": "worker_context" if worker_context else + "name": name, "scope": ("worker_entry" if worker_entry else "worker_context") if worker_context else ("bare" if source_root is None else "guarded"), "helper_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), "probe_sha256": hashlib.sha256(probe.encode("utf-8")).hexdigest(), @@ -834,14 +1042,20 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, or (worker_context and (child["boundary"] != "BEFORE_UNSLOTH" or child["stage"] not in {"READ_WORKLOAD", "ISOLATED_CHILD", "WORKER_IMPORT", "WORKER_CLOSURE", "WORKER_GUARD", - "TRAINER_PREFIX", "BOOTSTRAP_ENV", "TORCH_IMPORT"} + "TRAINER_PREFIX", "BOOTSTRAP_ENV", "TORCH_IMPORT", + "ENTRY_ADMISSION", "WORKER_MAIN", "WORKER_ORIGIN", + "WORKER_ARGUMENTS", "WORKER_POSTCHECK"} or type(child["context"]) is not dict or (child["category"] == "PASS" and (child["stage"] != "TORCH_IMPORT" or - set(child["context"]) != {"environment_supplied_sha256", + set(child["context"]) != ({"environment_supplied_sha256", "environment_observed_sha256", "startup_env_delta", "python_isolated", "cwd_sha256", "file_sha256", "sys_path_sha256"} + | ({"argv_sha256", "runpy_main_bound", + "environment_before_torch_sha256", + "environment_after_torch_sha256"} if worker_entry else set())) or child["context"]["python_isolated"] is not True + or (worker_entry and child["context"]["runpy_main_bound"] is not True) or child["context"]["startup_env_delta"] not in {"NONE", "LC_CTYPE_C_UTF8"})) or (child["category"] != "PASS" and child["context"] != {}) @@ -868,7 +1082,7 @@ def diagnose(*, docker: Path, docker_config: Path, release_path: Path, "container_id": cid, "image_id": image, "daemon_id": daemon_id, "runtime_release_digest": digest, "training_executed": False, "gpu_used": None if gpu_device is not None else False, - "scope": "worker_context" if worker_context else + "scope": ("worker_entry" if worker_entry else "worker_context") if worker_context else ("bare" if source_root is None else "guarded"), "claim_sha256": label, "source_identity": source_identity, "context_identity": context_identity, @@ -899,6 +1113,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--expected-closure-digest") parser.add_argument("--gpu-device") parser.add_argument("--worker-context", action="store_true") + parser.add_argument("--worker-entry", action="store_true") parser.add_argument("--expected-workload-sha256") parser.add_argument("--expected-runtime-sha256") parser.add_argument("--expected-trainer-sha256") @@ -913,6 +1128,7 @@ def main(argv: list[str] | None = None) -> int: expected_manifest_sha256=args.expected_manifest_sha256, expected_closure_digest=args.expected_closure_digest, gpu_device=args.gpu_device, worker_context=args.worker_context, + worker_entry=args.worker_entry, expected_workload_sha256=args.expected_workload_sha256, expected_runtime_sha256=args.expected_runtime_sha256, expected_trainer_sha256=args.expected_trainer_sha256) diff --git a/tests/scripts/test_diagnose_docker_torch_import.py b/tests/scripts/test_diagnose_docker_torch_import.py index ac8cae41..612ff6dc 100644 --- a/tests/scripts/test_diagnose_docker_torch_import.py +++ b/tests/scripts/test_diagnose_docker_torch_import.py @@ -2,6 +2,8 @@ import hashlib import json +import base64 +import re import shutil import subprocess import sys @@ -200,6 +202,182 @@ def test_worker_context_program_is_isolated_and_excludes_trainer_main(tmp_path: assert identity["child_env_lc_ctype_sha256"] != identity["child_env_sha256"] +def _entry_fixture(tmp_path: Path, *, disable_trace: bool = False, + import_error: bool = False): + """Use real isolated runpy and a non-ML Torch stub at the stop boundary.""" + engine = tmp_path / "engine" + trainer = engine / "Trainers/sft/train_sft.py" + trainer.parent.mkdir(parents=True) + worker = engine / "tuner/runtime/offline_sft_worker.py" + worker.parent.mkdir(parents=True) + worker.write_text("""import runpy,sys +from pathlib import Path +def mark(value): + pass +def main(): + if sys.argv[1:]!=['--','--max-steps','1','--no-load-in-4bit']: + print('OFFLINE_SFT_WORKER_REJECTED phase=ARGUMENTS category=REJECTION',file=sys.stderr) + return 2 + root=Path(__file__).parents[2] + trainer=root/'Trainers/sft/train_sft.py' + sys.path[:0]=[str(trainer.parent),str(root)] + sys.argv=[str(trainer),*sys.argv[2:]] + try: + runpy.run_path(str(trainer),run_name='__main__',init_globals={'__packaged_runtime_phase__':mark}) + except ImportError: + print('OFFLINE_SFT_WORKER_REJECTED phase=TRAINER category=IMPORT milestone=TORCH_IMPORT',file=sys.stderr) + return 2 + except BaseException: + print('OFFLINE_SFT_WORKER_REJECTED phase=TRAINER category=OTHER milestone=TORCH_IMPORT',file=sys.stderr) + return 2 + return 0 +if __name__=='__main__': + raise SystemExit(main()) +""") + trainer.write_text("""import os,sys +from pathlib import Path +def _mark_packaged_runtime_phase(value): + marker=globals().get('__packaged_runtime_phase__') + if marker is not None: marker(value) +assert __name__=='__main__' +assert sys.modules['__main__'].__dict__ is globals() +_mark_packaged_runtime_phase("BOOTSTRAP_ENV") +os.environ['TORCH_COMPILE_DISABLE']='1' +_mark_packaged_runtime_phase("TORCH_IMPORT") +import torch # noqa: E402 +_mark_packaged_runtime_phase("UNSLOTH_IMPORT") +from unsloth import is_bfloat16_supported # noqa: E402 +Path(__file__).with_name('TRAINER_FELL_THROUGH').write_text('bad') +""") + (trainer.parent / "torch.py").write_text( + "from pathlib import Path\nimport sys\n" + + ("import types\nsys.settrace(None)\n" + "sys.modules['unsloth']=types.ModuleType('unsloth')\n" + "sys.modules['unsloth'].is_bfloat16_supported=True\n" + if disable_trace else "") + + "Path(__file__).with_name('TORCH_IMPORTED').write_text('yes')\n" + + ("raise ImportError('private detail')\n" if import_error else "")) + (trainer.parent / "unsloth.py").write_text( + "from pathlib import Path\n" + "Path(__file__).with_name('UNSLOTH_IMPORTED').write_text('bad')\n" + "is_bfloat16_supported=True\n") + environment = {"PATH": "/usr/bin:/bin", "PYTHONNOUSERSITE": "1", + "PYTHONSAFEPATH": "1", "PYTHONDONTWRITEBYTECODE": "1"} + observed_locale = {**environment, "LC_CTYPE": "C.UTF-8"} + arguments = ["--", "--max-steps", "1", "--no-load-in-4bit"] + digest = lambda obj: hashlib.sha256(json.dumps( + obj, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + source = diagnostic._WORKER_ENTRY_CHILD + for key, value in { + "DIAGNOSTIC_ENGINE_ROOT": str(engine), + "DIAGNOSTIC_STDERR_PATH": str(tmp_path / "worker-entry.stderr"), + "EXPECTED_TRAINER_SHA256": hashlib.sha256(trainer.read_bytes()).hexdigest(), + "EXPECTED_WORKER_SHA256": hashlib.sha256(worker.read_bytes()).hexdigest(), + "EXPECTED_SUPPLIED_ENV_SHA256": digest(environment), + "EXPECTED_LC_ENV_SHA256": digest(observed_locale), + "EXPECTED_TRAINER_ARGV_SHA256": digest([str(trainer), *arguments[1:]]), + }.items(): + source = source.replace(key, value) + return trainer, source, environment, arguments + + +@pytest.mark.parametrize("disable_trace", [False, True]) +def test_real_isolated_worker_entry_stops_before_unsloth(tmp_path: Path, + disable_trace: bool) -> None: + trainer, source, environment, arguments = _entry_fixture( + tmp_path, disable_trace=disable_trace) + with (tmp_path / "worker-entry.stderr").open("xb") as stderr: + completed = subprocess.run([sys.executable, "-I", "-c", source, *arguments], + cwd=tmp_path, env=environment, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=stderr, timeout=20) + assert completed.returncode == 0 + result = json.loads(completed.stdout) + assert result["boundary"] == "BEFORE_UNSLOTH" + if disable_trace: + assert result["category"] == "OTHER" + else: + assert result["category"] == "PASS" + assert result["stage"] == "TORCH_IMPORT" + assert result["context"]["runpy_main_bound"] is True + assert (result["context"]["environment_before_torch_sha256"] == + result["context"]["environment_after_torch_sha256"]) + assert (result["context"]["environment_before_torch_sha256"] != + result["context"]["environment_observed_sha256"]) + assert (trainer.parent / "TORCH_IMPORTED").exists() + assert not (trainer.parent / "UNSLOTH_IMPORTED").exists() + assert not (trainer.parent / "TRAINER_FELL_THROUGH").exists() + + +def test_real_isolated_worker_entry_reports_closed_torch_import_failure( + tmp_path: Path) -> None: + trainer, source, environment, arguments = _entry_fixture( + tmp_path, import_error=True) + with (tmp_path / "worker-entry.stderr").open("xb") as stderr: + completed = subprocess.run([sys.executable, "-I", "-c", source, *arguments], + cwd=tmp_path, env=environment, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=stderr, timeout=20) + assert completed.returncode == 0 + result = json.loads(completed.stdout) + assert result == {"schema_version": "synaptic-docker-torch-import-child/v1", + "category": "IMPORT", "stage": "TORCH_IMPORT", "context": {}, + "boundary": "BEFORE_UNSLOTH"} + assert (trainer.parent / "TORCH_IMPORTED").exists() + assert not (trainer.parent / "UNSLOTH_IMPORTED").exists() + + +def test_isolated_worker_entry_rejects_trainer_drift_before_worker( + tmp_path: Path) -> None: + trainer, source, environment, arguments = _entry_fixture(tmp_path) + trainer.write_bytes(trainer.read_bytes() + b"\n") + with (tmp_path / "worker-entry.stderr").open("xb") as stderr: + completed = subprocess.run([sys.executable, "-I", "-c", source, *arguments], + cwd=tmp_path, env=environment, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=stderr, timeout=20) + assert completed.returncode == 0 + result = json.loads(completed.stdout) + assert result["category"] == "OTHER" and result["stage"] == "ENTRY_ADMISSION" + assert result["context"] == {} + assert not (trainer.parent / "TORCH_IMPORTED").exists() + + +def test_worker_entry_source_and_mode_bind_before_docker(tmp_path: Path, + monkeypatch) -> None: + flags, workload_path = _stage_worker_context(tmp_path) + docker, config, state = _fake_docker(tmp_path) + monkeypatch.setattr(diagnostic, "_run", lambda *args, **kwargs: + pytest.fail("Docker touched")) + with pytest.raises(ValueError, match="SOURCE_IDENTITY_INVALID"): + _call(tmp_path, docker, config, worker_entry=True, **flags) + assert json.loads(state.read_text())["status"] == "absent" + workload = json.loads(workload_path.read_text()) + workload["configuration"]["document"]["dataset"] = { + "ref": "prepared://sha256/" + "a" * 64} + workload_path.write_text(json.dumps(workload, sort_keys=True, separators=(",", ":"))) + flags["expected_workload_sha256"] = hashlib.sha256(workload_path.read_bytes()).hexdigest() + identity = diagnostic._worker_context_identity( + flags["source_root"], flags["expected_workload_sha256"], + flags["expected_runtime_sha256"], flags["expected_trainer_sha256"], + flags["expected_closure_digest"]) + probe = diagnostic._worker_probe(identity, flags["expected_closure_digest"], + entry_mode=True) + ast.parse(probe) + assert "EXPECTED_" not in probe and "ENTRY_MODE" not in probe + match = re.search(r"base64\.b64decode\('([A-Za-z0-9+/=]+)'\)", probe) + assert match is not None + child = base64.b64decode(match.group(1)).decode() + ast.parse(child) + assert "DIAGNOSTIC_ENGINE_ROOT" not in child + assert "DIAGNOSTIC_STDERR_PATH" not in child + assert "runpy.run_path(str(worker),run_name='__main__')" in child + assert "sys.settrace(trace)" in child and "_UnslothBarrier" in child + trainer = flags["source_root"] / "Trainers/sft/train_sft.py" + trainer.write_bytes(trainer.read_bytes() + b"\n") + with pytest.raises(ValueError, match="SOURCE_IDENTITY_INVALID"): + _call(tmp_path, docker, config, worker_entry=True, **flags) + assert json.loads(state.read_text())["status"] == "absent" + assert not (tmp_path / "out").exists() + + @pytest.mark.parametrize("mutation,expected", [ ("none", "LC_CTYPE_C_UTF8"), ("no_coercion", "NONE"), From df43eb695874fc3c44c406e936fbe6f7c8ec25e8 Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:28:45 -0400 Subject: [PATCH 10/12] docs: record bounded worker-entry torch probe evidence --- .../docker-torch-import-r2-diagnosis.md | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/docs/review/docker-torch-import-r2-diagnosis.md b/docs/review/docker-torch-import-r2-diagnosis.md index cf6785a8..f65e135a 100644 --- a/docs/review/docker-torch-import-r2-diagnosis.md +++ b/docs/review/docker-torch-import-r2-diagnosis.md @@ -567,3 +567,63 @@ fresh bounded GPU-0 import-only diagnostic container, including the synthetic sealed descriptor and real worker/runpy entry. This preparation grants none of that live authority; all r2/r4/r6/r8/r11 claims and containers remain consumed and preserved. + +### r13 execution of the admitted worker-entry probe (2026-10-20) + +Root subsequently accepted the exact r12 command at clean, pushed +`14f44a3ef389c9c04225b058626bb3cfbba61d45` for **one** fresh diagnostic +container. The operator read back the root admission, then checked the remote +clean checkout and pushed ref, helper and trusted verifier SHA-256 above, exact +release/daemon/image/source-lock/manifest/closure/workload/runtime/trainer +commitments, private source/ancestor custody, and absence of a visible active +source writer. The fresh parent was mode 0700 and the named output child was +absent. Windows and WSL reported GPU 0 +`GPU-ac4398bb-fc15-d722-b39c-2790b4b5f9cc` with no active compute process. +The one-use current-preflight/command admission was saved and read back in +Nexus as `Windows r13 worker-entry one-use pre-start admission — 2026-10-20` +before Docker effects. These are point-in-time host observations; a read-only +bind does not freeze the retained host tree. + +The command above ran **once**, from the remote clean checkout, with no retry. +The retained output is +`/home/profsynapse/.local/share/synaptic-qualification/torch-import-r12-20261020/worker-entry-gpu0-1`. +It returned closed `PASS` at `TORCH_IMPORT`, boundary `BEFORE_UNSLOTH`, with +`python_isolated=true`, `runpy_main_bound=true`, `training_executed=false`, +and no import barrier activation. The separately supplied environment digest +was `1bb0e06d9708dd6fd637f59eaf134dcbe8165f43e120a0b94daad6c79a63699c`; +the observed startup digest was +`a1eeabbf01741b374ca4e410d62119cf8d882b97d33da962257ec7570bdcb08c`, +with the explicitly admitted `LC_CTYPE_C_UTF8` interpreter addition. The +before- and after-Torch environment digests both were +`712d7eee6b813d815b996f2d3db8266c691dcd5d9f02ea5b9b09073409bd2efc`. +These are reconstructed source/plan context digests, not measurements of r2's +historical child environment. `gpu_used` remains unknown: device exposure is +verified, but this diagnostic does not establish CUDA use. + +The single retained container is +`d9f8c6dda19ce32a4e3326762cc3ecf929451ef011162ba595b1dd9b12a8636f`. +The durable event sequence is exactly `CREATE_CLAIMED`, `CREATED`, +`START_CLAIMED`, `STARTED`, `TERMINAL_OBSERVED`; the independent daemon +inspection found it `exited` with code 0. It started at +`2026-10-20T16:27:48.793457215Z` and finished at +`2026-10-20T16:27:54.515077872Z`. Inspection confirmed the expected daemon, +image, claim label and name; device request `0`; 2 CPU, 4 GiB, network none, +read-only root, all capabilities dropped, read-only `/source` bind, and +64-MiB `/tmp` and `/artifacts` tmpfs. The container and all prior diagnostic +containers and claims remain retained. The claim digest (over canonical claim +bytes without its final newline) is +`9f22951917cf0eb35ce4e2084c64046087ff4777ce7339bde8039ec04cee6e50`. +Retained file SHA-256 values are: + +| File | SHA-256 | +| --- | --- | +| `claim.json` | `86e6202f20064fb12087e532f26debcef6e4047b5d6dcea8bdf3c81ac4648031` | +| `events.jsonl` | `47dd59c98b9da7c15250859f5ee7ccaf3f110070d5a1e73785754e188373953f` | +| `result.json` | `90d84bf65199e6b4aa3d1fd9f88a4bb7d79677902c085d684987ae921bb3f7e5` | + +The wrapper, trace/import barrier, `-I -c` entry, synthetic sealed descriptor, +reduced privilege/filesystem/memory settings, and absent historical model and +dataset bytes remain material differences from r2's direct `-I` worker script +entry. The successful r13 import therefore narrows the observed boundary but +does **not** reproduce r2 or establish the cause of its exit 23. It grants no +additional probe, model, or training authority. From ea775b0f7df4f2730a3944a9b6f16718669e9039 Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:56:40 -0400 Subject: [PATCH 11/12] Prepare Docker SFT model before fresh source admission --- .../fine-tuning/reference/sft-training.md | 11 +- .../fine-tuning/reference/sft-training.md | 11 +- .skills/fine-tuning/reference/sft-training.md | 11 +- .../docker-torch-import-r2-diagnosis.md | 37 ++++ examples/training_qualification/README.md | 29 ++- .../training_qualification/docker_host.py | 106 ++++++++++- .../prepare-docker.config.example.json | 2 + .../training_qualification/prepare_docker.py | 146 ++++++++++++--- .../test_docker_material_preparation.py | 171 ++++++++++++++++++ 9 files changed, 479 insertions(+), 45 deletions(-) diff --git a/.agents/skills/fine-tuning/reference/sft-training.md b/.agents/skills/fine-tuning/reference/sft-training.md index dfddf7ba..d1b32d7a 100644 --- a/.agents/skills/fine-tuning/reference/sft-training.md +++ b/.agents/skills/fine-tuning/reference/sft-training.md @@ -450,10 +450,13 @@ See the checked-in example README for commands and exact fixture coverage. The checkout-owned `examples/training_qualification/prepare_docker.py` closes material setup for the public Docker consumer. With an installed engine, -`plan` validates declared release/model/layout, `prepare` authenticates genuine -sources and existing immutable v1 input before publishing canonical material, -and `prepare-model` explicitly downloads the pinned public snapshot into the -private artifact-mount cache. See `examples/training_qualification/README.md` +`plan` validates declared release/model/layout, `prepare-model` first prepares +the pinned public snapshot into a fresh private artifact cache, and `prepare` +then authenticates fresh source and existing immutable v1 input before publishing +canonical material. A private read-only seed cache may reduce downloads only +after pinned upstream metadata and every reused Git/LFS member are verified. +The old seed claim is never adopted. `prepare` binds the complete model receipt; +public start rechecks its bytes before Docker effects. See `examples/training_qualification/README.md` for the template and exact commands. Full public preflight remains deferred until host composition; missing image/runtime/model qualification still refuses. The CPU `probe-filesystem` command measures actual native writer modes and host diff --git a/.claude/skills/fine-tuning/reference/sft-training.md b/.claude/skills/fine-tuning/reference/sft-training.md index dfddf7ba..d1b32d7a 100644 --- a/.claude/skills/fine-tuning/reference/sft-training.md +++ b/.claude/skills/fine-tuning/reference/sft-training.md @@ -450,10 +450,13 @@ See the checked-in example README for commands and exact fixture coverage. The checkout-owned `examples/training_qualification/prepare_docker.py` closes material setup for the public Docker consumer. With an installed engine, -`plan` validates declared release/model/layout, `prepare` authenticates genuine -sources and existing immutable v1 input before publishing canonical material, -and `prepare-model` explicitly downloads the pinned public snapshot into the -private artifact-mount cache. See `examples/training_qualification/README.md` +`plan` validates declared release/model/layout, `prepare-model` first prepares +the pinned public snapshot into a fresh private artifact cache, and `prepare` +then authenticates fresh source and existing immutable v1 input before publishing +canonical material. A private read-only seed cache may reduce downloads only +after pinned upstream metadata and every reused Git/LFS member are verified. +The old seed claim is never adopted. `prepare` binds the complete model receipt; +public start rechecks its bytes before Docker effects. See `examples/training_qualification/README.md` for the template and exact commands. Full public preflight remains deferred until host composition; missing image/runtime/model qualification still refuses. The CPU `probe-filesystem` command measures actual native writer modes and host diff --git a/.skills/fine-tuning/reference/sft-training.md b/.skills/fine-tuning/reference/sft-training.md index dfddf7ba..d1b32d7a 100644 --- a/.skills/fine-tuning/reference/sft-training.md +++ b/.skills/fine-tuning/reference/sft-training.md @@ -450,10 +450,13 @@ See the checked-in example README for commands and exact fixture coverage. The checkout-owned `examples/training_qualification/prepare_docker.py` closes material setup for the public Docker consumer. With an installed engine, -`plan` validates declared release/model/layout, `prepare` authenticates genuine -sources and existing immutable v1 input before publishing canonical material, -and `prepare-model` explicitly downloads the pinned public snapshot into the -private artifact-mount cache. See `examples/training_qualification/README.md` +`plan` validates declared release/model/layout, `prepare-model` first prepares +the pinned public snapshot into a fresh private artifact cache, and `prepare` +then authenticates fresh source and existing immutable v1 input before publishing +canonical material. A private read-only seed cache may reduce downloads only +after pinned upstream metadata and every reused Git/LFS member are verified. +The old seed claim is never adopted. `prepare` binds the complete model receipt; +public start rechecks its bytes before Docker effects. See `examples/training_qualification/README.md` for the template and exact commands. Full public preflight remains deferred until host composition; missing image/runtime/model qualification still refuses. The CPU `probe-filesystem` command measures actual native writer modes and host diff --git a/docs/review/docker-torch-import-r2-diagnosis.md b/docs/review/docker-torch-import-r2-diagnosis.md index f65e135a..28bfce7f 100644 --- a/docs/review/docker-torch-import-r2-diagnosis.md +++ b/docs/review/docker-torch-import-r2-diagnosis.md @@ -627,3 +627,40 @@ dataset bytes remain material differences from r2's direct `-I` worker script entry. The successful r13 import therefore narrows the observed boundary but does **not** reproduce r2 or establish the cause of its exit 23. It grants no additional probe, model, or training authority. + +### r15 public SFT model-first preparation (2026-10-20) + +The retained r14 public-consumer planning root and its r3 configuration are +historical preparation only. No r14 training was submitted. The r2 worker's +exit 23 remains unexplained; the successful reduced import probes do not +establish a runtime fix. The r14 flow exposed an ordering problem: the +short-lived pushed-source proof was minted before a 9.3-GB model staging step. + +The Docker example now accepts a separate, fresh `model_output_root` and an +optional private `model_seed_root`. `prepare-model` consumes its own one-use +claim, copies seed members into a fresh private cache through bounded regular +file reads, and uses the unchanged shared model preparer to verify every reused +member against current pinned Hub Git/LFS metadata. Missing members take the +normal cold download path. Corrupt, redirected or changed seed members reject; +old cache bytes and claims remain untouched. The model receipt records the +declared model/configuration/runtime and a complete bounded destination file +inventory. `prepare` checks that receipt and bytes **before** source +authentication, then compiles a fresh public consumer that binds the receipt. +The existing public `start` composition rechecks the receipt and actual bytes +before Docker effects. This adds no image, package, trainer or runtime change; +the Modal/HF closures and runtime locks remain current without refresh. + +For review, the proposed next attempt must use a new pushed host commit with +the exact new engine gitlink, a fresh private plan/config/model/source/artifact +root, and a new execution attempt claim. The prior r14 host/gitlink/config and +every earlier model/source/start claim remain consumed or historical; they +cannot be reinterpreted as fresh authority. The ordered commands are the +checked-in example's `plan`, `model-preflight`, `prepare-model`, `prepare`, +public `launch.py plan`, then one separately admitted public `launch.py start`. +The model stage has no hard SDK download deadline; only the subsequent source +proof uses its 600-second validity window. Before any live step, independently +review exact pushed source and genuine host gitlink, wheel/consumer closure, +release and image C pins, daemon/device, source and model custody, prepared +input publication, fresh names, and the original 3-step full-precision SFT +recipe with 900-second/2-CPU/16-GiB/GPU-0/network-none bounds. Do not run a +model request, container or training job on this implementation evidence alone. diff --git a/examples/training_qualification/README.md b/examples/training_qualification/README.md index 8f3c8ee2..2f933204 100644 --- a/examples/training_qualification/README.md +++ b/examples/training_qualification/README.md @@ -178,8 +178,8 @@ Run from a neutral directory with the installed engine interpreter: ```sh python -I /absolute/checkout/examples/training_qualification/prepare_docker.py plan /private/prepare.json python -I /absolute/checkout/examples/training_qualification/prepare_docker.py model-preflight /private/prepare.json -python -I /absolute/checkout/examples/training_qualification/prepare_docker.py prepare /private/prepare.json python -I /absolute/checkout/examples/training_qualification/prepare_docker.py prepare-model /private/prepare.json +python -I /absolute/checkout/examples/training_qualification/prepare_docker.py prepare /private/prepare.json python -I /absolute/checkout/examples/training_qualification/launch.py start /private/prepared/consumer.json ``` @@ -192,14 +192,25 @@ pull, build, download a model or start a container. Original host manifests and pushed source evidence must be genuine and fresh. Its source admission helper is shared with the HF preparer without changing HF canonical outputs. -`prepare-model` is the explicit public-model network boundary: it uses the existing -pinned model preparer with anonymous credentials, private persistent/SDK scratch -under the preparation output and destination under the authenticated artifact -mount's `run/cache`. The existing verifier checks pinned file metadata/content; -the native snapshot validator checks the resulting offline path. The source -mount remains empty until the original claimed stage. Writable runtime directories -are private and disjoint; the exact prepared-input slot is never adopted. A -create-only local attempt prevents automatic repeat after an uncertain download. +`prepare-model` is the explicit public-model network boundary. With the template's +`model_output_root`, it consumes a fresh private model claim and prepares the +pinned model before any short-lived execution-source proof is minted. It uses +anonymous upstream metadata, private persistent/SDK scratch, and a fresh artifact +mount's `run/cache`. Optional `model_seed_root` points to an independently held, +private read-only repository cache; each present member is copied into fresh roots +and checked against current pinned Git/LFS metadata. A corrupt, redirected or +changed seed member rejects; absent members use the normal cold download path. +The seed's old claim and bytes stay untouched and never become new authority. +The native snapshot validator checks the resulting offline path and a complete +bounded file inventory is retained in `model-preparation.json`. + +`prepare` first rechecks that receipt and actual model bytes against its declared +configuration, then authenticates fresh pushed source and compiles material. Its +consumer binds the exact receipt bytes; public `start` rechecks the model inventory +before Docker effects. The source mount remains absent until the source stage. +Writable runtime directories are private and disjoint; the exact prepared-input +slot is never adopted. Each model and source root is one-use. A failed or uncertain +model preparation cannot be replayed under the same claim. Vendor download behavior is not a hard job deadline or bill cap. No manual weight upload, source inventory widening or restored training authority is involved. diff --git a/examples/training_qualification/docker_host.py b/examples/training_qualification/docker_host.py index 802940d5..b75174b3 100644 --- a/examples/training_qualification/docker_host.py +++ b/examples/training_qualification/docker_host.py @@ -31,6 +31,7 @@ DockerTrainingPreparationBridgeV1, ) from tuner.execution.providers.docker_provider_v1.local_source import _Root +from tuner.execution.providers.modal.mounted_io import hash_regular from tuner.execution.providers.docker_provider_v1.local_artifacts import ( DockerLocalArtifactsV1, _DIR, @@ -106,6 +107,100 @@ def _profile(raw): return profile +def model_inventory(artifact_root, snapshot): + """Hash the complete private offline snapshot through bounded regular reads.""" + artifact_root = Path(artifact_root) + snapshot = Path(snapshot) + with ExitStack() as stack: + guard = _Root(artifact_root) + stack.callback(guard.stack.close) + guard.check() + if not snapshot.is_relative_to(artifact_root) or ".." in snapshot.parts: + raise ValueError("docker_model_snapshot_outside_artifact_root") + rows = [] + total = 0 + parent = guard.fd + chain = [] + for part in snapshot.relative_to(artifact_root).parts: + child = os.open(part, _DIR, dir_fd=parent) + stack.callback(os.close, child) + chain.append((parent, part, child)) + parent = child + pending = [(snapshot, parent)] + seen = 0 + directories = 0 + while pending: + directory, directory_fd = pending.pop() + with os.scandir(directory_fd) as entries: + for entry in entries: + seen += 1 + if seen > 20_000: + raise ValueError("docker_model_inventory_bound") + info = entry.stat(follow_symlinks=False) + if stat.S_ISDIR(info.st_mode) and not entry.is_symlink(): + directories += 1 + if directories > 256: + raise ValueError("docker_model_inventory_bound") + child = os.open(entry.name, _DIR, dir_fd=directory_fd) + stack.callback(os.close, child) + chain.append((directory_fd, entry.name, child)) + pending.append((directory / entry.name, child)) + elif stat.S_ISREG(info.st_mode) and not entry.is_symlink(): + if len(rows) >= 20_000 or total + info.st_size > 1 << 40: + raise ValueError("docker_model_inventory_bound") + path = directory / entry.name + size, digest = hash_regular(artifact_root, path, info.st_size) + if size != info.st_size: + raise ValueError("docker_model_member_changed") + rows.append({"path": path.relative_to(snapshot).as_posix(), + "size_bytes": size, "sha256": digest}) + total += size + else: + raise ValueError("docker_model_member_invalid") + DockerLocalArtifactsV1._recheck(chain) + guard.check() + if not rows: + raise ValueError("docker_model_snapshot_empty") + return sorted(rows, key=lambda item: item["path"]) + + +def verify_model_receipt(raw, *, selected, model, configuration_sha256, + runtime_declaration_sha256): + """Bind fresh prepared bytes to the later public consumer's exact inputs.""" + from Trainers.sft.runtime_v1 import RuntimeRoots, _require_local_model_snapshot + + receipt = _json(raw) + artifact_root = Path(selected["artifact_host_root"]) + cache = artifact_root / "run/cache" + expected = (cache / "model" / ("models--" + model["ref"].replace("/", "--")) + / "snapshots" / model["revision"]) + if (set(receipt) != {"schema_version", "model", "configuration_sha256", + "runtime_declaration_sha256", "artifact_host_root", + "snapshot", "files"} + or receipt["schema_version"] != "synaptic-docker-model-preparation/v1" + or receipt["model"] != model + or receipt["configuration_sha256"] != configuration_sha256 + or receipt["runtime_declaration_sha256"] != runtime_declaration_sha256 + or receipt["artifact_host_root"] != str(artifact_root) + or receipt["snapshot"] != str(expected) + or not isinstance(receipt["files"], list)): + raise ValueError("docker_model_receipt_mismatch") + roots = RuntimeRoots( + engine=Path(selected["source_host_root"]) / "engine", + project=Path(selected["source_host_root"]) / "project", + artifacts=artifact_root / "run/artifacts", state=artifact_root / "run/state", + tracking=artifact_root / "run/tracking", cache=cache, + tmp=artifact_root / "run/tmp", + ) + _require_local_model_snapshot(model, roots, { + "SYNAPTIC_CACHE_ROOT": str(cache), "SYNAPTIC_MODEL_SNAPSHOT": str(expected), + "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", + }) + if model_inventory(artifact_root, expected) != receipt["files"]: + raise ValueError("docker_model_receipt_bytes_changed") + return receipt + + def _components(config): if config["provider"] != "docker": raise ValueError("Docker provider required") @@ -129,7 +224,7 @@ def _components(config): "command_seconds", "read_seconds", } - if set(selected) - {"allow_current_cancel_authorization"} != fields: + if set(selected) - {"allow_current_cancel_authorization", "model_preparation_file"} != fields: raise ValueError("Docker configuration fields invalid") profile = _profile(read_committed(selected["profile_file"])) recipes = default_recipe_registry() @@ -151,6 +246,15 @@ def _components(config): document["artifact_policy"]["retain_checkpoints"], ), ) + if "model_preparation_file" in selected: + verify_model_receipt( + read_committed(selected["model_preparation_file"]), + selected=selected, model=rich.resolved_config.to_dict()["model"], + configuration_sha256=hashlib.sha256( + rich.resolved_config.canonical_json.encode()).hexdigest(), + runtime_declaration_sha256=hashlib.sha256( + rich.execution_context.canonical_json.encode()).hexdigest(), + ) checked, resolved = DockerTrainingPreparationBridgeV1(profile).coordinator_material( rich, material, recipes ) diff --git a/examples/training_qualification/prepare-docker.config.example.json b/examples/training_qualification/prepare-docker.config.example.json index 2a4bf654..7a620961 100644 --- a/examples/training_qualification/prepare-docker.config.example.json +++ b/examples/training_qualification/prepare-docker.config.example.json @@ -7,6 +7,8 @@ "run_id": "REPLACE_UNUSED_RUN", "request_id": "REPLACE_UNUSED_REQUEST", "output_root": "/absolute/private/REPLACE_UNUSED_PREPARATION", + "model_output_root": "/absolute/private/REPLACE_UNUSED_MODEL_PREPARATION", + "model_seed_root": null, "prepared_publication": "/absolute/REPLACE_VERIFIED_V1_PUBLICATION", "resolved_config": "/absolute/REPLACE_CANONICAL_SFT_CONFIG.json", "runtime_release": "/absolute/REPLACE_ACTUAL_VERIFIED_RUNTIME_RELEASE.json", diff --git a/examples/training_qualification/prepare_docker.py b/examples/training_qualification/prepare_docker.py index 60d66004..dc04ac4e 100644 --- a/examples/training_qualification/prepare_docker.py +++ b/examples/training_qualification/prepare_docker.py @@ -128,6 +128,7 @@ def filesystem_probe(configuration): _bind_hub_api, prepare_model_snapshot, ) +from tuner.execution.providers.modal.mounted_io import copy_regular, hash_regular from tuner.project.context import ProjectContext from tuner.project.git_verification import ( GitCliLocalSourceInspector, @@ -242,6 +243,16 @@ def prepare(config, *, pushed_sources, authenticator, clock): normalized = common._document(config["resolved_config"]) if normalized.to_dict().get("dataset") != identity.to_dict(): raise ValueError("docker_prepared_configuration_mismatch") + model_receipt_path = None + if "model_output_root" in config: + model_receipt_path = Path(config["model_output_root"]) / "model-preparation.json" + docker_host.verify_model_receipt( + common.stable_read(model_receipt_path, 2 << 20), + selected=config["consumer"]["docker"], + model=normalized.to_dict()["model"], + configuration_sha256=sha(normalized.canonical_json.encode()), + runtime_declaration_sha256=sha(runtime.canonical_json.encode()), + ) context = ProjectContext.host( project_root=Path(config["project"]["root"]), engine_root=Path(config["project"]["engine_root"]), @@ -364,6 +375,12 @@ def spec(name): consumer["docker"]["material_file"] = spec("material.json") consumer["docker"]["profile_file"] = spec("docker-profile.json") consumer["docker"]["prepared_publication"] = config["prepared_publication"] + if model_receipt_path is not None: + data = common.stable_read(model_receipt_path, 2 << 20) + consumer["docker"]["model_preparation_file"] = { + "path": str(model_receipt_path), "sha256": sha(data), + "size_bytes": len(data), + } launch._write(root / "consumer-candidate.json", consumer) consumer = launch.load_config(root / "consumer-candidate.json") preview = docker_host.preview(consumer) @@ -399,34 +416,87 @@ def prepare_model( from Trainers.sft.runtime_v1 import RuntimeRoots, _require_local_model_snapshot sdk_preflight() - output = Path(config["output_root"]) - launch._private(output) - consumer = launch.load_config(output / "consumer.json") - _, rich, _, _, _ = docker_host._components(consumer) - if rich.execution_context != declaration(config): - raise ValueError("docker_model_configuration_changed") - model = rich.resolved_config.to_dict()["model"] - selected = consumer["docker"] + model_first = "model_output_root" in config + runtime = declaration(config) + if model_first: + output = Path(config["model_output_root"]) + if (not output.is_absolute() or output.exists() or output.is_symlink() + or output == Path(config["output_root"])): + raise ValueError("docker_model_output_used_or_invalid") + launch._private(output.parent) + if Path(config["output_root"]).exists() or Path(config["output_root"]).is_symlink(): + raise ValueError("docker_source_preparation_already_claimed") + selected = config["consumer"]["docker"] + normalized = common._document(config["resolved_config"]) + model = normalized.to_dict()["model"] + roots = runtime.to_dict()["roots"] + else: + output = Path(config["output_root"]) + launch._private(output) + consumer = launch.load_config(output / "consumer.json") + _, rich, _, _, _ = docker_host._components(consumer) + if rich.execution_context != runtime: + raise ValueError("docker_model_configuration_changed") + normalized = rich.resolved_config + model = normalized.to_dict()["model"] + selected = consumer["docker"] + roots = rich.execution_source.roots source, artifact = ( Path(selected["source_host_root"]), Path(selected["artifact_host_root"]), ) - for root in (source, artifact): + if (not source.is_absolute() or not artifact.is_absolute() + or source == artifact or source in artifact.parents + or artifact in source.parents): + raise ValueError("docker_model_roots_invalid") + if model_first and (output in {source, artifact} or output in source.parents + or output in artifact.parents or source in output.parents + or artifact in output.parents): + raise ValueError("docker_model_output_overlaps_mount") + if model_first and (source.exists() or source.is_symlink() + or artifact.exists() or artifact.is_symlink()): + raise ValueError("docker_model_fresh_roots_required") + seed = config.get("model_seed_root") if model_first else None + if seed is not None: + seed = Path(seed) + if (not seed.is_absolute() or seed in {output, source, artifact} + or seed in output.parents or seed in artifact.parents + or output in seed.parents or artifact in seed.parents + or source in seed.parents or seed in source.parents): + raise ValueError("docker_model_seed_invalid") + launch._private(seed) + if model_first: + # Consume the exact one-use model claim before creating any model bytes. + output.mkdir(mode=0o700) + launch._private(output) + launch._write( + output / "model-preparation-attempt.json", + {"model": model, "destination": str(artifact / "run/cache"), + "source_mutation": False, "seed_root": None if seed is None else str(seed)}, + ) + for root in ((artifact,) if model_first else (source, artifact)): if not root.exists(): launch._private(root.parent) root.mkdir(mode=0o700) launch._private(root) with ExitStack() as held: guards = [] - for root in (source, artifact): + for root in ((artifact,) if model_first else (source, artifact)): guard = _Root(root) held.callback(guard.stack.close) guards.append(guard) guard.check() - if any(source.iterdir()): + if model_first and (source.exists() or source.is_symlink()): + raise ValueError("docker_source_changed") + if source.exists() and any(source.iterdir()): raise ValueError("docker_source_not_empty") + seed_guard = None + if seed is not None: + seed_guard = _Root(seed) + held.callback(seed_guard.stack.close) + seed_guard.check() mapped = {} - for name, raw in rich.execution_source.roots.items(): + for name, raw in roots.items(): path = PurePosixPath(raw) if path.is_relative_to("/source"): mapped[name] = source.joinpath(*path.relative_to("/source").parts) @@ -446,17 +516,32 @@ def prepare_model( path.mkdir(mode=0o700) launch._private(path) persistent, scratch = output / "model-persistent", output / "model-scratch" - launch._write( - output / "model-preparation-attempt.json", - { - "model": model, - "destination": str(mapped["cache"]), - "source_mutation": False, - }, - ) + if not model_first: + launch._write( + output / "model-preparation-attempt.json", + {"model": model, "destination": str(mapped["cache"]), + "source_mutation": False}, + ) for path in (persistent, scratch): path.mkdir(mode=0o700) - snapshot = model_preparer( + if seed is not None: + relative = (Path("models--" + model["ref"].replace("/", "--")) + / "snapshots" / model["revision"]) + # Copy a private repository seed into this attempt's fresh cache. + # The unchanged shared preparer authenticates every reused member + # against pinned upstream Git/LFS metadata before destination use. + for row in docker_host.model_inventory(seed, seed / relative): + name = Path(row["path"]) + copied = copy_regular( + seed, seed / relative / name, persistent, + persistent / relative / name, maximum=row["size_bytes"], + ) + if copied != (row["size_bytes"], row["sha256"]): + raise ValueError("docker_model_seed_changed") + if hash_regular(seed, seed / relative / name, row["size_bytes"]) != copied: + raise ValueError("docker_model_seed_changed") + seed_guard.check() + options = dict( model_ref=model["ref"], revision=model["revision"], token=False, @@ -464,6 +549,7 @@ def prepare_model( destination_root=mapped["cache"], scratch_root=scratch, ) + snapshot = model_preparer(**options) expected = ( mapped["cache"] / "model" @@ -484,9 +570,13 @@ def prepare_model( ) for guard in guards: guard.check() - for root in (source, artifact): + if seed_guard is not None: + seed_guard.check() + for root in ((artifact,) if model_first else (source, artifact)): launch._private(root) - if any(source.iterdir()): + if model_first and (source.exists() or source.is_symlink()): + raise ValueError("docker_source_changed") + if source.exists() and any(source.iterdir()): raise ValueError("docker_source_changed") result = { "snapshot": str(checked), @@ -495,6 +585,16 @@ def prepare_model( "source_mutation": False, "provider_qualified": False, } + if model_first: + result = { + "schema_version": "synaptic-docker-model-preparation/v1", + "model": model, + "configuration_sha256": sha(normalized.canonical_json.encode()), + "runtime_declaration_sha256": sha(runtime.canonical_json.encode()), + "artifact_host_root": str(artifact), + "snapshot": str(checked), + "files": docker_host.model_inventory(artifact, checked), + } launch._write(output / "model-preparation.json", result) return result diff --git a/tests/examples/test_docker_material_preparation.py b/tests/examples/test_docker_material_preparation.py index ff47087c..3b60ab7f 100644 --- a/tests/examples/test_docker_material_preparation.py +++ b/tests/examples/test_docker_material_preparation.py @@ -2,8 +2,11 @@ """Genuine Git/publication/compiler and no-daemon Docker preparation controls.""" import importlib.util +import hashlib import json from pathlib import Path +import sys +from types import SimpleNamespace import pytest from tests.examples.test_hf_material_preparation import configured as source_configured # noqa: F401 @@ -11,6 +14,9 @@ from tests.runtime.test_packaged_runtime_releases import _release from tuner.training.coordinator_material import CoordinatorResolvedMaterial from tuner.training import default_recipe_registry +from tuner.execution.providers.modal.model_snapshot import ( + ModelSnapshotPreparationError, prepare_model_snapshot, +) ROOT = Path(__file__).resolve().parents[2] spec = importlib.util.spec_from_file_location( @@ -183,6 +189,171 @@ def model_preparer(**kwargs): assert len(calls) == 1 +def test_model_first_claim_precedes_source_and_receipt_binds_fresh_prepare(configured): + c, verifier, auth, clock, _ = configured + c["model_output_root"] = str(Path(c["output_root"]).parent / "model-attempt") + selected = c["consumer"]["docker"] + source = Path(selected["source_host_root"]) + artifact = Path(selected["artifact_host_root"]) + calls = [] + + def model_preparer(**kwargs): + assert not source.exists() + assert not Path(c["output_root"]).exists() + assert (Path(c["model_output_root"]) / "model-preparation-attempt.json").exists() + calls.append(kwargs) + target = (kwargs["destination_root"] / "model" + / ("models--" + kwargs["model_ref"].replace("/", "--")) + / "snapshots" / kwargs["revision"]) + target.mkdir(parents=True, mode=0o700) + (target / "synthetic-config.json").write_text('{"fixture":true}') + return target + + receipt = helper.prepare_model( + c, model_preparer=model_preparer, sdk_preflight=lambda: None, + ) + assert receipt["schema_version"] == "synaptic-docker-model-preparation/v1" + assert not source.exists() + assert len(calls) == 1 + with pytest.raises(ValueError): + helper.prepare_model(c, model_preparer=model_preparer, sdk_preflight=lambda: None) + assert len(calls) == 1 + member = (artifact / "run/cache/model" + / ("models--" + calls[0]["model_ref"].replace("/", "--")) + / "snapshots" / calls[0]["revision"] / "synthetic-config.json") + original = member.read_bytes() + member.write_bytes(b"changed before source") + with pytest.raises(ValueError): + helper.prepare(c, pushed_sources=verifier, authenticator=auth, clock=clock) + assert not Path(c["output_root"]).exists() + member.write_bytes(original) + helper.prepare(c, pushed_sources=verifier, authenticator=auth, clock=clock) + consumer = helper.launch.load_config(Path(c["output_root"]) / "consumer.json") + assert "model_preparation_file" in consumer["docker"] + assert not source.exists() + member.write_text("changed") + with pytest.raises(ValueError): + helper.docker_host.preview(consumer) + + +def test_model_first_failure_consumes_only_fresh_model_claim(configured): + c, _, _, _, _ = configured + c["model_output_root"] = str(Path(c["output_root"]).parent / "model-attempt") + selected = c["consumer"]["docker"] + + def fail(**kwargs): + assert (Path(c["model_output_root"]) / "model-preparation-attempt.json").exists() + raise RuntimeError("private fixture failure") + + with pytest.raises(RuntimeError): + helper.prepare_model(c, model_preparer=fail, sdk_preflight=lambda: None) + assert not Path(c["output_root"]).exists() + assert not Path(selected["source_host_root"]).exists() + with pytest.raises(ValueError): + helper.prepare_model(c, model_preparer=fail, sdk_preflight=lambda: None) + + +def test_model_first_rejects_source_created_during_expensive_preparation(configured): + c, _, _, _, _ = configured + c["model_output_root"] = str(Path(c["output_root"]).parent / "model-attempt") + source = Path(c["consumer"]["docker"]["source_host_root"]) + + def drift(**kwargs): + source.mkdir(mode=0o700) + target = (kwargs["destination_root"] / "model" + / ("models--" + kwargs["model_ref"].replace("/", "--")) + / "snapshots" / kwargs["revision"]) + target.mkdir(parents=True, mode=0o700) + (target / "synthetic-config.json").write_text('{"fixture":true}') + return target + + with pytest.raises(ValueError, match="docker_source_changed"): + helper.prepare_model(c, model_preparer=drift, sdk_preflight=lambda: None) + assert (Path(c["model_output_root"]) / "model-preparation-attempt.json").exists() + assert not (Path(c["model_output_root"]) / "model-preparation.json").exists() + + +@pytest.mark.parametrize("seed_state", ["verified", "corrupt", "symlink", "mutated", "cold"]) +def test_model_first_seed_is_independently_verified(configured, monkeypatch, seed_state): + c, _, _, _, _ = configured + c["model_output_root"] = str(Path(c["output_root"]).parent / "model-attempt") + model = json.loads(Path(c["resolved_config"]).read_bytes())["model"] + content = b'{"fixture":true}' + name = "synthetic-config.json" + seed = Path(c["output_root"]).parent / "retained-cache" + relative = (Path("models--" + model["ref"].replace("/", "--")) + / "snapshots" / model["revision"]) + if seed_state != "cold": + seed.mkdir(mode=0o700) + (seed / "old-attempt-claim.json").write_bytes(b"retained claim") + member = seed / relative + member.mkdir(parents=True, mode=0o700) + (member / name).write_bytes(content) + if seed_state == "corrupt": + (member / name).write_bytes(b"wrong fixture") + elif seed_state == "symlink": + (member / name).unlink() + (member / name).symlink_to(Path(c["resolved_config"])) + c["model_seed_root"] = str(seed) + calls = [] + + class API: + def __init__(self, *, endpoint, token): + pass + + def model_info(self, repo_id, *, revision, files_metadata, token): + assert (repo_id, revision) == (model["ref"], model["revision"]) + calls.append("metadata") + return SimpleNamespace( + sha=revision, + siblings=[SimpleNamespace( + rfilename=name, size=len(content), lfs=None, + blob_id=hashlib.sha1( + f"blob {len(content)}\0".encode() + content + ).hexdigest(), + )], + ) + + def download(**kwargs): + calls.append("download") + (kwargs["local_dir"] / name).write_bytes(content) + return str(kwargs["local_dir"]) + + monkeypatch.setitem(sys.modules, "huggingface_hub", SimpleNamespace( + HfApi=API, snapshot_download=download, + )) + if seed_state == "mutated": + original_copy = helper.copy_regular + + def mutate_after_copy(*args, **kwargs): + result = original_copy(*args, **kwargs) + (seed / relative / name).write_bytes(b"x" * len(content)) + return result + + monkeypatch.setattr(helper, "copy_regular", mutate_after_copy) + if seed_state in {"corrupt", "symlink", "mutated"}: + with pytest.raises((ValueError, ModelSnapshotPreparationError)): + helper.prepare_model( + c, model_preparer=prepare_model_snapshot, sdk_preflight=lambda: None, + ) + assert "download" not in calls + assert not Path(c["output_root"]).exists() + else: + receipt = helper.prepare_model( + c, model_preparer=prepare_model_snapshot, sdk_preflight=lambda: None, + ) + assert receipt["files"] == [{ + "path": name, "size_bytes": len(content), + "sha256": hashlib.sha256(content).hexdigest(), + }] + assert ("download" in calls) == (seed_state == "cold") + if seed_state == "verified": + assert (seed / relative / name).read_bytes() == content + if seed_state != "cold": + assert (seed / "old-attempt-claim.json").read_bytes() == b"retained claim" + assert Path(c["model_output_root"]).exists() + + @pytest.mark.parametrize("kind", ["source_member", "artifact_symlink", "artifact_mode"]) def test_hostile_model_roots_refuse_before_model_calls(configured, kind): c, verifier, auth, clock, _ = configured From 4a8bb138672d60318479286797ffd75572407054 Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 13:15:03 -0400 Subject: [PATCH 12/12] Keep Docker model checks on start and retain cold receipt --- .../fine-tuning/reference/sft-training.md | 5 +- .../fine-tuning/reference/sft-training.md | 5 +- .skills/fine-tuning/reference/sft-training.md | 5 +- .../docker-torch-import-r2-diagnosis.md | 15 +++ examples/training_qualification/README.md | 9 +- .../training_qualification/docker_host.py | 107 +++++++++++++----- .../training_qualification/prepare_docker.py | 4 +- .../test_docker_material_preparation.py | 71 ++++++++++-- .../test_docker_training_qualification.py | 94 +++++++++++++++ 9 files changed, 273 insertions(+), 42 deletions(-) diff --git a/.agents/skills/fine-tuning/reference/sft-training.md b/.agents/skills/fine-tuning/reference/sft-training.md index d1b32d7a..976e3a92 100644 --- a/.agents/skills/fine-tuning/reference/sft-training.md +++ b/.agents/skills/fine-tuning/reference/sft-training.md @@ -456,7 +456,10 @@ then authenticates fresh source and existing immutable v1 input before publishin canonical material. A private read-only seed cache may reduce downloads only after pinned upstream metadata and every reused Git/LFS member are verified. The old seed claim is never adopted. `prepare` binds the complete model receipt; -public start rechecks its bytes before Docker effects. See `examples/training_qualification/README.md` +public start rechecks its bytes before Docker effects. A missing pinned revision +in an otherwise private seed takes the normal cold path; redirects reject. +Cold public reads use the receipt retained with the attempt and do not require +current model weights or the original preparation root. See `examples/training_qualification/README.md` for the template and exact commands. Full public preflight remains deferred until host composition; missing image/runtime/model qualification still refuses. The CPU `probe-filesystem` command measures actual native writer modes and host diff --git a/.claude/skills/fine-tuning/reference/sft-training.md b/.claude/skills/fine-tuning/reference/sft-training.md index d1b32d7a..976e3a92 100644 --- a/.claude/skills/fine-tuning/reference/sft-training.md +++ b/.claude/skills/fine-tuning/reference/sft-training.md @@ -456,7 +456,10 @@ then authenticates fresh source and existing immutable v1 input before publishin canonical material. A private read-only seed cache may reduce downloads only after pinned upstream metadata and every reused Git/LFS member are verified. The old seed claim is never adopted. `prepare` binds the complete model receipt; -public start rechecks its bytes before Docker effects. See `examples/training_qualification/README.md` +public start rechecks its bytes before Docker effects. A missing pinned revision +in an otherwise private seed takes the normal cold path; redirects reject. +Cold public reads use the receipt retained with the attempt and do not require +current model weights or the original preparation root. See `examples/training_qualification/README.md` for the template and exact commands. Full public preflight remains deferred until host composition; missing image/runtime/model qualification still refuses. The CPU `probe-filesystem` command measures actual native writer modes and host diff --git a/.skills/fine-tuning/reference/sft-training.md b/.skills/fine-tuning/reference/sft-training.md index d1b32d7a..976e3a92 100644 --- a/.skills/fine-tuning/reference/sft-training.md +++ b/.skills/fine-tuning/reference/sft-training.md @@ -456,7 +456,10 @@ then authenticates fresh source and existing immutable v1 input before publishin canonical material. A private read-only seed cache may reduce downloads only after pinned upstream metadata and every reused Git/LFS member are verified. The old seed claim is never adopted. `prepare` binds the complete model receipt; -public start rechecks its bytes before Docker effects. See `examples/training_qualification/README.md` +public start rechecks its bytes before Docker effects. A missing pinned revision +in an otherwise private seed takes the normal cold path; redirects reject. +Cold public reads use the receipt retained with the attempt and do not require +current model weights or the original preparation root. See `examples/training_qualification/README.md` for the template and exact commands. Full public preflight remains deferred until host composition; missing image/runtime/model qualification still refuses. The CPU `probe-filesystem` command measures actual native writer modes and host diff --git a/docs/review/docker-torch-import-r2-diagnosis.md b/docs/review/docker-torch-import-r2-diagnosis.md index 28bfce7f..d2b5c5b1 100644 --- a/docs/review/docker-torch-import-r2-diagnosis.md +++ b/docs/review/docker-torch-import-r2-diagnosis.md @@ -664,3 +664,18 @@ release and image C pins, daemon/device, source and model custody, prepared input publication, fresh names, and the original 3-step full-precision SFT recipe with 900-second/2-CPU/16-GiB/GPU-0/network-none bounds. Do not run a model request, container or training job on this implementation evidence alone. + +The r16 independent review found that cold known-run operations had inherited +the live model-byte check, an absent seeded revision was rejected instead of +treated as a cold miss, and inventory hashing reopened the root pathname after +acquiring a scan descriptor. The r17 host-only repair retains the exact model +receipt inside the public attempt. Start still validates current model bytes +before its sole create; cold known-run operations check the committed receipt, +configuration and run identity without reading weights or the old preparation +root. A missing seed revision now downloads through the unchanged pinned +preparer, while existing redirects and mutated members reject. Inventory hashes +leaves through their held scan-parent descriptors and compares identities before +and after reading. Tiny CPU public-start and transient-ancestor-swap fixtures +exercise these boundaries; they do not qualify the desktop model or diagnose r2. +Public plan authenticates the receipt commitment without reading weights; the +sole live-byte recheck in the public consumer occurs on start before effects. diff --git a/examples/training_qualification/README.md b/examples/training_qualification/README.md index 2f933204..31b0dd32 100644 --- a/examples/training_qualification/README.md +++ b/examples/training_qualification/README.md @@ -200,6 +200,8 @@ mount's `run/cache`. Optional `model_seed_root` points to an independently held, private read-only repository cache; each present member is copied into fresh roots and checked against current pinned Git/LFS metadata. A corrupt, redirected or changed seed member rejects; absent members use the normal cold download path. +An absent entire pinned revision under an otherwise private seed is also a +cold miss; an existing redirected revision or member rejects. The seed's old claim and bytes stay untouched and never become new authority. The native snapshot validator checks the resulting offline path and a complete bounded file inventory is retained in `model-preparation.json`. @@ -207,7 +209,12 @@ bounded file inventory is retained in `model-preparation.json`. `prepare` first rechecks that receipt and actual model bytes against its declared configuration, then authenticates fresh pushed source and compiles material. Its consumer binds the exact receipt bytes; public `start` rechecks the model inventory -before Docker effects. The source mount remains absent until the source stage. +before Docker effects. Public start retains its own committed receipt copy. +Public `plan` authenticates receipt identity but does not attest current model +bytes; `start` performs that live check. +Later cold observe/cancel/reconcile/verify/download authenticate that retained +receipt, configuration and run identity without reopening weights or depending +on the preparation root. The source mount remains absent until the source stage. Writable runtime directories are private and disjoint; the exact prepared-input slot is never adopted. Each model and source root is one-use. A failed or uncertain model preparation cannot be replayed under the same claim. diff --git a/examples/training_qualification/docker_host.py b/examples/training_qualification/docker_host.py index b75174b3..93c6f9db 100644 --- a/examples/training_qualification/docker_host.py +++ b/examples/training_qualification/docker_host.py @@ -31,7 +31,6 @@ DockerTrainingPreparationBridgeV1, ) from tuner.execution.providers.docker_provider_v1.local_source import _Root -from tuner.execution.providers.modal.mounted_io import hash_regular from tuner.execution.providers.docker_provider_v1.local_artifacts import ( DockerLocalArtifactsV1, _DIR, @@ -107,7 +106,47 @@ def _profile(raw): return profile -def model_inventory(artifact_root, snapshot): +def _hash_scanned_member(parent_fd, name, observed, maximum): + """Read the scanned leaf through its held parent, including identity checks.""" + if not stat.S_ISREG(observed.st_mode) or not 0 <= observed.st_size <= maximum: + raise ValueError("docker_model_member_invalid") + try: + fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=parent_fd) + except OSError: + raise ValueError("docker_model_member_changed") from None + try: + before = os.fstat(fd) + identity = (observed.st_dev, observed.st_ino, observed.st_size, observed.st_mtime_ns) + if not stat.S_ISREG(before.st_mode) or ( + before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns + ) != identity: + raise ValueError("docker_model_member_changed") + digest = hashlib.sha256() + size = 0 + while True: + chunk = os.read(fd, min(1 << 20, maximum + 1 - size)) + if not chunk: + break + size += len(chunk) + if size > maximum: + raise ValueError("docker_model_inventory_bound") + digest.update(chunk) + after = os.fstat(fd) + current = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + if size != observed.st_size or ( + after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns + ) != identity or ( + current.st_dev, current.st_ino, current.st_size, current.st_mtime_ns + ) != identity: + raise ValueError("docker_model_member_changed") + return size, digest.hexdigest() + except OSError: + raise ValueError("docker_model_member_changed") from None + finally: + os.close(fd) + + +def model_inventory(artifact_root, snapshot, *, missing_ok=False): """Hash the complete private offline snapshot through bounded regular reads.""" artifact_root = Path(artifact_root) snapshot = Path(snapshot) @@ -122,7 +161,16 @@ def model_inventory(artifact_root, snapshot): parent = guard.fd chain = [] for part in snapshot.relative_to(artifact_root).parts: - child = os.open(part, _DIR, dir_fd=parent) + try: + child = os.open(part, _DIR, dir_fd=parent) + except FileNotFoundError: + if not missing_ok: + raise + DockerLocalArtifactsV1._recheck(chain) + guard.check() + return [] + except OSError: + raise ValueError("docker_model_snapshot_path_invalid") from None stack.callback(os.close, child) chain.append((parent, part, child)) parent = child @@ -149,9 +197,9 @@ def model_inventory(artifact_root, snapshot): if len(rows) >= 20_000 or total + info.st_size > 1 << 40: raise ValueError("docker_model_inventory_bound") path = directory / entry.name - size, digest = hash_regular(artifact_root, path, info.st_size) - if size != info.st_size: - raise ValueError("docker_model_member_changed") + size, digest = _hash_scanned_member( + directory_fd, entry.name, info, info.st_size + ) rows.append({"path": path.relative_to(snapshot).as_posix(), "size_bytes": size, "sha256": digest}) total += size @@ -165,10 +213,8 @@ def model_inventory(artifact_root, snapshot): def verify_model_receipt(raw, *, selected, model, configuration_sha256, - runtime_declaration_sha256): + runtime_declaration_sha256, live_bytes=True): """Bind fresh prepared bytes to the later public consumer's exact inputs.""" - from Trainers.sft.runtime_v1 import RuntimeRoots, _require_local_model_snapshot - receipt = _json(raw) artifact_root = Path(selected["artifact_host_root"]) cache = artifact_root / "run/cache" @@ -185,23 +231,26 @@ def verify_model_receipt(raw, *, selected, model, configuration_sha256, or receipt["snapshot"] != str(expected) or not isinstance(receipt["files"], list)): raise ValueError("docker_model_receipt_mismatch") - roots = RuntimeRoots( - engine=Path(selected["source_host_root"]) / "engine", - project=Path(selected["source_host_root"]) / "project", - artifacts=artifact_root / "run/artifacts", state=artifact_root / "run/state", - tracking=artifact_root / "run/tracking", cache=cache, - tmp=artifact_root / "run/tmp", - ) - _require_local_model_snapshot(model, roots, { - "SYNAPTIC_CACHE_ROOT": str(cache), "SYNAPTIC_MODEL_SNAPSHOT": str(expected), - "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", - }) - if model_inventory(artifact_root, expected) != receipt["files"]: - raise ValueError("docker_model_receipt_bytes_changed") + if live_bytes: + from Trainers.sft.runtime_v1 import RuntimeRoots, _require_local_model_snapshot + + roots = RuntimeRoots( + engine=Path(selected["source_host_root"]) / "engine", + project=Path(selected["source_host_root"]) / "project", + artifacts=artifact_root / "run/artifacts", state=artifact_root / "run/state", + tracking=artifact_root / "run/tracking", cache=cache, + tmp=artifact_root / "run/tmp", + ) + _require_local_model_snapshot(model, roots, { + "SYNAPTIC_CACHE_ROOT": str(cache), "SYNAPTIC_MODEL_SNAPSHOT": str(expected), + "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", + }) + if model_inventory(artifact_root, expected) != receipt["files"]: + raise ValueError("docker_model_receipt_bytes_changed") return receipt -def _components(config): +def _components(config, *, live_model_bytes=False): if config["provider"] != "docker": raise ValueError("Docker provider required") selected = config["docker"] @@ -254,6 +303,7 @@ def _components(config): rich.resolved_config.canonical_json.encode()).hexdigest(), runtime_declaration_sha256=hashlib.sha256( rich.execution_context.canonical_json.encode()).hexdigest(), + live_bytes=live_model_bytes, ) checked, resolved = DockerTrainingPreparationBridgeV1(profile).coordinator_material( rich, material, recipes @@ -281,10 +331,13 @@ def preview(config): def retain_inputs(config, root): # Preserve the existing canonical documents, never serialize a new rich type. - for key, name in ( + references = [ ("material_file", "docker-material.json"), ("profile_file", "docker-profile.json"), - ): + ] + if "model_preparation_file" in config["docker"]: + references.append(("model_preparation_file", "docker-model-preparation.json")) + for key, name in references: raw = read_committed(config["docker"][key]) target = root / name _write_bytes(target, raw) @@ -422,7 +475,9 @@ def open_host( if mode == "start": _require_runtime_root_platform() selected = config["docker"] - profile, rich, material, recipes, basis = _components(config) + profile, rich, material, recipes, basis = _components( + config, live_model_bytes=mode == "start" + ) run = TrainingRunRef(material.run_id, basis.project_ref) if config["run"] != run.to_dict(): raise ValueError("configured run differs from material") diff --git a/examples/training_qualification/prepare_docker.py b/examples/training_qualification/prepare_docker.py index dc04ac4e..38c9ddf6 100644 --- a/examples/training_qualification/prepare_docker.py +++ b/examples/training_qualification/prepare_docker.py @@ -530,7 +530,9 @@ def prepare_model( # Copy a private repository seed into this attempt's fresh cache. # The unchanged shared preparer authenticates every reused member # against pinned upstream Git/LFS metadata before destination use. - for row in docker_host.model_inventory(seed, seed / relative): + for row in docker_host.model_inventory( + seed, seed / relative, missing_ok=True + ): name = Path(row["path"]) copied = copy_regular( seed, seed / relative / name, persistent, diff --git a/tests/examples/test_docker_material_preparation.py b/tests/examples/test_docker_material_preparation.py index 3b60ab7f..aec1c6da 100644 --- a/tests/examples/test_docker_material_preparation.py +++ b/tests/examples/test_docker_material_preparation.py @@ -4,9 +4,11 @@ import importlib.util import hashlib import json +import os from pathlib import Path import sys from types import SimpleNamespace +from contextlib import contextmanager import pytest from tests.examples.test_hf_material_preparation import configured as source_configured # noqa: F401 @@ -232,8 +234,9 @@ def model_preparer(**kwargs): assert "model_preparation_file" in consumer["docker"] assert not source.exists() member.write_text("changed") + assert helper.docker_host.preview(consumer)["run"] with pytest.raises(ValueError): - helper.docker_host.preview(consumer) + helper.docker_host._components(consumer, live_model_bytes=True) def test_model_first_failure_consumes_only_fresh_model_claim(configured): @@ -273,7 +276,10 @@ def drift(**kwargs): assert not (Path(c["model_output_root"]) / "model-preparation.json").exists() -@pytest.mark.parametrize("seed_state", ["verified", "corrupt", "symlink", "mutated", "cold"]) +@pytest.mark.parametrize("seed_state", [ + "verified", "corrupt", "symlink", "revision_symlink", "mutated", + "absent_revision", "cold", +]) def test_model_first_seed_is_independently_verified(configured, monkeypatch, seed_state): c, _, _, _, _ = configured c["model_output_root"] = str(Path(c["output_root"]).parent / "model-attempt") @@ -287,13 +293,17 @@ def test_model_first_seed_is_independently_verified(configured, monkeypatch, see seed.mkdir(mode=0o700) (seed / "old-attempt-claim.json").write_bytes(b"retained claim") member = seed / relative - member.mkdir(parents=True, mode=0o700) - (member / name).write_bytes(content) - if seed_state == "corrupt": - (member / name).write_bytes(b"wrong fixture") - elif seed_state == "symlink": - (member / name).unlink() - (member / name).symlink_to(Path(c["resolved_config"])) + if seed_state == "revision_symlink": + member.parent.mkdir(parents=True, mode=0o700) + member.symlink_to(Path(c["resolved_config"]), target_is_directory=True) + elif seed_state != "absent_revision": + member.mkdir(parents=True, mode=0o700) + (member / name).write_bytes(content) + if seed_state == "corrupt": + (member / name).write_bytes(b"wrong fixture") + elif seed_state == "symlink": + (member / name).unlink() + (member / name).symlink_to(Path(c["resolved_config"])) c["model_seed_root"] = str(seed) calls = [] @@ -331,7 +341,7 @@ def mutate_after_copy(*args, **kwargs): return result monkeypatch.setattr(helper, "copy_regular", mutate_after_copy) - if seed_state in {"corrupt", "symlink", "mutated"}: + if seed_state in {"corrupt", "symlink", "revision_symlink", "mutated"}: with pytest.raises((ValueError, ModelSnapshotPreparationError)): helper.prepare_model( c, model_preparer=prepare_model_snapshot, sdk_preflight=lambda: None, @@ -346,7 +356,7 @@ def mutate_after_copy(*args, **kwargs): "path": name, "size_bytes": len(content), "sha256": hashlib.sha256(content).hexdigest(), }] - assert ("download" in calls) == (seed_state == "cold") + assert ("download" in calls) == (seed_state in {"cold", "absent_revision"}) if seed_state == "verified": assert (seed / relative / name).read_bytes() == content if seed_state != "cold": @@ -354,6 +364,45 @@ def mutate_after_copy(*args, **kwargs): assert Path(c["model_output_root"]).exists() +def test_model_inventory_hashes_held_scan_during_transient_ancestor_swap( + tmp_path, monkeypatch +): + root, alternate, held = (tmp_path / name for name in ("model", "alternate", "held")) + for directory, content in ((root, b"trusted"), (alternate, b"hostile")): + directory.mkdir(mode=0o700) + snapshot = directory / "snapshot" + snapshot.mkdir(mode=0o700) + (snapshot / "member.bin").write_bytes(content) + original_scandir = os.scandir + swapped = False + + @contextmanager + def swap_while_scanning(directory_fd): + nonlocal swapped + with original_scandir(directory_fd) as entries: + if not swapped: + swapped = True + os.rename(root, held) + os.rename(alternate, root) + try: + yield entries + finally: + os.rename(root, alternate) + os.rename(held, root) + else: + yield entries + + monkeypatch.setattr(helper.docker_host.os, "scandir", swap_while_scanning) + rows = helper.docker_host.model_inventory(root, root / "snapshot") + assert swapped + assert rows == [{ + "path": "member.bin", "size_bytes": len(b"trusted"), + "sha256": hashlib.sha256(b"trusted").hexdigest(), + }] + assert (root / "snapshot/member.bin").read_bytes() == b"trusted" + assert (alternate / "snapshot/member.bin").read_bytes() == b"hostile" + + @pytest.mark.parametrize("kind", ["source_member", "artifact_symlink", "artifact_mode"]) def test_hostile_model_roots_refuse_before_model_calls(configured, kind): c, verifier, auth, clock, _ = configured diff --git a/tests/examples/test_docker_training_qualification.py b/tests/examples/test_docker_training_qualification.py index 14bd532f..ff8a8000 100644 --- a/tests/examples/test_docker_training_qualification.py +++ b/tests/examples/test_docker_training_qualification.py @@ -304,6 +304,100 @@ def no_resolution(*args, **kwargs): assert run.to_dict() == preview["run"] +def bind_tiny_model_receipt(config, kwargs): + _, rich, _, _, _ = docker_host._components(config) + model = rich.resolved_config.to_dict()["model"] + artifact = kwargs["artifact_root"] + snapshot = (artifact / "run/cache/model" + / ("models--" + model["ref"].replace("/", "--")) + / "snapshots" / model["revision"]) + current = artifact + for part in snapshot.relative_to(artifact).parts: + current /= part + current.mkdir(mode=0o700) + member = snapshot / "synthetic-config.json" + member.write_bytes(b'{"fixture":true}') + receipt = { + "schema_version": "synaptic-docker-model-preparation/v1", + "model": model, + "configuration_sha256": hashlib.sha256( + rich.resolved_config.canonical_json.encode() + ).hexdigest(), + "runtime_declaration_sha256": hashlib.sha256( + rich.execution_context.canonical_json.encode() + ).hexdigest(), + "artifact_host_root": str(artifact), + "snapshot": str(snapshot), + "files": docker_host.model_inventory(artifact, snapshot), + } + receipt_path = artifact.parent / "model-preparation.json" + raw = json.dumps(receipt, sort_keys=True).encode() + config["docker"]["model_preparation_file"] = committed(receipt_path, raw) + return artifact, snapshot, member, receipt_path, raw + + +def test_model_first_public_start_rejects_changed_bytes_before_create(configured): + config, kwargs, commands, _, _, options = configured + _, _, member, _, _ = bind_tiny_model_receipt(config, kwargs) + member.write_bytes(b"changed bytes") + assert launch.preview(config)["run"] + with pytest.raises(ValueError): + launch.launch(config, adapter_options=options) + assert not [c for c in commands if c[:2] == ("container", "create")] + assert Path(config["state_root"]).exists() + with pytest.raises(FileExistsError): + launch.launch(config, adapter_options=options) + + +def test_model_first_public_start_and_cold_reads_without_weights( + configured, monkeypatch +): + config, kwargs, commands, state, roles, options = configured + artifact, snapshot, member, receipt_path, raw = bind_tiny_model_receipt(config, kwargs) + assert launch.preview(config)["run"] + receipt_path.write_bytes(b"changed receipt") + with pytest.raises(ValueError): + launch.launch(config, adapter_options=options) + assert not [c for c in commands if c[:2] == ("container", "create")] + assert not Path(config["state_root"]).exists() + receipt_path.write_bytes(raw) + result = launch.launch(config, adapter_options=options) + assert result["start"]["accepted"] + assert len([c for c in commands if c[:2] == ("container", "create")]) == 1 + retained = launch.load_config(Path(config["state_root"]) / "config.json") + retained_receipt = Path(retained["docker"]["model_preparation_file"]["path"]) + assert retained_receipt != receipt_path and retained_receipt.read_bytes() == raw + receipt_path.unlink() + member.unlink() + snapshot.rmdir() + assert launch.preview(retained)["run"] == result["start"]["run"] + assert launch.known(retained, "observe", adapter_options=options)["state"] == "running" + state["status"] = "exited" + (artifact / "native").mkdir(mode=0o700) + values = native_values(kwargs, monkeypatch) + for role, relative in roles.items(): + (artifact / relative).write_bytes(values[role]) + assert launch.known(retained, "observe", adapter_options=options)["state"] == "succeeded" + assert launch.known(retained, "verify", adapter_options=options)["verified"] + output = Path(config["state_root"]) / "downloads" + downloaded = launch.known( + retained, "download", destination=output, adapter_options=options + ) + assert len(downloaded["paths"]) == 5 + for role in roles: + assert (output / (role + ".artifact")).read_bytes() == values[role] + wrong_run = json.loads(json.dumps(retained)) + wrong_run["run"]["run_id"] = "another-run" + with pytest.raises(ValueError): + launch.known(wrong_run, "observe", adapter_options=options) + retained_receipt.write_bytes(b"changed receipt") + with pytest.raises(ValueError): + launch.known(retained, "observe", adapter_options=options) + with pytest.raises(ValueError): + launch.preview(retained) + assert len([c for c in commands if c[:2] == ("container", "create")]) == 1 + + def test_unknown_submit_and_existing_attempt_never_create_again(configured): config, _, commands, state, _, options = configured state["unknown_create"] = True