diff --git a/.agents/skills/fine-tuning/SKILL.md b/.agents/skills/fine-tuning/SKILL.md index 73c91863..1172feb9 100644 --- a/.agents/skills/fine-tuning/SKILL.md +++ b/.agents/skills/fine-tuning/SKILL.md @@ -57,6 +57,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Check SFT loss masking (tokenizer only) | `python tuner.py doctor sft-mask --dataset-path --model ` | | Inspect native training host candidate | `python tuner.py doctor native-training --json` — read-only platform/Python/package metadata; see `reference/native-local-training.md` | | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | +| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for bounded guarded, worker-context prefix, and worker-entry probes; each live command needs exact review; see `reference/sft-training.md` | | Native MLX prepared SFT candidate | Internal exact-token batch/loss and runtime comparison only; no native launch CLI or API provider yet; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.agents/skills/fine-tuning/reference/sft-training.md b/.agents/skills/fine-tuning/reference/sft-training.md index 3029c594..976e3a92 100644 --- a/.agents/skills/fine-tuning/reference/sft-training.md +++ b/.agents/skills/fine-tuning/reference/sft-training.md @@ -450,10 +450,16 @@ See the checked-in example README for commands and exact fixture coverage. The checkout-owned `examples/training_qualification/prepare_docker.py` closes material setup for the public Docker consumer. With an installed engine, -`plan` validates declared release/model/layout, `prepare` authenticates genuine -sources and existing immutable v1 input before publishing canonical material, -and `prepare-model` explicitly downloads the pinned public snapshot into the -private artifact-mount cache. See `examples/training_qualification/README.md` +`plan` validates declared release/model/layout, `prepare-model` first prepares +the pinned public snapshot into a fresh private artifact cache, and `prepare` +then authenticates fresh source and existing immutable v1 input before publishing +canonical material. A private read-only seed cache may reduce downloads only +after pinned upstream metadata and every reused Git/LFS member are verified. +The old seed claim is never adopted. `prepare` binds the complete model receipt; +public start rechecks its bytes before Docker effects. A missing pinned revision +in an otherwise private seed takes the normal cold path; redirects reject. +Cold public reads use the receipt retained with the attempt and do not require +current model weights or the original preparation root. See `examples/training_qualification/README.md` for the template and exact commands. Full public preflight remains deferred until host composition; missing image/runtime/model qualification still refuses. The CPU `probe-filesystem` command measures actual native writer modes and host @@ -492,6 +498,89 @@ the consumed claim, roots, container and retained logs; the diagnostics never authorize replay. See `docs/review/offline-sft-worker-diagnostics.md` for the October 20 evidence boundary, CPU regressions and fresh-run prerequisites. +For a fresh, credential-free Docker Torch import reproduction, review the exact +command before effects. The checked-in `scripts/diagnose_docker_torch_import.py` +requires `--docker`, an empty `--docker-config`, `--release`, independently +verified `--expected-release-sha256`, `--daemon-id`, and a nonexistent private +`--output-dir`. Guarded mode additionally requires `--source-root`, +`--source-lock`, independently reviewed `--expected-source-lock-sha256`, +`--expected-manifest-sha256`, and `--expected-closure-digest` from the retained +attempt. It validates every staged worker member before executing mounted code; +the result records those commitments. Run from the reviewed checkout so its +trusted closure verifier is the one checked in with the helper. + +The helper claims a unique name before Docker create, authenticates the exact +new container before start, and retains it. Docker CLI output is capped at 8 KiB +in memory; the container requests JSON log rotation at 1 MiB with one file. +The import execution has a +120-second host deadline. At expiry the helper issues at most one stop request +to the authenticated new container and separately inspects terminal state; +cleanup takes additional bounded time, and a stop acknowledgment alone is not +quiescence. An uncertain create/start/stop remains consumed in `claim.json` and +`events.jsonl`, with no automatic replay or container removal. The default +probe asks for at most 2 CPUs, 4 GiB, no network/GPU/model mount, and a +read-only root. +Bare mode mounts no source. Guarded mode mounts the independently authenticated +retained source and runs the worker closure check, module guard, trainer path +setup, environment bootstrap and Torch import. Neither runs training. `PASS` +only says Torch imports in the reduced probe; it does not diagnose an earlier +failure. Review the pinned package/runtime source before changing a profile, +image or runtime policy. +The exact r2 CPU evidence and its unresolved runtime difference are in +`docs/review/docker-torch-import-r2-diagnosis.md`. + +The guarded-only `--gpu-device 0` option is an import-only comparison that +requires separate exact root review before any live use. It requests only +device 0, verifies Docker retained that device request before start, removes +the CPU probe's visibility suppressors, and records the scope in its claim. +`gpu_used=null` means actual device use was not measured. The same 8-KiB +output budget, 120-second execution deadline, owned cleanup, source +authentication and closed child codes apply. It neither invokes the trainer +nor loads a model; a PASS or failure cannot retrospectively diagnose an +earlier trainer attempt. See the exact proposed command and remaining +environment/user/filesystem gaps in the r2 diagnosis note. + +For the worker-context child, keep supplied environment bytes distinct from +the environment observed after isolated Python startup. Require every +supplied key unchanged; the only reviewed addition is +`LC_CTYPE=C.UTF-8`, recorded as a closed delta with separate digests. +Any other addition or change rejects. Do not reset the observed environment +or reinterpret it as a historical capture of the failed r2 child. See the r10 +repair and exact unexecuted command in the diagnosis note. + +The optional `--worker-context` mode requires independently reviewed +`--expected-workload-sha256`, `--expected-runtime-sha256`, and +`--expected-trainer-sha256` alongside guarded source commitments. It +reconstructs the pinned workload's isolated child environment, launches a +separate `-I` child, loads the authenticated worker closure and guard, and +executes only the authenticated trainer prefix through `import torch`. +The prefix excludes the `UNSLOTH_IMPORT` marker and all later trainer/model +code. It records closed environment, path, file and cwd digests and keeps the +existing claim, output budget, deadline and owned cleanup. Its writable +`/artifacts` is a small private tmpfs with no model weights. Review the exact +workload/source hashes, device scope, host user, caps and filesystem before +each fresh use. A reconstructed child environment is a source-and-plan +deduction; it is not a historical capture of the failed child. + +`--worker-context --worker-entry` selects a separate, still import-only +diagnostic. It reconstructs the pinned runtime's trainer argument vector, +including a one-byte sealed stand-in for the prepared dataset descriptor, +then invokes the authenticated worker's real `main()` and nested +`runpy.run_path` trainer entry. A source-bound line trace raises at the +`UNSLOTH_IMPORT` marker after Torch import, while a narrow import-call and +module-finder barrier prevents Unsloth entry even if tracing is disabled or +that module name was cached. A PASS +requires the worker's closed stop label and real trainer `__main__` module +binding. The worker still performs its own argument, closure and import-guard +checks. It records closed environment digests before and after Torch import, +separate from the supplied and interpreter-startup digests. The diagnostic +starts through `-I -c` to install that boundary, so it +does not reproduce the original direct `-I` script startup. The sealed data +byte, reduced filesystem and absent model weights also differ from the failed +attempt. Review and admit a fresh one-use command before any live execution; +do not infer the historical failure cause from either result. See the r12 +proposal in the diagnosis note. + After an intentional worker source change, check and refresh existing manifests with the checked-in maintainers in dependency order: `regenerate_offline_sft_worker_closure.py`, diff --git a/.claude/skills/fine-tuning/SKILL.md b/.claude/skills/fine-tuning/SKILL.md index 73c91863..1172feb9 100644 --- a/.claude/skills/fine-tuning/SKILL.md +++ b/.claude/skills/fine-tuning/SKILL.md @@ -57,6 +57,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Check SFT loss masking (tokenizer only) | `python tuner.py doctor sft-mask --dataset-path --model ` | | Inspect native training host candidate | `python tuner.py doctor native-training --json` — read-only platform/Python/package metadata; see `reference/native-local-training.md` | | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | +| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for bounded guarded, worker-context prefix, and worker-entry probes; each live command needs exact review; see `reference/sft-training.md` | | Native MLX prepared SFT candidate | Internal exact-token batch/loss and runtime comparison only; no native launch CLI or API provider yet; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.claude/skills/fine-tuning/reference/sft-training.md b/.claude/skills/fine-tuning/reference/sft-training.md index 3029c594..976e3a92 100644 --- a/.claude/skills/fine-tuning/reference/sft-training.md +++ b/.claude/skills/fine-tuning/reference/sft-training.md @@ -450,10 +450,16 @@ See the checked-in example README for commands and exact fixture coverage. The checkout-owned `examples/training_qualification/prepare_docker.py` closes material setup for the public Docker consumer. With an installed engine, -`plan` validates declared release/model/layout, `prepare` authenticates genuine -sources and existing immutable v1 input before publishing canonical material, -and `prepare-model` explicitly downloads the pinned public snapshot into the -private artifact-mount cache. See `examples/training_qualification/README.md` +`plan` validates declared release/model/layout, `prepare-model` first prepares +the pinned public snapshot into a fresh private artifact cache, and `prepare` +then authenticates fresh source and existing immutable v1 input before publishing +canonical material. A private read-only seed cache may reduce downloads only +after pinned upstream metadata and every reused Git/LFS member are verified. +The old seed claim is never adopted. `prepare` binds the complete model receipt; +public start rechecks its bytes before Docker effects. A missing pinned revision +in an otherwise private seed takes the normal cold path; redirects reject. +Cold public reads use the receipt retained with the attempt and do not require +current model weights or the original preparation root. See `examples/training_qualification/README.md` for the template and exact commands. Full public preflight remains deferred until host composition; missing image/runtime/model qualification still refuses. The CPU `probe-filesystem` command measures actual native writer modes and host @@ -492,6 +498,89 @@ the consumed claim, roots, container and retained logs; the diagnostics never authorize replay. See `docs/review/offline-sft-worker-diagnostics.md` for the October 20 evidence boundary, CPU regressions and fresh-run prerequisites. +For a fresh, credential-free Docker Torch import reproduction, review the exact +command before effects. The checked-in `scripts/diagnose_docker_torch_import.py` +requires `--docker`, an empty `--docker-config`, `--release`, independently +verified `--expected-release-sha256`, `--daemon-id`, and a nonexistent private +`--output-dir`. Guarded mode additionally requires `--source-root`, +`--source-lock`, independently reviewed `--expected-source-lock-sha256`, +`--expected-manifest-sha256`, and `--expected-closure-digest` from the retained +attempt. It validates every staged worker member before executing mounted code; +the result records those commitments. Run from the reviewed checkout so its +trusted closure verifier is the one checked in with the helper. + +The helper claims a unique name before Docker create, authenticates the exact +new container before start, and retains it. Docker CLI output is capped at 8 KiB +in memory; the container requests JSON log rotation at 1 MiB with one file. +The import execution has a +120-second host deadline. At expiry the helper issues at most one stop request +to the authenticated new container and separately inspects terminal state; +cleanup takes additional bounded time, and a stop acknowledgment alone is not +quiescence. An uncertain create/start/stop remains consumed in `claim.json` and +`events.jsonl`, with no automatic replay or container removal. The default +probe asks for at most 2 CPUs, 4 GiB, no network/GPU/model mount, and a +read-only root. +Bare mode mounts no source. Guarded mode mounts the independently authenticated +retained source and runs the worker closure check, module guard, trainer path +setup, environment bootstrap and Torch import. Neither runs training. `PASS` +only says Torch imports in the reduced probe; it does not diagnose an earlier +failure. Review the pinned package/runtime source before changing a profile, +image or runtime policy. +The exact r2 CPU evidence and its unresolved runtime difference are in +`docs/review/docker-torch-import-r2-diagnosis.md`. + +The guarded-only `--gpu-device 0` option is an import-only comparison that +requires separate exact root review before any live use. It requests only +device 0, verifies Docker retained that device request before start, removes +the CPU probe's visibility suppressors, and records the scope in its claim. +`gpu_used=null` means actual device use was not measured. The same 8-KiB +output budget, 120-second execution deadline, owned cleanup, source +authentication and closed child codes apply. It neither invokes the trainer +nor loads a model; a PASS or failure cannot retrospectively diagnose an +earlier trainer attempt. See the exact proposed command and remaining +environment/user/filesystem gaps in the r2 diagnosis note. + +For the worker-context child, keep supplied environment bytes distinct from +the environment observed after isolated Python startup. Require every +supplied key unchanged; the only reviewed addition is +`LC_CTYPE=C.UTF-8`, recorded as a closed delta with separate digests. +Any other addition or change rejects. Do not reset the observed environment +or reinterpret it as a historical capture of the failed r2 child. See the r10 +repair and exact unexecuted command in the diagnosis note. + +The optional `--worker-context` mode requires independently reviewed +`--expected-workload-sha256`, `--expected-runtime-sha256`, and +`--expected-trainer-sha256` alongside guarded source commitments. It +reconstructs the pinned workload's isolated child environment, launches a +separate `-I` child, loads the authenticated worker closure and guard, and +executes only the authenticated trainer prefix through `import torch`. +The prefix excludes the `UNSLOTH_IMPORT` marker and all later trainer/model +code. It records closed environment, path, file and cwd digests and keeps the +existing claim, output budget, deadline and owned cleanup. Its writable +`/artifacts` is a small private tmpfs with no model weights. Review the exact +workload/source hashes, device scope, host user, caps and filesystem before +each fresh use. A reconstructed child environment is a source-and-plan +deduction; it is not a historical capture of the failed child. + +`--worker-context --worker-entry` selects a separate, still import-only +diagnostic. It reconstructs the pinned runtime's trainer argument vector, +including a one-byte sealed stand-in for the prepared dataset descriptor, +then invokes the authenticated worker's real `main()` and nested +`runpy.run_path` trainer entry. A source-bound line trace raises at the +`UNSLOTH_IMPORT` marker after Torch import, while a narrow import-call and +module-finder barrier prevents Unsloth entry even if tracing is disabled or +that module name was cached. A PASS +requires the worker's closed stop label and real trainer `__main__` module +binding. The worker still performs its own argument, closure and import-guard +checks. It records closed environment digests before and after Torch import, +separate from the supplied and interpreter-startup digests. The diagnostic +starts through `-I -c` to install that boundary, so it +does not reproduce the original direct `-I` script startup. The sealed data +byte, reduced filesystem and absent model weights also differ from the failed +attempt. Review and admit a fresh one-use command before any live execution; +do not infer the historical failure cause from either result. See the r12 +proposal in the diagnosis note. + After an intentional worker source change, check and refresh existing manifests with the checked-in maintainers in dependency order: `regenerate_offline_sft_worker_closure.py`, diff --git a/.skills/fine-tuning/SKILL.md b/.skills/fine-tuning/SKILL.md index 73c91863..1172feb9 100644 --- a/.skills/fine-tuning/SKILL.md +++ b/.skills/fine-tuning/SKILL.md @@ -57,6 +57,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Check SFT loss masking (tokenizer only) | `python tuner.py doctor sft-mask --dataset-path --model ` | | Inspect native training host candidate | `python tuner.py doctor native-training --json` — read-only platform/Python/package metadata; see `reference/native-local-training.md` | | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | +| Diagnose Torch import in a retained Docker image | `python scripts/diagnose_docker_torch_import.py --help` for bounded guarded, worker-context prefix, and worker-entry probes; each live command needs exact review; see `reference/sft-training.md` | | Native MLX prepared SFT candidate | Internal exact-token batch/loss and runtime comparison only; no native launch CLI or API provider yet; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.skills/fine-tuning/reference/sft-training.md b/.skills/fine-tuning/reference/sft-training.md index 3029c594..976e3a92 100644 --- a/.skills/fine-tuning/reference/sft-training.md +++ b/.skills/fine-tuning/reference/sft-training.md @@ -450,10 +450,16 @@ See the checked-in example README for commands and exact fixture coverage. The checkout-owned `examples/training_qualification/prepare_docker.py` closes material setup for the public Docker consumer. With an installed engine, -`plan` validates declared release/model/layout, `prepare` authenticates genuine -sources and existing immutable v1 input before publishing canonical material, -and `prepare-model` explicitly downloads the pinned public snapshot into the -private artifact-mount cache. See `examples/training_qualification/README.md` +`plan` validates declared release/model/layout, `prepare-model` first prepares +the pinned public snapshot into a fresh private artifact cache, and `prepare` +then authenticates fresh source and existing immutable v1 input before publishing +canonical material. A private read-only seed cache may reduce downloads only +after pinned upstream metadata and every reused Git/LFS member are verified. +The old seed claim is never adopted. `prepare` binds the complete model receipt; +public start rechecks its bytes before Docker effects. A missing pinned revision +in an otherwise private seed takes the normal cold path; redirects reject. +Cold public reads use the receipt retained with the attempt and do not require +current model weights or the original preparation root. See `examples/training_qualification/README.md` for the template and exact commands. Full public preflight remains deferred until host composition; missing image/runtime/model qualification still refuses. The CPU `probe-filesystem` command measures actual native writer modes and host @@ -492,6 +498,89 @@ the consumed claim, roots, container and retained logs; the diagnostics never authorize replay. See `docs/review/offline-sft-worker-diagnostics.md` for the October 20 evidence boundary, CPU regressions and fresh-run prerequisites. +For a fresh, credential-free Docker Torch import reproduction, review the exact +command before effects. The checked-in `scripts/diagnose_docker_torch_import.py` +requires `--docker`, an empty `--docker-config`, `--release`, independently +verified `--expected-release-sha256`, `--daemon-id`, and a nonexistent private +`--output-dir`. Guarded mode additionally requires `--source-root`, +`--source-lock`, independently reviewed `--expected-source-lock-sha256`, +`--expected-manifest-sha256`, and `--expected-closure-digest` from the retained +attempt. It validates every staged worker member before executing mounted code; +the result records those commitments. Run from the reviewed checkout so its +trusted closure verifier is the one checked in with the helper. + +The helper claims a unique name before Docker create, authenticates the exact +new container before start, and retains it. Docker CLI output is capped at 8 KiB +in memory; the container requests JSON log rotation at 1 MiB with one file. +The import execution has a +120-second host deadline. At expiry the helper issues at most one stop request +to the authenticated new container and separately inspects terminal state; +cleanup takes additional bounded time, and a stop acknowledgment alone is not +quiescence. An uncertain create/start/stop remains consumed in `claim.json` and +`events.jsonl`, with no automatic replay or container removal. The default +probe asks for at most 2 CPUs, 4 GiB, no network/GPU/model mount, and a +read-only root. +Bare mode mounts no source. Guarded mode mounts the independently authenticated +retained source and runs the worker closure check, module guard, trainer path +setup, environment bootstrap and Torch import. Neither runs training. `PASS` +only says Torch imports in the reduced probe; it does not diagnose an earlier +failure. Review the pinned package/runtime source before changing a profile, +image or runtime policy. +The exact r2 CPU evidence and its unresolved runtime difference are in +`docs/review/docker-torch-import-r2-diagnosis.md`. + +The guarded-only `--gpu-device 0` option is an import-only comparison that +requires separate exact root review before any live use. It requests only +device 0, verifies Docker retained that device request before start, removes +the CPU probe's visibility suppressors, and records the scope in its claim. +`gpu_used=null` means actual device use was not measured. The same 8-KiB +output budget, 120-second execution deadline, owned cleanup, source +authentication and closed child codes apply. It neither invokes the trainer +nor loads a model; a PASS or failure cannot retrospectively diagnose an +earlier trainer attempt. See the exact proposed command and remaining +environment/user/filesystem gaps in the r2 diagnosis note. + +For the worker-context child, keep supplied environment bytes distinct from +the environment observed after isolated Python startup. Require every +supplied key unchanged; the only reviewed addition is +`LC_CTYPE=C.UTF-8`, recorded as a closed delta with separate digests. +Any other addition or change rejects. Do not reset the observed environment +or reinterpret it as a historical capture of the failed r2 child. See the r10 +repair and exact unexecuted command in the diagnosis note. + +The optional `--worker-context` mode requires independently reviewed +`--expected-workload-sha256`, `--expected-runtime-sha256`, and +`--expected-trainer-sha256` alongside guarded source commitments. It +reconstructs the pinned workload's isolated child environment, launches a +separate `-I` child, loads the authenticated worker closure and guard, and +executes only the authenticated trainer prefix through `import torch`. +The prefix excludes the `UNSLOTH_IMPORT` marker and all later trainer/model +code. It records closed environment, path, file and cwd digests and keeps the +existing claim, output budget, deadline and owned cleanup. Its writable +`/artifacts` is a small private tmpfs with no model weights. Review the exact +workload/source hashes, device scope, host user, caps and filesystem before +each fresh use. A reconstructed child environment is a source-and-plan +deduction; it is not a historical capture of the failed child. + +`--worker-context --worker-entry` selects a separate, still import-only +diagnostic. It reconstructs the pinned runtime's trainer argument vector, +including a one-byte sealed stand-in for the prepared dataset descriptor, +then invokes the authenticated worker's real `main()` and nested +`runpy.run_path` trainer entry. A source-bound line trace raises at the +`UNSLOTH_IMPORT` marker after Torch import, while a narrow import-call and +module-finder barrier prevents Unsloth entry even if tracing is disabled or +that module name was cached. A PASS +requires the worker's closed stop label and real trainer `__main__` module +binding. The worker still performs its own argument, closure and import-guard +checks. It records closed environment digests before and after Torch import, +separate from the supplied and interpreter-startup digests. The diagnostic +starts through `-I -c` to install that boundary, so it +does not reproduce the original direct `-I` script startup. The sealed data +byte, reduced filesystem and absent model weights also differ from the failed +attempt. Review and admit a fresh one-use command before any live execution; +do not infer the historical failure cause from either result. See the r12 +proposal in the diagnosis note. + After an intentional worker source change, check and refresh existing manifests with the checked-in maintainers in dependency order: `regenerate_offline_sft_worker_closure.py`, diff --git a/docs/review/docker-torch-import-r2-diagnosis.md b/docs/review/docker-torch-import-r2-diagnosis.md new file mode 100644 index 00000000..d2b5c5b1 --- /dev/null +++ b/docs/review/docker-torch-import-r2-diagnosis.md @@ -0,0 +1,681 @@ +# Docker r2 Torch import boundary — CPU diagnosis, 2026-10-20 + +## Retained failure + +The one authorized r2 public start on engine `0d515ae1ae235d0858802228ce4af8dba8fccd03` +and host `85ea6b56af89f6dcb7c0a2c7644e95c511762486` was accepted at +13:47:50Z and failed at 13:47:52Z. Container +`5256d2446d3751e423a6ff535c3ed9a13a2bc5df92765a61db5da700bf7e601d` +remains retained with exit 23. Its 81-byte trainer stderr, SHA-256 +`95d34d74eae9cb55a223eeda8428ee4387ee79095e5c0f39a0ceb60da9f45513`, +contains only `OFFLINE_SFT_WORKER_REJECTED phase=TRAINER category=IMPORT +milestone=TORCH_IMPORT`. This locates the first Torch import boundary in +`Trainers/sft/train_sft.py`; it does not identify an import dependency or a +remedy. The earlier bare marker on container +`07ecde6d5b798abf51eb85595be165039cdc1043295cbd0a5b0d3d526064bfd7` remains historical +and uninterpreted. + +The release file SHA-256 is +`5f65824ceb5e250a58314da0233b9c07f1d63f1b0a39892d9b36af694938bfe6`; +its immutable image ID is +`sha256:e737a0a091474c4892690690e712bc1f932f108856a3d5df60383e1cd9436e1e`. +The retained final runtime capture records CPython 3.12.3 and Torch +`2.11.0+cu128`. Source, image, release and installed wheel were not changed. + +## Targeted CPU experiments + +The r4 version of `scripts/diagnose_docker_torch_import.py` bound that release +file hash, daemon ID and exact local image ID. It requested at most 2 CPUs, +4 GiB, no network or GPU devices, and had no credentials, model mount or +training entrypoint. Its `subprocess.run(timeout=120)` bounded the Docker CLI +call but **did not enforce a 120-second container lifetime**. Its +`capture_output=True` also buffered CLI output without a preallocation limit. +The earlier 120-second container claim was incorrect. The retained r4 results +are observations, not proof of a bounded lifecycle. + +The bare image probe returned `PASS` in container +`33af62c4fdda5d855b3b154c2937912b5266a88a48f2c0757b3664fe42a45917`. +The final guarded probe mounted the r2 staged source read-only, validated a +manifest against the digest in that same mount, installed its owned-module guard, ran +`init_trainer_env()`, and imported Torch successfully. Its retained container +is `2e5b6d06a4cc946541a475da72b7aa7e851b5840f86fd675c2099540b6fc1045`; +`evidence-guarded-final2/result.json` has SHA-256 +`326b59ebf3896a737732661895ef67b3efcf2527d65ce912b02ff20d9437ee33`. +The exact final checked-in helper source has SHA-256 +`c214fc8b9d617fe7171e7eac987d93187169b8a637fac9ef24b4323a70bf6c53`. +These artifacts are under the private Windows WSL root +`/home/profsynapse/.local/share/synaptic-qualification/torch-import-r4-20261020/`. +The r4 helper had no independently admitted source or closure commitment, so +this `PASS` cannot establish that it executed the exact r2 closure. The result +remains historical and is not relabeled after the helper repair. + +Earlier guarded diagnostic containers returned `READ_MANIFEST/OS` because +the probe dropped all capabilities while running as root against mode-0600 +staged files owned by the WSL user. Running the diagnostic as that owner +resolved this probe setup failure. Those containers and reports are retained; +their failures do not reinterpret r2. + +## Interpretation and next boundary + +Torch is importable in the exact image both bare and after the retained worker +closure, guard and trainer bootstrap under the finite CPU diagnostic. The r2 +container requested GPU 0 and used root, default capabilities and a writable +root filesystem; the CPU probes deliberately did not reproduce that full +runtime. The cause of r2's Torch import failure remains unresolved. In +particular, the evidence does not support changing packages, image, recipe, +model, GPU settings or SFT runtime code yet. + +Read-only inspection of the retained r2 container on 2026-10-20 confirmed +entrypoint `/opt/unsloth-venv/bin/python3`, command prefix +`/source/engine/Trainers/sft/runtime_v1.py --canonical-workload-file +/source/control/workload.json`, user `root`, workdir `/workspace`, runtime +`runc`, writable root, default capabilities, network `none`, and Docker GPU +device request `0`. The parent runtime creates a separate offline worker with +an explicit child environment (`Trainers/sft/runtime_v1.py:1480-1497`) and +executes the trainer through `runpy.run_path`; the trainer prepends its `src` +and engine roots to `sys.path` immediately before `init_trainer_env()` and +`import torch` (`Trainers/sft/train_sft.py:217-227`). The r4 guarded probe +cleared/replaced its environment, disabled CUDA/NVIDIA visibility, dropped +capabilities, used the staged-source owner, and skipped the parent runtime. +Its path setup also lacked the trainer's exact `src` insertion. The repaired +guarded probe mirrors that path order, but it still does not reproduce the +parent's environment, user, capabilities, writable root or GPU request. + +The r2 claim, container and all older failure evidence remain consumed and +preserved. No training start, image build, pull, model load, package install or +HF action occurred during this diagnosis. A further comparison of the +original GPU/runtime environment needs a separately reviewed exact scope; +this diagnostic does not authorize another training attempt. No runtime +closure or Modal lock refresh is needed for the diagnostic script and skill +changes, because no locked runtime member changed. + +## Repaired helper and next review boundary + +The successor helper claims a fresh unique diagnostic name before create, +caps aggregate Docker CLI output at 8 KiB, requests JSON log rotation at +1 MiB with one file, and +separates create, authenticated start, terminal inspection and one owned stop +request. Its 120 seconds are an execution deadline followed by a bounded +cleanup and inspection tail; a Docker stop response alone never proves physical +quiescence. If create/start/stop or identity is uncertain, the claim and events +remain and no automatic replay occurs. For guarded mode, independently supplied +r2 source-lock SHA-256 `d19aecb2d3ca1b7b6fa32a2445c45251e59d8cac2332293c68b73a61cbff4665`, +manifest SHA-256 `3842689403abc5b42e28d58545b88616b81cd8fc738ee5557e97159df65d00ef`, +and closure digest `8465ff2b7cedb352337e7cedc5bb17cc169892c2eedae0b280569f4a0cde5002` +are checked before mounted code executes and included in the result. These +values were read from the retained r2 material on 2026-10-20; live rechecking +before any new effect remains required. The reviewed successor CPU probe ran +once as r6 below. A GPU-exposed probe still needs separate scope review. + +The cleanup path attempts its single authenticated stop even if evidence +appends fail, then inspects terminal state independently even if the stop CLI +acknowledgement fails. Failed evidence writes retain an uncertain result; an +in-memory observation does not establish durable proof. Source verification +and a read-only container mount do not freeze files on the host. Before this +probe, verify private ownership/permissions of the retained source and its +ancestors, and ensure that no preparation or other writer is using that source. +This probe assumes that private host custody remains in force during execution. + +The reviewed r6 guarded CPU command is retained here as the exact executed +invocation. It ran from a clean WSL checkout of +`c48fa8299333d6057bc876899d484386f2059ddc` after confirming that its +helper and trusted closure verifier matched the reviewed source. Its parent +`/home/profsynapse/.local/share/synaptic-qualification/torch-import-r5-20261020` +was private mode 0700; the `guarded-cpu-1` child was absent before the run. + +```sh +python3 -I scripts/diagnose_docker_torch_import.py \ + --docker /usr/bin/docker \ + --docker-config /home/profsynapse/.local/share/synaptic-qualification/docker-sft-stable-image-c/docker-config \ + --release /home/profsynapse/.local/share/synaptic-qualification/docker-sft-stable-image-c/evidence/runtime-release.json \ + --expected-release-sha256 5f65824ceb5e250a58314da0233b9c07f1d63f1b0a39892d9b36af694938bfe6 \ + --daemon-id 24805003-b125-469b-8080-9da4daacd364 \ + --source-root /home/profsynapse/.local/share/synaptic-qualification/docker-public-plan-0d515-20261020/attempt-r2/source/engine \ + --source-lock /home/profsynapse/.local/share/synaptic-qualification/docker-public-plan-0d515-20261020/attempt-r2/material/source-lock.json \ + --expected-source-lock-sha256 d19aecb2d3ca1b7b6fa32a2445c45251e59d8cac2332293c68b73a61cbff4665 \ + --expected-manifest-sha256 3842689403abc5b42e28d58545b88616b81cd8fc738ee5557e97159df65d00ef \ + --expected-closure-digest 8465ff2b7cedb352337e7cedc5bb17cc169892c2eedae0b280569f4a0cde5002 \ + --output-dir /home/profsynapse/.local/share/synaptic-qualification/torch-import-r5-20261020/guarded-cpu-1 +``` + +This repeated the reduced CPU import boundary with repaired ownership and +source binding. It was **not** an exact r2 invocation, did not expose a GPU, +and did not authorize another SFT start. The original r2 claim remains consumed. + +The next comparison should hold the independently authenticated r2 source, +daemon and image fixed, then compare the offline child environment produced by +`runtime_v1.py` with the CPU diagnostic at the Torch import boundary. In +particular compare Python executable/`-I` invocation, worker closure and +module guard, final `sys.path`, `PATH`, `LD_LIBRARY_PATH`, `HF_HOME`, +`TRANSFORMERS_CACHE`, `TMPDIR`, and CUDA/NVIDIA visibility by key and approved +value or digest without printing secrets. Vary one of user/capabilities, +writable root and GPU visibility at a time in import-only containers under +reviewed bounds. A GPU-visible import-only experiment is a separate reviewed +scope; no trainer, model load or training is proposed. Treat any changed result +as evidence for a narrower hypothesis, not retrospective proof of r2's cause. + +## r6 result and read-only r2 context comparison + +The one authorized r6 run used the unchanged command above after read-only +preflight and a saved/read-back Nexus admission. The clean WSL checkout was +`c48fa8299333d6057bc876899d484386f2059ddc`; helper SHA-256 was +`290bf1aa8e10ce3cf8a7eb0f1ff8e5a394a7ac726495330eaf0ff24fe9a33ac1` +and trusted closure-verifier SHA-256 was +`2304eeb00667954510c34cc4aacaa818532a301fbeabf04c2fb165d6cd1d70b5`. +The retained r2 manifest, source lock and release matched the independently +approved hashes in the command. No matching preparer or worker process was +observed before start; the source and parent were privately owned. This +custody check does not freeze host files against a later writer. + +The r6 child returned `PASS` at `TORCH_IMPORT`, with `training_executed=false` +and `gpu_used=false`. Container +`8c578b6ec0930e9c7668f237672b9d11f4122524befe601d077504966ff203e0` +is retained and independently inspected as exited, code 0. Its claim label is +`f07f5348f19fc6bd7210929c78a1b7c2b900361da249ac5755ce6eabb8b2330d`; +the daemon and image IDs match r2. It started at +`2026-10-20T14:51:44.979964328Z` and finished at +`2026-10-20T14:51:48.957092702Z`. The claim, events and result under the +private `torch-import-r5-20261020/guarded-cpu-1` directory have SHA-256 +`a62c0bb6a610c2a27a7951fe7e7c95038720b9ed882a6c6044fe1aeffaf6c8dc`, +`0bab6899f60dd05b885e65fa9a4c90ee8f65d5c751357bf4fad8752943a2446d`, +and `c22892913e1c0b3aaff1bfc99b876c80a653000ae66c8f29d3e83023c00a1308`, +respectively. Events show one claimed create, one start and terminal exit 0; +no stop or deletion was needed. The r2/r4 claims and containers remain preserved. + +The retained r2 workload SHA-256 is +`1534308c977c2c1cb2b1d80d9cc5c07e31303db63b336686e1c5ccc2644b1064`; +its Docker runtime declaration SHA-256 is +`947809449ffce1fde2cdb6a928c2cec60444cc3d841d173405080b7f545e4aae`. +The staged r2 `runtime_v1.py`, `train_sft.py` and `offline_sft_worker.py` bytes +match engine commit `0d515ae1ae235d0858802228ce4af8dba8fccd03`. +The pinned release records CPython 3.12.3 and manifest digest +`c5d6c01fb1dd62276b033234982cd14846865500dd07eaed2a2102dea97a1535`. +Read-only inspection of both retained Docker container configs and these +sources gives the comparison below. + +| Boundary | Retained r2 | Guarded r6 CPU | +| --- | --- | --- | +| Image, daemon, source | Pinned image and daemon; staged engine closure `8465ff2b...` | Same image and daemon; independently authenticated same staged closure | +| Host container | Root, `/workspace`, default caps, writable root, GPU device request `0`, 16 GiB | UID/GID 1000, `/tmp`, all caps dropped, read-only root, no device request, 4 GiB | +| Common host settings | 2 CPUs, network `none`, `runc` | 2 CPUs, network `none`, `runc`; 120-second execution deadline plus bounded cleanup tail | +| Python entry | Image Python starts parent `runtime_v1.py`; parent starts separate `-I offline_sft_worker.py -- ...` child | Same image Python runs `-I -c` fixed diagnostic; no parent or worker entrypoint executes | +| Closure and import order | Worker loads closure, verifies origins, installs guard, adds trainer directory and engine root, then `runpy.run_path` enters `train_sft.py`; trainer adds engine and `src`, bootstraps, imports Torch | Probe loads same closure, verifies origins, installs guard, arranges the same leading engine/`src`/trainer/engine path order, bootstraps and imports Torch directly; later path tail and loaded-module history differ | +| Offline child environment | Source builds `child_env` from the planned environment with `clear_inherited=true`, removes `PYTHONPATH`, adds closure/root bindings and overrides offline/HF/cache flags; actual process environment was not separately captured | Probe clears inherited environment and installs fixed import-only values. It lacks r2's closure/root environment bindings and HF/cache paths; expected closure digest is embedded in fixed probe code | +| Visibility inside child | Planned environment has no `CUDA_VISIBLE_DEVICES`, `NVIDIA_VISIBLE_DEVICES` or `LD_LIBRARY_PATH`; parent Docker config has NVIDIA/LD entries, but `env=` prevents child inheritance | Probe sets CUDA visibility empty and NVIDIA visibility `void`; it also omits `LD_LIBRARY_PATH` | + +The r2 planned environment's sorted compact `jq -cS` serialization +(including its trailing LF) has SHA-256 +`e621be06bb27b0c258e8ef8ecfd35eccaa362236f9a194eb122bfe4445a798ed`. +Its `PATH` value SHA-256 is +`ff201961161ffc598d11b7e168870c3fdbe7485141fa414c3f67f2ce89244338`; +the r6 fixed probe `PATH` value SHA-256 is +`05f89650ff263293ba277e3c910f4343e0f9b1a37c6d27e0aab9cb3b772e0896`. +The r2 plan's `PYTHONPATH` equals the engine root but is explicitly removed +from `child_env`. Its declared `HF_HOME` and `TRANSFORMERS_CACHE` equal the +cache root plus their respective subdirectories, matching the child builder's +overrides. These are source-and-plan deductions, not a captured r2 child +environment. No environment value, raw argv or exception text is published. + +The strongest next single-factor import-only comparison is GPU visibility in +the otherwise unchanged guarded diagnostic. This holds the authenticated +image/source, user, capabilities, read-only root, fixed probe and deadline +constant while changing the device request and removing the two explicit +visibility suppressors. Docker documents `--gpus device=0` for exposing only +GPU index 0 in its [GPU access guide](https://docs.docker.com/engine/containers/gpu/). +A changed result would implicate GPU exposure under +these reduced conditions; either result would leave r2's exact parent/child +environment, user/caps and filesystem differences unresolved. The existing +helper now has a fixed guarded-only `--gpu-device 0` option. It requests only +device 0, checks the resulting Docker device request before start, includes +this scope in its claim, and reports `gpu_used=null` because device use is not +measured. It retains the 8-KiB CLI output budget, 120-second execution +deadline, single owned stop and independent terminal inspection. The fixed +child still only installs the source guard, bootstraps and imports Torch; it +does not invoke the trainer, load a model or run training. + +Root separately reviewed and authorized the following command for one r8 +import-only run. It required a clean checkout of `98e4a49ee175102f645e8c497acaff90fd5c4248`, +fresh read-only daemon/image/source/custody checks, a new private mode-0700 +`/home/profsynapse/.local/share/synaptic-qualification/torch-import-r7-20261020` +parent, an absent `guarded-gpu0-1` child, and a saved/read-back one-use +admission. This exact command ran once and is consumed: + +```sh +python3 -I scripts/diagnose_docker_torch_import.py \ + --docker /usr/bin/docker \ + --docker-config /home/profsynapse/.local/share/synaptic-qualification/docker-sft-stable-image-c/docker-config \ + --release /home/profsynapse/.local/share/synaptic-qualification/docker-sft-stable-image-c/evidence/runtime-release.json \ + --expected-release-sha256 5f65824ceb5e250a58314da0233b9c07f1d63f1b0a39892d9b36af694938bfe6 \ + --daemon-id 24805003-b125-469b-8080-9da4daacd364 \ + --source-root /home/profsynapse/.local/share/synaptic-qualification/docker-public-plan-0d515-20261020/attempt-r2/source/engine \ + --source-lock /home/profsynapse/.local/share/synaptic-qualification/docker-public-plan-0d515-20261020/attempt-r2/material/source-lock.json \ + --expected-source-lock-sha256 d19aecb2d3ca1b7b6fa32a2445c45251e59d8cac2332293c68b73a61cbff4665 \ + --expected-manifest-sha256 3842689403abc5b42e28d58545b88616b81cd8fc738ee5557e97159df65d00ef \ + --expected-closure-digest 8465ff2b7cedb352337e7cedc5bb17cc169892c2eedae0b280569f4a0cde5002 \ + --gpu-device 0 \ + --output-dir /home/profsynapse/.local/share/synaptic-qualification/torch-import-r7-20261020/guarded-gpu0-1 +``` + +## r8 single GPU-visible import result + +Before the single r8 start, a fresh clean WSL checkout and the reviewed helper +SHA-256 `95849f03b1133302cef1bc7ceec1e20674897877152c9b35b64af2b86cf718c3` +were authenticated. The trusted closure verifier SHA-256 was +`2304eeb00667954510c34cc4aacaa818532a301fbeabf04c2fb165d6cd1d70b5`. +Release, image, daemon, retained source lock, manifest and closure identities +matched those above. The source and ancestors passed private-custody checks, +no matching writer was observed, and the private mode-0700 output parent had +no `guarded-gpu0-1` child. Windows and WSL GPU queries both identified device +0 as `GPU-ac4398bb-fc15-d722-b39c-2790b4b5f9cc` (RTX 3090) and returned +no active compute-process row. The one-use Nexus admission was saved and read +back before Docker create. + +The closed r8 result is `PASS` at `TORCH_IMPORT`, `gpu_device="0"`, +`gpu_used=null` (device use not measured), and `training_executed=false`. +Container `d41a2c26ee089777b7583e6100e67c5a899eedc7ff2c72b9beddae22cfe669c2` +is retained. Independent Docker inspection found the exact claim label +`e076d416d5d8ea04d97ae245d989db7d1d41d4ef74efe036c3aec723dff57dd2`, +image ID, one device request with `DeviceIDs=["0"]` and `Capabilities=[["gpu"]]`, +2 CPUs, 4 GiB, network `none`, read-only root, UID/GID 1000, all capabilities +dropped, and terminal `exited` with code 0. It ran from +`2026-10-20T15:11:08.249437944Z` to `2026-10-20T15:11:11.665541884Z`. +Events record one claimed create, one start and terminal exit 0; no stop or +deletion occurred. The retained `claim.json`, `events.jsonl` and `result.json` +under `torch-import-r7-20261020/guarded-gpu0-1` have SHA-256 +`a7a85de97044ed8b2381f20be1c5e0846fc9ddb1b0217bde6b1da7b3d95ae89c`, +`05b15a0b21e2dc653572f5f4f51f12e5b98371a65fe216a5b76969efecdc158d`, +and `439d5a19c1b838840ee8cd49085a95b68acfa50e8182e3dceca43b52722794f1`, +respectively. The r2/r4/r6 claims and containers remain unchanged. + +Both guarded r6 CPU and r8 device-0 imports passed in the reduced diagnostic. +Adding device-0 exposure and removing the two visibility suppressors did not +reproduce r2's closed import rejection under those conditions. It does not +recreate the r2 parent/isolated-child environment, Python entry, root/default +capabilities, writable filesystem or 16-GiB limit, and cannot establish or +rule out an interaction with them. The cause of r2 remains unproven. This +consumed diagnostic grants no replay, broader GPU comparison or training +authority. + +## r9 prepared worker import boundary — no live execution + +The r8 result leaves the original isolated child invocation and trainer +preamble untested. The next proposed **single** import-only comparison uses +the retained r2 workload, source lock, closure and image. This is preparation +for root review; no new container, device request or probe was started in r9. +It supersedes the earlier idea of varying incidental host settings one at a +time. + +The retained r2 workload is SHA-256 +`1534308c977c2c1cb2b1d80d9cc5c07e31303db63b336686e1c5ccc2644b1064`. +Its pinned `runtime_v1.py` and `train_sft.py` are SHA-256 +`460121aaee3eb83764f67d9307cfb57a3b25c117611d987577b9631b6956a1b3` +and `75c2ab22fbc5dd80b8ddd5e6d0da816d1451cda0cd620c34717336a39a8a7644`. +The helper verifies all three against independently supplied expected hashes +before Docker create and again before start, in addition to the existing +release, daemon, image, source lock, manifest and closure checks. The workload +file is included in source custody checks. The source mount remains read-only; +private host custody and no concurrent writer remain necessary during a run. + +`--worker-context` derives the child environment from the pinned workload +using the exact `runtime_v1.py:1479-1497` transform: start from its declared +variables, remove `PYTHONPATH`, bind the closure and runtime roots, then +override the workload fingerprint, isolated Python flags, HF/cache/offline +values and model-snapshot **path**. It starts a separate `-I` interpreter +with that replacement environment and cwd `/artifacts/run/tmp`. The +model-snapshot path is a string only; no model weights are mounted or loaded. +The child evaluates the authenticated worker module without its `__main__` +entry, loads the closure, verifies origins, installs the owned-module guard +and inserts the worker's trainer-directory/engine path pair. It then compiles +the authenticated `train_sft.py` prefix ending with `import torch`, with +the real trainer `__file__`, `__main__` name and phase callback. The +`UNSLOTH_IMPORT` marker and every later statement are excluded from the +compiled prefix. Marker order, exact Torch/Unsloth adjacency, source hashes +and the last AST statement fail closed on drift. The child returns only +closed category/stage and SHA-256 digests of its preamble environment, cwd, +trainer file path and `sys.path`; the helper checks the environment digest +against its own reconstruction. No raw environment, argv, path, exception or +traceback enters the report. + +The proposed host stays within 2 CPUs, 4 GiB and a 120-second execution +deadline plus bounded owned cleanup. It keeps r8's UID/GID of the staged +source owner, dropped capabilities, read-only root, network `none`, and +device-0 request; it adds only a 64-MiB `/artifacts` tmpfs for private +runtime roots and no model mount. The separately launched child's cwd +matches r2; the outer container still uses `/tmp`. This is closer to r2 +at the child import boundary, but root/default capabilities, 16-GiB memory, +writable root, actual model cache, and the full parent admission/preparation +remain different. The original child environment was not captured, so this +is a source-and-plan reconstruction rather than a historical byte capture. +Even a changed import outcome would narrow a hypothesis, not prove r2's +cause or authorize training. + +The exact **unexecuted** proposal for root review is: + +```sh +python3 -I scripts/diagnose_docker_torch_import.py \ + --docker /usr/bin/docker \ + --docker-config /home/profsynapse/.local/share/synaptic-qualification/docker-sft-stable-image-c/docker-config \ + --release /home/profsynapse/.local/share/synaptic-qualification/docker-sft-stable-image-c/evidence/runtime-release.json \ + --expected-release-sha256 5f65824ceb5e250a58314da0233b9c07f1d63f1b0a39892d9b36af694938bfe6 \ + --daemon-id 24805003-b125-469b-8080-9da4daacd364 \ + --source-root /home/profsynapse/.local/share/synaptic-qualification/docker-public-plan-0d515-20261020/attempt-r2/source/engine \ + --source-lock /home/profsynapse/.local/share/synaptic-qualification/docker-public-plan-0d515-20261020/attempt-r2/material/source-lock.json \ + --expected-source-lock-sha256 d19aecb2d3ca1b7b6fa32a2445c45251e59d8cac2332293c68b73a61cbff4665 \ + --expected-manifest-sha256 3842689403abc5b42e28d58545b88616b81cd8fc738ee5557e97159df65d00ef \ + --expected-closure-digest 8465ff2b7cedb352337e7cedc5bb17cc169892c2eedae0b280569f4a0cde5002 \ + --worker-context \ + --expected-workload-sha256 1534308c977c2c1cb2b1d80d9cc5c07e31303db63b336686e1c5ccc2644b1064 \ + --expected-runtime-sha256 460121aaee3eb83764f67d9307cfb57a3b25c117611d987577b9631b6956a1b3 \ + --expected-trainer-sha256 75c2ab22fbc5dd80b8ddd5e6d0da816d1451cda0cd620c34717336a39a8a7644 \ + --gpu-device 0 \ + --output-dir /home/profsynapse/.local/share/synaptic-qualification/torch-import-r9-20261020/worker-context-gpu0-1 +``` + +The output parent must first be created privately at mode 0700 and the named +child must be absent. Root review must confirm a clean pushed helper/verifier, +the exact remote identities and source custody, no active device-0 compute +job, the proposed UID/caps/tmpfs/readonly filesystem and bounds, then save +and read back a fresh one-use admission before any start. A collision or drift +rejects without renaming or retrying; r2/r4/r6/r8 claims remain consumed. + +## r10 startup-environment evidence repair — no live execution + +Independent review found that the r9 helper compared a digest of the +environment supplied to the isolated child with a digest observed *after* +CPython startup. The retained r2 child plan supplies no locale keys. CPython +may add `LC_CTYPE=C.UTF-8` when coercing the default C locale; its +[3.12 command-line documentation](https://docs.python.org/3.12/using/cmdline.html#envvar-PYTHONCOERCECLOCALE) +describes the insertion, and the +[v3.12.3 preconfiguration source](https://github.com/python/cpython/blob/v3.12.3/Python/preconfig.c) +contains that coercion path. A real local isolated CPython 3.12.15 subprocess +with the fixed no-locale input added exactly this key/value. The pinned image's +CPython 3.12.3 behavior has **not** been observed by executing this proposal. + +The repaired helper passes the exact supplied mapping to the child through a +bounded private stdin pipe as well as `env=`. Before mounted worker code, the +child checks the supplied digest against its independently bound expectation. +At the existing preamble observation point it requires every supplied key +and value unchanged. It accepts either no added keys or exactly +`LC_CTYPE=C.UTF-8`; any other addition or changed supplied value rejects. +It reports separate `environment_supplied_sha256` and +`environment_observed_sha256` plus the closed delta `NONE` or +`LC_CTYPE_C_UTF8`. The host verifies both digests against its own +reconstruction, including the exact allowed locale variant. It never +prints environment values, resets them, or claims to have captured r2's +historical child. The owned lifecycle, 120-second execution deadline, +8-KiB Docker CLI output limit, source commitments and stop-before-Unsloth +boundary are unchanged. + +The proposed command in the r9 section above is unchanged and remains +**unexecuted**. For root delta review, the repaired helper SHA-256 is +`c7543b50c4aa41b91ed8293474b4f1719c69342550515a8d367b06dd3f1fd229`; +the trusted closure verifier remains +`2304eeb00667954510c34cc4aacaa818532a301fbeabf04c2fb165d6cd1d70b5`. +Root must review the pushed commit and exact command, repeat the current +identity/custody/device preflight, and save/read a fresh one-use admission +before any container start. No r10 container, GPU request, ML import or +training effect occurred. + +## r11 single worker-context import result + +Root accepted the r10 repair and authorized exactly one run of the unchanged +r9 command above. Before Docker effects, the fresh private mode-0700 remote +parent `torch-import-r9-20261020` had no `worker-context-gpu0-1` child. +Its clean detached checkout matched pushed +`4b56cdcdb6c4d5f35d5e7af2a03e322f39c50622`; helper SHA-256 was +`c7543b50c4aa41b91ed8293474b4f1719c69342550515a8d367b06dd3f1fd229` +and trusted closure verifier SHA-256 was +`2304eeb00667954510c34cc4aacaa818532a301fbeabf04c2fb165d6cd1d70b5`. +The exact release, source lock, manifest, closure, workload, runtime and trainer +hashes in the command matched read-only remote validation. The explicit +Docker daemon and image IDs matched. Source files and ancestors had private +owner/root custody, no symlink or group/world write; a process/descriptor +scan found no matching active preparer or open writer. This does not +atomically freeze host files, so private custody remains a scope assumption. +Windows and WSL reported no active compute-process rows for device 0 +(`GPU-ac4398bb-fc15-d722-b39c-2790b4b5f9cc`, RTX 3090). The exact +command and preflight were saved and read back in the Nexus one-use state +`Docker Torch r11 worker-context one-use admission — 2026-10-20` before +start. An immediate checkout/source/output/GPU recheck passed. + +The single helper invocation returned closed `PASS` at `TORCH_IMPORT` with +`boundary=BEFORE_UNSLOTH`, `scope=worker_context`, +`training_executed=false` and `gpu_used=null` (actual device use was not +measured). The supplied child environment SHA-256 was +`1bb0e06d9708dd6fd637f59eaf134dcbe8165f43e120a0b94daad6c79a63699c`. +The observed preamble environment SHA-256 was +`a1eeabbf01741b374ca4e410d62119cf8d882b97d33da962257ec7570bdcb08c`, +with the only accepted startup delta `LC_CTYPE_C_UTF8`. The reported +cwd, trainer `__file__` and `sys.path` digests were respectively +`55846bb91af8c8ccb1548b812885d0ca9d413863279b9030c6a6a1b2327d00a6`, +`4863bf5f38bf9e9cf9b2c8d44473326e77b6aa2c55d9fe039fd20092bb68ad9a`, +and `fbb8a8e665063c6fa0c114cb5edc272b4ecdbbab8c521dac9653a2803c1c0a0d`. +No raw environment, argv or exception text was published. + +Container +`d829fc00a2f9a48437bae1ccfd562d7846b97228b4c526759928fe98c9a618cd` +is retained with exact claim label +`db3afc2a9434fadb65ecaf38d97f26b7e1f7039ea66265554ab5615071209bc4`. +Independent Docker inspection confirmed image/daemon ownership, the single +GPU device-0 request, 2 CPUs, 4 GiB, 64 pids, network `none`, read-only root, +all capabilities dropped, UID/GID 1000, a read-only `/source` mount, private +64-MiB `/artifacts` tmpfs, and no model mount. It started at +`2026-10-20T15:58:15.445432531Z` and finished at +`2026-10-20T15:58:19.681177983Z`, independently observed `exited` with +code 0. Events contain one `CREATE_CLAIMED`, `CREATED`, +`START_CLAIMED`, `STARTED` and `TERMINAL_OBSERVED`; no stop or deletion +was needed. The private `worker-context-gpu0-1` evidence has SHA-256: + +| File | SHA-256 | +| --- | --- | +| `claim.json` | `e65eed000fa69e18d387d18484b0055eff1cf4a9dbb349d645e311a25b5fe99a` | +| `events.jsonl` | `e5a7460c94e94b90f1c3142ed505e34c5bf2c8854447c26ad74b0b840b9bddb1` | +| `result.json` | `1d4e4a9d2e2d9f43a2250b6634c63c270852e0b2babd027d61779f30306e054d` | + +Read-only inspection still found the earlier r2/r4/r6/r8 containers retained +in their previous terminal states. The r11 claim is consumed; no second probe +or training followed. This PASS establishes Torch import through the +authenticated, source-derived child environment and trainer prefix under the +specified reduced host context. It is not a capture of r2's historical child +and leaves root/default capabilities, 16-GiB memory, writable root, actual +model cache and full parent preparation different. The r2 failure cause +remains unproven. + +## r12 worker-entry comparison and proposed import-only probe — unexecuted + +Read-only inspection of the retained r2 workload and containers on +2026-10-20 found a prepared dataset reference; the r2 parent source therefore +passes one sealed prepared-dataset descriptor to its isolated worker. The +retained workload remains SHA-256 +`1534308c977c2c1cb2b1d80d9cc5c07e31303db63b336686e1c5ccc2644b1064`. +Its planned environment is bound by that workload, but no historical child +environment or complete child argv was captured. Source inspection establishes +how `runtime_v1.py` builds them, not their actual r2 process bytes. +Read-only execution of the hash-pinned runtime's argument appender against +the hash-pinned workload produced 34 appended tokens with SHA-256 +`7d2de6641ff52cd26c4515b2c0f65bed94b9eb11ee8ae02b0910596b26fd5937`; +this is a source/plan reconstruction, not a retained child argv digest. + +| Boundary | Retained r2 source/container | Consumed r11 probe | Proposed r12 worker entry | +| --- | --- | --- | --- | +| Isolated launch | Parent calls `/opt/unsloth-venv/bin/python3 -I` on the worker script with trainer arguments | Separate `-I -c` child; evaluates worker under a diagnostic name, skipping its main and argument validation | Separate `-I -c` child enters the authenticated worker with `runpy.run_path(..., run_name="__main__")`; worker runs its real admission and nested trainer `runpy.run_path` | +| Trainer module | Worker sets full trainer argv and `runpy` temporarily binds trainer as `sys.modules['__main__']` | Compiled AST prefix executes in an unattached dictionary with only the trainer filename in argv | Trace checks trainer filename, reconstructed argv and actual `sys.modules['__main__']` namespace, then stops at the authenticated `UNSLOTH_IMPORT` marker | +| Environment | Parent replaces inherited environment after removing `PYTHONPATH`; r2 process bytes were not retained | Source/plan reconstruction, supplied through a private stdin pipe | Same source/plan reconstruction, with separate supplied and observed digests; stdin is `/dev/null` | +| Descriptors and stdio | Prepared input gives one sealed `pass_fds` descriptor; stdin `/dev/null`, stdout/stderr exclusive regular files | No prepared descriptor; stdout pipe and stderr `/dev/null` | One sealed one-byte **synthetic** descriptor and `pass_fds`; stdout/stderr private exclusive files under the 64-MiB artifacts tmpfs | +| Container and data | Root, default capabilities, writable root, 16 GiB; `/source` read-only, `/artifacts` writable and populated with model/cache/prepared state; device 0 | Source-owner UID/GID, caps dropped, read-only root, 4 GiB, private artifacts tmpfs, no model or dataset bytes; device 0 | Keeps those reduced r11 host settings and device 0; no model or historical dataset bytes mounted | + +The new `--worker-context --worker-entry` mode uses the pinned runtime source's +`_append_sft_arguments` and retained workload fields to reconstruct the +trainer flags and values, while substituting a sealed one-byte descriptor for +the historical prepared data. The worker's own argument validator, closure +check, origin verification and import guard execute. The complete authenticated +trainer source runs through `import torch`; a line trace raises at the unique +`UNSLOTH_IMPORT` marker, before `from unsloth ...`. A narrow import-call and +module-finder barrier prevents Unsloth entry if tracing is disabled, even if +the module name was cached. +The worker's closed `phase=TRAINER category=OTHER milestone=TORCH_IMPORT` stop +label, real trainer `__main__` binding and no barrier activation are required +for `PASS`. It records environment digests immediately before Torch import and +after it returns, alongside the separate supplied and interpreter-startup +digests; no environment values are published. Any worker failure reports only +fixed stage/category labels. The +helper retains source, release, image and daemon checks, claim-before-create, +8-KiB Docker CLI output budget, 120-second execution deadline and exactly-once +owned cleanup. It does not call trainer main, load weights or train. + +The wrapper and trace/import barrier are diagnostic additions that can affect +timing or a transitive Unsloth import; r2 used direct +`-I` worker script startup. The synthetic dataset content, reduced host +privileges/filesystem and absent model cache remain differences. A PASS or +failure would test this narrower worker-entry hypothesis under those reduced +conditions and could not retroactively identify r2's cause. No container, +Docker start/exec, device request or ML import occurred while preparing r12. + +The **unexecuted** proposed single command, requiring root review and a new +one-use admission, is: + +```sh +python3 -I scripts/diagnose_docker_torch_import.py \ + --docker /usr/bin/docker \ + --docker-config /home/profsynapse/.local/share/synaptic-qualification/docker-sft-stable-image-c/docker-config \ + --release /home/profsynapse/.local/share/synaptic-qualification/docker-sft-stable-image-c/evidence/runtime-release.json \ + --expected-release-sha256 5f65824ceb5e250a58314da0233b9c07f1d63f1b0a39892d9b36af694938bfe6 \ + --daemon-id 24805003-b125-469b-8080-9da4daacd364 \ + --source-root /home/profsynapse/.local/share/synaptic-qualification/docker-public-plan-0d515-20261020/attempt-r2/source/engine \ + --source-lock /home/profsynapse/.local/share/synaptic-qualification/docker-public-plan-0d515-20261020/attempt-r2/material/source-lock.json \ + --expected-source-lock-sha256 d19aecb2d3ca1b7b6fa32a2445c45251e59d8cac2332293c68b73a61cbff4665 \ + --expected-manifest-sha256 3842689403abc5b42e28d58545b88616b81cd8fc738ee5557e97159df65d00ef \ + --expected-closure-digest 8465ff2b7cedb352337e7cedc5bb17cc169892c2eedae0b280569f4a0cde5002 \ + --worker-context --worker-entry \ + --expected-workload-sha256 1534308c977c2c1cb2b1d80d9cc5c07e31303db63b336686e1c5ccc2644b1064 \ + --expected-runtime-sha256 460121aaee3eb83764f67d9307cfb57a3b25c117611d987577b9631b6956a1b3 \ + --expected-trainer-sha256 75c2ab22fbc5dd80b8ddd5e6d0da816d1451cda0cd620c34717336a39a8a7644 \ + --gpu-device 0 \ + --output-dir /home/profsynapse/.local/share/synaptic-qualification/torch-import-r12-20261020/worker-entry-gpu0-1 +``` + +Before any effect, review the clean pushed checkout and helper SHA-256 +`67224904f0124d6defb38003a9da23406848957ec2a5b2935d5c971781485daa`, +trusted verifier SHA-256 +`2304eeb00667954510c34cc4aacaa818532a301fbeabf04c2fb165d6cd1d70b5`, +all pins above, source/ancestor private custody without an active writer, +unchanged daemon/image, idle GPU 0, and the exact host settings and code delta. +Create only the fresh parent at mode 0700 and require the named child absent; +collision or drift rejects without another name. Save and read back a specific +one-use admission before start. The extra authority sought is exactly one +fresh bounded GPU-0 import-only diagnostic container, including the synthetic +sealed descriptor and real worker/runpy entry. This preparation grants none +of that live authority; all r2/r4/r6/r8/r11 claims and containers remain +consumed and preserved. + +### r13 execution of the admitted worker-entry probe (2026-10-20) + +Root subsequently accepted the exact r12 command at clean, pushed +`14f44a3ef389c9c04225b058626bb3cfbba61d45` for **one** fresh diagnostic +container. The operator read back the root admission, then checked the remote +clean checkout and pushed ref, helper and trusted verifier SHA-256 above, exact +release/daemon/image/source-lock/manifest/closure/workload/runtime/trainer +commitments, private source/ancestor custody, and absence of a visible active +source writer. The fresh parent was mode 0700 and the named output child was +absent. Windows and WSL reported GPU 0 +`GPU-ac4398bb-fc15-d722-b39c-2790b4b5f9cc` with no active compute process. +The one-use current-preflight/command admission was saved and read back in +Nexus as `Windows r13 worker-entry one-use pre-start admission — 2026-10-20` +before Docker effects. These are point-in-time host observations; a read-only +bind does not freeze the retained host tree. + +The command above ran **once**, from the remote clean checkout, with no retry. +The retained output is +`/home/profsynapse/.local/share/synaptic-qualification/torch-import-r12-20261020/worker-entry-gpu0-1`. +It returned closed `PASS` at `TORCH_IMPORT`, boundary `BEFORE_UNSLOTH`, with +`python_isolated=true`, `runpy_main_bound=true`, `training_executed=false`, +and no import barrier activation. The separately supplied environment digest +was `1bb0e06d9708dd6fd637f59eaf134dcbe8165f43e120a0b94daad6c79a63699c`; +the observed startup digest was +`a1eeabbf01741b374ca4e410d62119cf8d882b97d33da962257ec7570bdcb08c`, +with the explicitly admitted `LC_CTYPE_C_UTF8` interpreter addition. The +before- and after-Torch environment digests both were +`712d7eee6b813d815b996f2d3db8266c691dcd5d9f02ea5b9b09073409bd2efc`. +These are reconstructed source/plan context digests, not measurements of r2's +historical child environment. `gpu_used` remains unknown: device exposure is +verified, but this diagnostic does not establish CUDA use. + +The single retained container is +`d9f8c6dda19ce32a4e3326762cc3ecf929451ef011162ba595b1dd9b12a8636f`. +The durable event sequence is exactly `CREATE_CLAIMED`, `CREATED`, +`START_CLAIMED`, `STARTED`, `TERMINAL_OBSERVED`; the independent daemon +inspection found it `exited` with code 0. It started at +`2026-10-20T16:27:48.793457215Z` and finished at +`2026-10-20T16:27:54.515077872Z`. Inspection confirmed the expected daemon, +image, claim label and name; device request `0`; 2 CPU, 4 GiB, network none, +read-only root, all capabilities dropped, read-only `/source` bind, and +64-MiB `/tmp` and `/artifacts` tmpfs. The container and all prior diagnostic +containers and claims remain retained. The claim digest (over canonical claim +bytes without its final newline) is +`9f22951917cf0eb35ce4e2084c64046087ff4777ce7339bde8039ec04cee6e50`. +Retained file SHA-256 values are: + +| File | SHA-256 | +| --- | --- | +| `claim.json` | `86e6202f20064fb12087e532f26debcef6e4047b5d6dcea8bdf3c81ac4648031` | +| `events.jsonl` | `47dd59c98b9da7c15250859f5ee7ccaf3f110070d5a1e73785754e188373953f` | +| `result.json` | `90d84bf65199e6b4aa3d1fd9f88a4bb7d79677902c085d684987ae921bb3f7e5` | + +The wrapper, trace/import barrier, `-I -c` entry, synthetic sealed descriptor, +reduced privilege/filesystem/memory settings, and absent historical model and +dataset bytes remain material differences from r2's direct `-I` worker script +entry. The successful r13 import therefore narrows the observed boundary but +does **not** reproduce r2 or establish the cause of its exit 23. It grants no +additional probe, model, or training authority. + +### r15 public SFT model-first preparation (2026-10-20) + +The retained r14 public-consumer planning root and its r3 configuration are +historical preparation only. No r14 training was submitted. The r2 worker's +exit 23 remains unexplained; the successful reduced import probes do not +establish a runtime fix. The r14 flow exposed an ordering problem: the +short-lived pushed-source proof was minted before a 9.3-GB model staging step. + +The Docker example now accepts a separate, fresh `model_output_root` and an +optional private `model_seed_root`. `prepare-model` consumes its own one-use +claim, copies seed members into a fresh private cache through bounded regular +file reads, and uses the unchanged shared model preparer to verify every reused +member against current pinned Hub Git/LFS metadata. Missing members take the +normal cold download path. Corrupt, redirected or changed seed members reject; +old cache bytes and claims remain untouched. The model receipt records the +declared model/configuration/runtime and a complete bounded destination file +inventory. `prepare` checks that receipt and bytes **before** source +authentication, then compiles a fresh public consumer that binds the receipt. +The existing public `start` composition rechecks the receipt and actual bytes +before Docker effects. This adds no image, package, trainer or runtime change; +the Modal/HF closures and runtime locks remain current without refresh. + +For review, the proposed next attempt must use a new pushed host commit with +the exact new engine gitlink, a fresh private plan/config/model/source/artifact +root, and a new execution attempt claim. The prior r14 host/gitlink/config and +every earlier model/source/start claim remain consumed or historical; they +cannot be reinterpreted as fresh authority. The ordered commands are the +checked-in example's `plan`, `model-preflight`, `prepare-model`, `prepare`, +public `launch.py plan`, then one separately admitted public `launch.py start`. +The model stage has no hard SDK download deadline; only the subsequent source +proof uses its 600-second validity window. Before any live step, independently +review exact pushed source and genuine host gitlink, wheel/consumer closure, +release and image C pins, daemon/device, source and model custody, prepared +input publication, fresh names, and the original 3-step full-precision SFT +recipe with 900-second/2-CPU/16-GiB/GPU-0/network-none bounds. Do not run a +model request, container or training job on this implementation evidence alone. + +The r16 independent review found that cold known-run operations had inherited +the live model-byte check, an absent seeded revision was rejected instead of +treated as a cold miss, and inventory hashing reopened the root pathname after +acquiring a scan descriptor. The r17 host-only repair retains the exact model +receipt inside the public attempt. Start still validates current model bytes +before its sole create; cold known-run operations check the committed receipt, +configuration and run identity without reading weights or the old preparation +root. A missing seed revision now downloads through the unchanged pinned +preparer, while existing redirects and mutated members reject. Inventory hashes +leaves through their held scan-parent descriptors and compares identities before +and after reading. Tiny CPU public-start and transient-ancestor-swap fixtures +exercise these boundaries; they do not qualify the desktop model or diagnose r2. +Public plan authenticates the receipt commitment without reading weights; the +sole live-byte recheck in the public consumer occurs on start before effects. diff --git a/examples/training_qualification/README.md b/examples/training_qualification/README.md index 8f3c8ee2..31b0dd32 100644 --- a/examples/training_qualification/README.md +++ b/examples/training_qualification/README.md @@ -178,8 +178,8 @@ Run from a neutral directory with the installed engine interpreter: ```sh python -I /absolute/checkout/examples/training_qualification/prepare_docker.py plan /private/prepare.json python -I /absolute/checkout/examples/training_qualification/prepare_docker.py model-preflight /private/prepare.json -python -I /absolute/checkout/examples/training_qualification/prepare_docker.py prepare /private/prepare.json python -I /absolute/checkout/examples/training_qualification/prepare_docker.py prepare-model /private/prepare.json +python -I /absolute/checkout/examples/training_qualification/prepare_docker.py prepare /private/prepare.json python -I /absolute/checkout/examples/training_qualification/launch.py start /private/prepared/consumer.json ``` @@ -192,14 +192,32 @@ pull, build, download a model or start a container. Original host manifests and pushed source evidence must be genuine and fresh. Its source admission helper is shared with the HF preparer without changing HF canonical outputs. -`prepare-model` is the explicit public-model network boundary: it uses the existing -pinned model preparer with anonymous credentials, private persistent/SDK scratch -under the preparation output and destination under the authenticated artifact -mount's `run/cache`. The existing verifier checks pinned file metadata/content; -the native snapshot validator checks the resulting offline path. The source -mount remains empty until the original claimed stage. Writable runtime directories -are private and disjoint; the exact prepared-input slot is never adopted. A -create-only local attempt prevents automatic repeat after an uncertain download. +`prepare-model` is the explicit public-model network boundary. With the template's +`model_output_root`, it consumes a fresh private model claim and prepares the +pinned model before any short-lived execution-source proof is minted. It uses +anonymous upstream metadata, private persistent/SDK scratch, and a fresh artifact +mount's `run/cache`. Optional `model_seed_root` points to an independently held, +private read-only repository cache; each present member is copied into fresh roots +and checked against current pinned Git/LFS metadata. A corrupt, redirected or +changed seed member rejects; absent members use the normal cold download path. +An absent entire pinned revision under an otherwise private seed is also a +cold miss; an existing redirected revision or member rejects. +The seed's old claim and bytes stay untouched and never become new authority. +The native snapshot validator checks the resulting offline path and a complete +bounded file inventory is retained in `model-preparation.json`. + +`prepare` first rechecks that receipt and actual model bytes against its declared +configuration, then authenticates fresh pushed source and compiles material. Its +consumer binds the exact receipt bytes; public `start` rechecks the model inventory +before Docker effects. Public start retains its own committed receipt copy. +Public `plan` authenticates receipt identity but does not attest current model +bytes; `start` performs that live check. +Later cold observe/cancel/reconcile/verify/download authenticate that retained +receipt, configuration and run identity without reopening weights or depending +on the preparation root. The source mount remains absent until the source stage. +Writable runtime directories are private and disjoint; the exact prepared-input +slot is never adopted. Each model and source root is one-use. A failed or uncertain +model preparation cannot be replayed under the same claim. Vendor download behavior is not a hard job deadline or bill cap. No manual weight upload, source inventory widening or restored training authority is involved. diff --git a/examples/training_qualification/docker_host.py b/examples/training_qualification/docker_host.py index 802940d5..93c6f9db 100644 --- a/examples/training_qualification/docker_host.py +++ b/examples/training_qualification/docker_host.py @@ -106,7 +106,151 @@ def _profile(raw): return profile -def _components(config): +def _hash_scanned_member(parent_fd, name, observed, maximum): + """Read the scanned leaf through its held parent, including identity checks.""" + if not stat.S_ISREG(observed.st_mode) or not 0 <= observed.st_size <= maximum: + raise ValueError("docker_model_member_invalid") + try: + fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=parent_fd) + except OSError: + raise ValueError("docker_model_member_changed") from None + try: + before = os.fstat(fd) + identity = (observed.st_dev, observed.st_ino, observed.st_size, observed.st_mtime_ns) + if not stat.S_ISREG(before.st_mode) or ( + before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns + ) != identity: + raise ValueError("docker_model_member_changed") + digest = hashlib.sha256() + size = 0 + while True: + chunk = os.read(fd, min(1 << 20, maximum + 1 - size)) + if not chunk: + break + size += len(chunk) + if size > maximum: + raise ValueError("docker_model_inventory_bound") + digest.update(chunk) + after = os.fstat(fd) + current = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + if size != observed.st_size or ( + after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns + ) != identity or ( + current.st_dev, current.st_ino, current.st_size, current.st_mtime_ns + ) != identity: + raise ValueError("docker_model_member_changed") + return size, digest.hexdigest() + except OSError: + raise ValueError("docker_model_member_changed") from None + finally: + os.close(fd) + + +def model_inventory(artifact_root, snapshot, *, missing_ok=False): + """Hash the complete private offline snapshot through bounded regular reads.""" + artifact_root = Path(artifact_root) + snapshot = Path(snapshot) + with ExitStack() as stack: + guard = _Root(artifact_root) + stack.callback(guard.stack.close) + guard.check() + if not snapshot.is_relative_to(artifact_root) or ".." in snapshot.parts: + raise ValueError("docker_model_snapshot_outside_artifact_root") + rows = [] + total = 0 + parent = guard.fd + chain = [] + for part in snapshot.relative_to(artifact_root).parts: + try: + child = os.open(part, _DIR, dir_fd=parent) + except FileNotFoundError: + if not missing_ok: + raise + DockerLocalArtifactsV1._recheck(chain) + guard.check() + return [] + except OSError: + raise ValueError("docker_model_snapshot_path_invalid") from None + stack.callback(os.close, child) + chain.append((parent, part, child)) + parent = child + pending = [(snapshot, parent)] + seen = 0 + directories = 0 + while pending: + directory, directory_fd = pending.pop() + with os.scandir(directory_fd) as entries: + for entry in entries: + seen += 1 + if seen > 20_000: + raise ValueError("docker_model_inventory_bound") + info = entry.stat(follow_symlinks=False) + if stat.S_ISDIR(info.st_mode) and not entry.is_symlink(): + directories += 1 + if directories > 256: + raise ValueError("docker_model_inventory_bound") + child = os.open(entry.name, _DIR, dir_fd=directory_fd) + stack.callback(os.close, child) + chain.append((directory_fd, entry.name, child)) + pending.append((directory / entry.name, child)) + elif stat.S_ISREG(info.st_mode) and not entry.is_symlink(): + if len(rows) >= 20_000 or total + info.st_size > 1 << 40: + raise ValueError("docker_model_inventory_bound") + path = directory / entry.name + size, digest = _hash_scanned_member( + directory_fd, entry.name, info, info.st_size + ) + rows.append({"path": path.relative_to(snapshot).as_posix(), + "size_bytes": size, "sha256": digest}) + total += size + else: + raise ValueError("docker_model_member_invalid") + DockerLocalArtifactsV1._recheck(chain) + guard.check() + if not rows: + raise ValueError("docker_model_snapshot_empty") + return sorted(rows, key=lambda item: item["path"]) + + +def verify_model_receipt(raw, *, selected, model, configuration_sha256, + runtime_declaration_sha256, live_bytes=True): + """Bind fresh prepared bytes to the later public consumer's exact inputs.""" + receipt = _json(raw) + artifact_root = Path(selected["artifact_host_root"]) + cache = artifact_root / "run/cache" + expected = (cache / "model" / ("models--" + model["ref"].replace("/", "--")) + / "snapshots" / model["revision"]) + if (set(receipt) != {"schema_version", "model", "configuration_sha256", + "runtime_declaration_sha256", "artifact_host_root", + "snapshot", "files"} + or receipt["schema_version"] != "synaptic-docker-model-preparation/v1" + or receipt["model"] != model + or receipt["configuration_sha256"] != configuration_sha256 + or receipt["runtime_declaration_sha256"] != runtime_declaration_sha256 + or receipt["artifact_host_root"] != str(artifact_root) + or receipt["snapshot"] != str(expected) + or not isinstance(receipt["files"], list)): + raise ValueError("docker_model_receipt_mismatch") + if live_bytes: + from Trainers.sft.runtime_v1 import RuntimeRoots, _require_local_model_snapshot + + roots = RuntimeRoots( + engine=Path(selected["source_host_root"]) / "engine", + project=Path(selected["source_host_root"]) / "project", + artifacts=artifact_root / "run/artifacts", state=artifact_root / "run/state", + tracking=artifact_root / "run/tracking", cache=cache, + tmp=artifact_root / "run/tmp", + ) + _require_local_model_snapshot(model, roots, { + "SYNAPTIC_CACHE_ROOT": str(cache), "SYNAPTIC_MODEL_SNAPSHOT": str(expected), + "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", + }) + if model_inventory(artifact_root, expected) != receipt["files"]: + raise ValueError("docker_model_receipt_bytes_changed") + return receipt + + +def _components(config, *, live_model_bytes=False): if config["provider"] != "docker": raise ValueError("Docker provider required") selected = config["docker"] @@ -129,7 +273,7 @@ def _components(config): "command_seconds", "read_seconds", } - if set(selected) - {"allow_current_cancel_authorization"} != fields: + if set(selected) - {"allow_current_cancel_authorization", "model_preparation_file"} != fields: raise ValueError("Docker configuration fields invalid") profile = _profile(read_committed(selected["profile_file"])) recipes = default_recipe_registry() @@ -151,6 +295,16 @@ def _components(config): document["artifact_policy"]["retain_checkpoints"], ), ) + if "model_preparation_file" in selected: + verify_model_receipt( + read_committed(selected["model_preparation_file"]), + selected=selected, model=rich.resolved_config.to_dict()["model"], + configuration_sha256=hashlib.sha256( + rich.resolved_config.canonical_json.encode()).hexdigest(), + runtime_declaration_sha256=hashlib.sha256( + rich.execution_context.canonical_json.encode()).hexdigest(), + live_bytes=live_model_bytes, + ) checked, resolved = DockerTrainingPreparationBridgeV1(profile).coordinator_material( rich, material, recipes ) @@ -177,10 +331,13 @@ def preview(config): def retain_inputs(config, root): # Preserve the existing canonical documents, never serialize a new rich type. - for key, name in ( + references = [ ("material_file", "docker-material.json"), ("profile_file", "docker-profile.json"), - ): + ] + if "model_preparation_file" in config["docker"]: + references.append(("model_preparation_file", "docker-model-preparation.json")) + for key, name in references: raw = read_committed(config["docker"][key]) target = root / name _write_bytes(target, raw) @@ -318,7 +475,9 @@ def open_host( if mode == "start": _require_runtime_root_platform() selected = config["docker"] - profile, rich, material, recipes, basis = _components(config) + profile, rich, material, recipes, basis = _components( + config, live_model_bytes=mode == "start" + ) run = TrainingRunRef(material.run_id, basis.project_ref) if config["run"] != run.to_dict(): raise ValueError("configured run differs from material") diff --git a/examples/training_qualification/prepare-docker.config.example.json b/examples/training_qualification/prepare-docker.config.example.json index 2a4bf654..7a620961 100644 --- a/examples/training_qualification/prepare-docker.config.example.json +++ b/examples/training_qualification/prepare-docker.config.example.json @@ -7,6 +7,8 @@ "run_id": "REPLACE_UNUSED_RUN", "request_id": "REPLACE_UNUSED_REQUEST", "output_root": "/absolute/private/REPLACE_UNUSED_PREPARATION", + "model_output_root": "/absolute/private/REPLACE_UNUSED_MODEL_PREPARATION", + "model_seed_root": null, "prepared_publication": "/absolute/REPLACE_VERIFIED_V1_PUBLICATION", "resolved_config": "/absolute/REPLACE_CANONICAL_SFT_CONFIG.json", "runtime_release": "/absolute/REPLACE_ACTUAL_VERIFIED_RUNTIME_RELEASE.json", diff --git a/examples/training_qualification/prepare_docker.py b/examples/training_qualification/prepare_docker.py index 60d66004..38c9ddf6 100644 --- a/examples/training_qualification/prepare_docker.py +++ b/examples/training_qualification/prepare_docker.py @@ -128,6 +128,7 @@ def filesystem_probe(configuration): _bind_hub_api, prepare_model_snapshot, ) +from tuner.execution.providers.modal.mounted_io import copy_regular, hash_regular from tuner.project.context import ProjectContext from tuner.project.git_verification import ( GitCliLocalSourceInspector, @@ -242,6 +243,16 @@ def prepare(config, *, pushed_sources, authenticator, clock): normalized = common._document(config["resolved_config"]) if normalized.to_dict().get("dataset") != identity.to_dict(): raise ValueError("docker_prepared_configuration_mismatch") + model_receipt_path = None + if "model_output_root" in config: + model_receipt_path = Path(config["model_output_root"]) / "model-preparation.json" + docker_host.verify_model_receipt( + common.stable_read(model_receipt_path, 2 << 20), + selected=config["consumer"]["docker"], + model=normalized.to_dict()["model"], + configuration_sha256=sha(normalized.canonical_json.encode()), + runtime_declaration_sha256=sha(runtime.canonical_json.encode()), + ) context = ProjectContext.host( project_root=Path(config["project"]["root"]), engine_root=Path(config["project"]["engine_root"]), @@ -364,6 +375,12 @@ def spec(name): consumer["docker"]["material_file"] = spec("material.json") consumer["docker"]["profile_file"] = spec("docker-profile.json") consumer["docker"]["prepared_publication"] = config["prepared_publication"] + if model_receipt_path is not None: + data = common.stable_read(model_receipt_path, 2 << 20) + consumer["docker"]["model_preparation_file"] = { + "path": str(model_receipt_path), "sha256": sha(data), + "size_bytes": len(data), + } launch._write(root / "consumer-candidate.json", consumer) consumer = launch.load_config(root / "consumer-candidate.json") preview = docker_host.preview(consumer) @@ -399,34 +416,87 @@ def prepare_model( from Trainers.sft.runtime_v1 import RuntimeRoots, _require_local_model_snapshot sdk_preflight() - output = Path(config["output_root"]) - launch._private(output) - consumer = launch.load_config(output / "consumer.json") - _, rich, _, _, _ = docker_host._components(consumer) - if rich.execution_context != declaration(config): - raise ValueError("docker_model_configuration_changed") - model = rich.resolved_config.to_dict()["model"] - selected = consumer["docker"] + model_first = "model_output_root" in config + runtime = declaration(config) + if model_first: + output = Path(config["model_output_root"]) + if (not output.is_absolute() or output.exists() or output.is_symlink() + or output == Path(config["output_root"])): + raise ValueError("docker_model_output_used_or_invalid") + launch._private(output.parent) + if Path(config["output_root"]).exists() or Path(config["output_root"]).is_symlink(): + raise ValueError("docker_source_preparation_already_claimed") + selected = config["consumer"]["docker"] + normalized = common._document(config["resolved_config"]) + model = normalized.to_dict()["model"] + roots = runtime.to_dict()["roots"] + else: + output = Path(config["output_root"]) + launch._private(output) + consumer = launch.load_config(output / "consumer.json") + _, rich, _, _, _ = docker_host._components(consumer) + if rich.execution_context != runtime: + raise ValueError("docker_model_configuration_changed") + normalized = rich.resolved_config + model = normalized.to_dict()["model"] + selected = consumer["docker"] + roots = rich.execution_source.roots source, artifact = ( Path(selected["source_host_root"]), Path(selected["artifact_host_root"]), ) - for root in (source, artifact): + if (not source.is_absolute() or not artifact.is_absolute() + or source == artifact or source in artifact.parents + or artifact in source.parents): + raise ValueError("docker_model_roots_invalid") + if model_first and (output in {source, artifact} or output in source.parents + or output in artifact.parents or source in output.parents + or artifact in output.parents): + raise ValueError("docker_model_output_overlaps_mount") + if model_first and (source.exists() or source.is_symlink() + or artifact.exists() or artifact.is_symlink()): + raise ValueError("docker_model_fresh_roots_required") + seed = config.get("model_seed_root") if model_first else None + if seed is not None: + seed = Path(seed) + if (not seed.is_absolute() or seed in {output, source, artifact} + or seed in output.parents or seed in artifact.parents + or output in seed.parents or artifact in seed.parents + or source in seed.parents or seed in source.parents): + raise ValueError("docker_model_seed_invalid") + launch._private(seed) + if model_first: + # Consume the exact one-use model claim before creating any model bytes. + output.mkdir(mode=0o700) + launch._private(output) + launch._write( + output / "model-preparation-attempt.json", + {"model": model, "destination": str(artifact / "run/cache"), + "source_mutation": False, "seed_root": None if seed is None else str(seed)}, + ) + for root in ((artifact,) if model_first else (source, artifact)): if not root.exists(): launch._private(root.parent) root.mkdir(mode=0o700) launch._private(root) with ExitStack() as held: guards = [] - for root in (source, artifact): + for root in ((artifact,) if model_first else (source, artifact)): guard = _Root(root) held.callback(guard.stack.close) guards.append(guard) guard.check() - if any(source.iterdir()): + if model_first and (source.exists() or source.is_symlink()): + raise ValueError("docker_source_changed") + if source.exists() and any(source.iterdir()): raise ValueError("docker_source_not_empty") + seed_guard = None + if seed is not None: + seed_guard = _Root(seed) + held.callback(seed_guard.stack.close) + seed_guard.check() mapped = {} - for name, raw in rich.execution_source.roots.items(): + for name, raw in roots.items(): path = PurePosixPath(raw) if path.is_relative_to("/source"): mapped[name] = source.joinpath(*path.relative_to("/source").parts) @@ -446,17 +516,34 @@ def prepare_model( path.mkdir(mode=0o700) launch._private(path) persistent, scratch = output / "model-persistent", output / "model-scratch" - launch._write( - output / "model-preparation-attempt.json", - { - "model": model, - "destination": str(mapped["cache"]), - "source_mutation": False, - }, - ) + if not model_first: + launch._write( + output / "model-preparation-attempt.json", + {"model": model, "destination": str(mapped["cache"]), + "source_mutation": False}, + ) for path in (persistent, scratch): path.mkdir(mode=0o700) - snapshot = model_preparer( + if seed is not None: + relative = (Path("models--" + model["ref"].replace("/", "--")) + / "snapshots" / model["revision"]) + # Copy a private repository seed into this attempt's fresh cache. + # The unchanged shared preparer authenticates every reused member + # against pinned upstream Git/LFS metadata before destination use. + for row in docker_host.model_inventory( + seed, seed / relative, missing_ok=True + ): + name = Path(row["path"]) + copied = copy_regular( + seed, seed / relative / name, persistent, + persistent / relative / name, maximum=row["size_bytes"], + ) + if copied != (row["size_bytes"], row["sha256"]): + raise ValueError("docker_model_seed_changed") + if hash_regular(seed, seed / relative / name, row["size_bytes"]) != copied: + raise ValueError("docker_model_seed_changed") + seed_guard.check() + options = dict( model_ref=model["ref"], revision=model["revision"], token=False, @@ -464,6 +551,7 @@ def prepare_model( destination_root=mapped["cache"], scratch_root=scratch, ) + snapshot = model_preparer(**options) expected = ( mapped["cache"] / "model" @@ -484,9 +572,13 @@ def prepare_model( ) for guard in guards: guard.check() - for root in (source, artifact): + if seed_guard is not None: + seed_guard.check() + for root in ((artifact,) if model_first else (source, artifact)): launch._private(root) - if any(source.iterdir()): + if model_first and (source.exists() or source.is_symlink()): + raise ValueError("docker_source_changed") + if source.exists() and any(source.iterdir()): raise ValueError("docker_source_changed") result = { "snapshot": str(checked), @@ -495,6 +587,16 @@ def prepare_model( "source_mutation": False, "provider_qualified": False, } + if model_first: + result = { + "schema_version": "synaptic-docker-model-preparation/v1", + "model": model, + "configuration_sha256": sha(normalized.canonical_json.encode()), + "runtime_declaration_sha256": sha(runtime.canonical_json.encode()), + "artifact_host_root": str(artifact), + "snapshot": str(checked), + "files": docker_host.model_inventory(artifact, checked), + } launch._write(output / "model-preparation.json", result) return result diff --git a/scripts/diagnose_docker_torch_import.py b/scripts/diagnose_docker_torch_import.py new file mode 100644 index 00000000..07808bb8 --- /dev/null +++ b/scripts/diagnose_docker_torch_import.py @@ -0,0 +1,1147 @@ +"""Bounded, credential-free Torch import diagnosis for an exact packaged image. + +This is a diagnostic, never training or qualification authority. It retains +the new container and writes only closed results to a fresh private directory. +""" + +from __future__ import annotations + +import argparse +import ast +import base64 +import hashlib +import json +import os +import re +import secrets +import selectors +import signal +import stat +import subprocess +import sys +import time +from pathlib import Path + + +SCHEMA = "synaptic-docker-torch-import-diagnostic/v1" +_SHA = re.compile(r"sha256:[0-9a-f]{64}\Z") +_CID = re.compile(r"[0-9a-f]{64}\Z") +_CATEGORIES = frozenset({"PASS", "MODULE_NOT_FOUND", "IMPORT", "OS", "RUNTIME", "OTHER"}) +_REASONS = frozenset({"DAEMON_ID_INVALID", "INPUT_INVALID", "DOCKER_INVALID", + "DOCKER_CHANGED", "DOCKER_COMMAND_REJECTED", "DIAGNOSTIC_OUTPUT_INVALID", + "RELEASE_INVALID", "RELEASE_HASH_INVALID", "OUTPUT_INVALID", "SOURCE_INVALID", "DAEMON_INVALID", "IMAGE_INVALID", + "CHILD_OUTPUT_INVALID", "CONTAINER_ID_INVALID", "CONTAINER_IMAGE_INVALID", + "IMAGE_CHANGED", "SOURCE_IDENTITY_INVALID", "GPU_SCOPE_INVALID", "CREATE_UNCERTAIN", "START_UNCERTAIN", + "DEADLINE_UNCERTAIN", "DEADLINE_STOPPED", "STOP_UNCONFIRMED", "CONTAINER_STATE_INVALID"}) +_CLI_BUDGET = 8192 +_RUN_SECONDS = 120 +_HIDDEN_GPU_ENV = ",'CUDA_VISIBLE_DEVICES':'','NVIDIA_VISIBLE_DEVICES':'void'" +_R2_ROOTS = {"engine": "/source/engine", "project": "/source/project", + "artifacts": "/artifacts/run/artifacts", "state": "/artifacts/run/state", + "tracking": "/artifacts/run/tracking", "cache": "/artifacts/run/cache", + "tmp": "/artifacts/run/tmp"} +_R2_ENV_KEYS = frozenset({"PATH", "PYTHONPATH", "PYTHONNOUSERSITE", + "PYTHONSAFEPATH", "HF_HOME", "HF_HUB_OFFLINE", "TRANSFORMERS_CACHE", + "TRANSFORMERS_OFFLINE", "WANDB_DISABLED", "SYNAPTIC_ENGINE_ROOT", + "SYNAPTIC_PROJECT_ROOT", "SYNAPTIC_ARTIFACT_ROOT", "SYNAPTIC_STATE_ROOT", + "SYNAPTIC_TRACKING_ROOT", "SYNAPTIC_CACHE_ROOT", "SYNAPTIC_TMP_ROOT"}) +_ROOT_ENV = {name: "SYNAPTIC_" + name.upper() + "_ROOT" for name in _R2_ROOTS} +_ROOT_ENV["artifacts"] = "SYNAPTIC_ARTIFACT_ROOT" +_INSPECT = "{{json .Id}}\n{{json .Name}}\n{{json .Image}}\n{{json .Config.Labels}}\n{{json .State.Status}}\n{{json .State.ExitCode}}" + +# This fixed program never formats an exception, inspects its attributes, or +# imports an engine/trainer module. The exact packaged interpreter imports +# Torch under the isolated, offline child environment. +_PROBE = """import json,os,sys +report_fd=os.dup(1) +null_fd=os.open('/dev/null',os.O_WRONLY) +os.dup2(null_fd,1);os.dup2(null_fd,2);os.close(null_fd) +os.environ.clear() +os.environ.update({'PATH':os.path.dirname(sys.executable)+':/usr/local/bin:/usr/bin:/bin','HOME':'/tmp','TMPDIR':'/tmp','LC_ALL':'C.UTF-8','PYTHONNOUSERSITE':'1','PYTHONSAFEPATH':'1','PYTHONDONTWRITEBYTECODE':'1','HF_HUB_OFFLINE':'1','TRANSFORMERS_OFFLINE':'1','WANDB_DISABLED':'true','CUDA_VISIBLE_DEVICES':'','NVIDIA_VISIBLE_DEVICES':'void'}) +try: + import torch + category='PASS' +except ModuleNotFoundError: + category='MODULE_NOT_FOUND' +except ImportError: + category='IMPORT' +except OSError: + category='OS' +except RuntimeError: + category='RUNTIME' +except BaseException: + category='OTHER' +os.write(report_fd,(json.dumps({'schema_version':'synaptic-docker-torch-import-child/v1','category':category},sort_keys=True,separators=(',',':'))+'\\n').encode('ascii')) +os.close(report_fd) +""" + +_GUARDED_PROBE = """import json,os,sys +report_fd=os.dup(1) +null_fd=os.open('/dev/null',os.O_WRONLY) +os.dup2(null_fd,1);os.dup2(null_fd,2);os.close(null_fd) +from pathlib import Path +os.environ.clear() +os.environ.update({'PATH':os.path.dirname(sys.executable)+':/usr/local/bin:/usr/bin:/bin','HOME':'/tmp','TMPDIR':'/tmp','LC_ALL':'C.UTF-8','PYTHONNOUSERSITE':'1','PYTHONSAFEPATH':'1','PYTHONDONTWRITEBYTECODE':'1','HF_HUB_OFFLINE':'1','TRANSFORMERS_OFFLINE':'1','WANDB_DISABLED':'true','CUDA_VISIBLE_DEVICES':'','NVIDIA_VISIBLE_DEVICES':'void'}) +root=Path('/source/engine') +sys.path.insert(0,str(root)) +stage='IMPORT_WORKER' +try: + from tuner.runtime.offline_sft_worker import load_offline_sft_worker_closure,install_owned_module_guard,verify_loaded_owned_module_origins + stage='READ_MANIFEST' + manifest=Path('/source/control/offline-sft-worker-v1.json') + declared='EXPECTED_CLOSURE_DIGEST' + stage='LOAD_CLOSURE' + closure=load_offline_sft_worker_closure(manifest,expected_digest=declared,engine_root=root) + stage='VERIFY_ORIGINS' + verify_loaded_owned_module_origins(closure,engine_root=root) + stage='INSTALL_GUARD' + install_owned_module_guard(closure,engine_root=root) + sys.path[:0]=[str(root),str(root/'Trainers/sft/src'),str(root/'Trainers/sft'),str(root)] + stage='BOOTSTRAP_ENV' + from shared.env_bootstrap import init_trainer_env + init_trainer_env() + stage='TORCH_IMPORT' + import torch + category='PASS' +except ModuleNotFoundError: + category='MODULE_NOT_FOUND' +except ImportError: + category='IMPORT' +except OSError: + category='OS' +except RuntimeError: + category='RUNTIME' +except BaseException: + category='OTHER' +os.write(report_fd,(json.dumps({'schema_version':'synaptic-docker-torch-import-child/v1','category':category,'stage':stage},sort_keys=True,separators=(',',':'))+'\\n').encode('ascii')) +os.close(report_fd) +""" + +# A second -I interpreter receives the reconstructed offline child environment. +# Only the authenticated trainer prefix is compiled. Its final statement is +# import torch; the UNSLOTH_IMPORT marker and every later byte are excluded. +_WORKER_PREFIX_CHILD = """import hashlib,json,os,sys +report_fd=os.dup(1) +null_fd=os.open('/dev/null',os.O_WRONLY) +os.dup2(null_fd,1);os.dup2(null_fd,2);os.close(null_fd) +from pathlib import Path +def _environment_evidence(supplied,observed): + if type(supplied) is not dict or type(observed) is not dict or 'LC_CTYPE' in supplied: + raise ValueError() + if any(type(key) is not str or type(value) is not str for key,value in supplied.items()): + raise ValueError() + if any(observed.get(key)!=value for key,value in supplied.items()): + raise ValueError() + additions={key:value for key,value in observed.items() if key not in supplied} + if additions=={}: + delta='NONE' + elif additions=={'LC_CTYPE':'C.UTF-8'}: + delta='LC_CTYPE_C_UTF8' + else: + raise ValueError() + return {'environment_supplied_sha256':hashlib.sha256(json.dumps(supplied,sort_keys=True,separators=(',',':')).encode()).hexdigest(), + 'environment_observed_sha256':hashlib.sha256(json.dumps(observed,sort_keys=True,separators=(',',':')).encode()).hexdigest(), + 'startup_env_delta':delta} +stage='WORKER_IMPORT' +category='OTHER' +context={} +try: + supplied_raw=sys.stdin.buffer.read(8193) + if not supplied_raw or len(supplied_raw)>8192: + raise ValueError() + supplied_env=json.loads(supplied_raw) + if _environment_evidence(supplied_env,dict(os.environ))['environment_supplied_sha256']!='EXPECTED_SUPPLIED_ENV_SHA256': + raise ValueError() + if sys.flags.isolated!=1 or sys.flags.no_user_site!=1 or not sys.flags.safe_path: + raise ValueError() + if os.environ.get('PYTHONNOUSERSITE')!='1' or os.environ.get('PYTHONSAFEPATH')!='1': + raise ValueError() + root=Path('/source/engine') + import runpy + worker=runpy.run_path(str(root/'tuner/runtime/offline_sft_worker.py'),run_name='__worker_diagnostic__') + stage='WORKER_CLOSURE' + closure=worker['load_offline_sft_worker_environment'](os.environ,engine_root=root) + stage='WORKER_GUARD' + worker['verify_loaded_owned_module_origins'](closure,engine_root=root) + worker['install_owned_module_guard'](closure,engine_root=root) + trainer=root/'Trainers/sft/train_sft.py' + sys.path[:0]=[str(trainer.parent),str(root)] + stage='TRAINER_PREFIX' + with trainer.open('rb') as stream: + source=stream.read(1024*1024+1) + if len(source)>1024*1024: + raise ValueError() + if hashlib.sha256(source).hexdigest()!='EXPECTED_TRAINER_SHA256': + raise ValueError() + lines=source.decode('utf-8').splitlines(keepends=True) + markers=('_mark_packaged_runtime_phase("BOOTSTRAP_ENV")','_mark_packaged_runtime_phase("TORCH_IMPORT")','_mark_packaged_runtime_phase("UNSLOTH_IMPORT")') + positions=[next((i for i,line in enumerate(lines) if line.strip()==marker),-1) for marker in markers] + if positions[0]<0 or not positions[0]1024*1024 or hashlib.sha256(worker_raw).hexdigest()!='EXPECTED_WORKER_SHA256': + raise ValueError() + with trainer.open('rb') as stream: + source=stream.read(1024*1024+1) + if len(source)>1024*1024 or hashlib.sha256(source).hexdigest()!='EXPECTED_TRAINER_SHA256': + raise ValueError() + lines=source.decode('utf-8').splitlines(keepends=True) + markers=('_mark_packaged_runtime_phase("BOOTSTRAP_ENV")','_mark_packaged_runtime_phase("TORCH_IMPORT")','_mark_packaged_runtime_phase("UNSLOTH_IMPORT")') + positions=[next((i for i,line in enumerate(lines) if line.strip()==marker),-1) for marker in markers] + if positions[0]<0 or not positions[0]=2 and closed[-1]==b'' else b'' + match=re.fullmatch(rb'OFFLINE_SFT_WORKER_REJECTED phase=(BOOTSTRAP|CLOSURE|BOOTSTRAP_ORIGIN|ARGUMENTS|IMPORT_GUARD|TRAINER|POSTCHECK) category=(REJECTION|IMPORT|OS|VALUE|SYSTEM_EXIT|OTHER)(?: milestone=(BOOTSTRAP_ENV|TORCH_IMPORT|UNSLOTH_IMPORT|TRAINER_IMPORT|TRAIN_CALL))?',label) + if blocked or match is None: + raise ValueError() + if reached and context['runpy_main_bound'] and match.groups()==(b'TRAINER',b'OTHER',b'TORCH_IMPORT'): + category='PASS' + else: + context={} + phase,kind,milestone=match.groups() + stage=({'BOOTSTRAP':'WORKER_MAIN','CLOSURE':'WORKER_CLOSURE','BOOTSTRAP_ORIGIN':'WORKER_ORIGIN','ARGUMENTS':'WORKER_ARGUMENTS','IMPORT_GUARD':'WORKER_GUARD','TRAINER':'TORCH_IMPORT' if milestone==b'TORCH_IMPORT' else 'TRAINER_PREFIX','POSTCHECK':'WORKER_POSTCHECK'})[phase.decode('ascii')] + category=kind.decode('ascii') if kind in (b'IMPORT',b'OS') else 'OTHER' +except ModuleNotFoundError: + category='MODULE_NOT_FOUND' +except ImportError: + category='IMPORT' +except OSError: + category='OS' +except RuntimeError: + category='RUNTIME' +except BaseException: + category='OTHER' +if category!='PASS': + context={} +os.write(1,(json.dumps({'schema_version':'synaptic-docker-torch-import-child/v1','category':category,'stage':stage,'context':context,'boundary':'BEFORE_UNSLOTH'},sort_keys=True,separators=(',',':'))+'\\n').encode('ascii')) +""" + +_WORKER_CONTEXT_PROBE = """import base64,hashlib,json,os,sys,subprocess +report_fd=os.dup(1) +null_fd=os.open('/dev/null',os.O_WRONLY) +os.dup2(null_fd,1);os.dup2(null_fd,2);os.close(null_fd) +from pathlib import Path +stage='READ_WORKLOAD' +category='OTHER' +context={} +try: + with Path('/source/control/workload.json').open('rb') as stream: + raw=stream.read(1024*1024+1) + if len(raw)>1024*1024: + raise ValueError() + if hashlib.sha256(raw).hexdigest()!='EXPECTED_WORKLOAD_SHA256': + raise ValueError() + workload=json.loads(raw) + runtime=workload['execution_source']['runtime'] + roots=runtime['roots'] + if roots!=EXPECTED_ROOTS or runtime['environment']['clear_inherited'] is not True: + raise ValueError() + planned=runtime['environment']['variables'] + if set(planned)!=set(EXPECTED_ENV_KEYS) or any(type(v) is not str for v in planned.values()): + raise ValueError() + environment=dict(planned) + environment.pop('PYTHONPATH') + environment.update({'SYNAPTIC_WORKER_CLOSURE_MANIFEST':'/source/control/offline-sft-worker-v1.json','SYNAPTIC_WORKER_CLOSURE_DIGEST':'EXPECTED_CLOSURE_DIGEST'}) + environment.update({key:value for key,value in ROOT_BINDINGS.items()}) + model=workload['configuration']['document']['model'] + ref=model['ref']; revision=model['revision'] + if not isinstance(ref,str) or not isinstance(revision,str) or not __import__('re').fullmatch('[0-9a-f]{40}(?:[0-9a-f]{24})?',revision) or any(not __import__('re').fullmatch('[A-Za-z0-9][A-Za-z0-9._-]*',part) for part in ref.split('/')) or len(ref.split('/')) not in (1,2): + raise ValueError() + snapshot=roots['cache']+'/model/models--'+'--'.join(ref.split('/'))+'/snapshots/'+revision + environment.update({'SYNAPTIC_WORKLOAD_FINGERPRINT':hashlib.sha256(b'synaptic-training-workload/v1\\0'+raw).hexdigest(),'PYTHONNOUSERSITE':'1','PYTHONSAFEPATH':'1','HF_HOME':roots['cache']+'/huggingface','TRANSFORMERS_CACHE':roots['cache']+'/transformers','HF_HUB_OFFLINE':'1','TRANSFORMERS_OFFLINE':'1','SYNAPTIC_MODEL_SNAPSHOT':snapshot,'WANDB_DISABLED':'true'}) + os.umask(0o077) + Path('/artifacts/run').mkdir(mode=0o700) + for name in ('artifacts','state','tracking','cache','tmp'): + Path(roots[name]).mkdir(mode=0o700) + for name in ('huggingface','transformers'): + (Path(roots['cache'])/name).mkdir(mode=0o700,exist_ok=True) + stage='ISOLATED_CHILD' + child=base64.b64decode('CHILD_SOURCE_B64') + if ENTRY_MODE: + import fcntl,runpy + runtime_path=Path('/source/engine/Trainers/sft/runtime_v1.py') + with runtime_path.open('rb') as stream: + runtime_raw=stream.read(1024*1024+1) + if len(runtime_raw)>1024*1024 or hashlib.sha256(runtime_raw).hexdigest()!='EXPECTED_RUNTIME_SHA256': + raise ValueError() + runtime_module=runpy.run_path(str(runtime_path),run_name='__diagnostic_runtime__') + config=workload['configuration']['document'];dataset=config['dataset'] + if not dataset['ref'].startswith('prepared://sha256/'): + raise ValueError() + descriptor=os.memfd_create('synaptic-diagnostic-input',os.MFD_CLOEXEC|os.MFD_ALLOW_SEALING) + try: + os.write(descriptor,b'0') + seals=fcntl.F_SEAL_SEAL|fcntl.F_SEAL_SHRINK|fcntl.F_SEAL_GROW|fcntl.F_SEAL_WRITE + fcntl.fcntl(descriptor,fcntl.F_ADD_SEALS,seals) + arguments=['--','--model-name',ref,'--model-revision',revision,'--anonymous-model', + '--model-cache-dir',roots['cache']+'/model','--model-snapshot',snapshot, + '--local-file','/proc/self/fd/'+str(descriptor), + '--output-root',roots['state']+'/runtime-v1-trainer/output', + '--run-timestamp','runtime-v1','--no-dashboard','--quiet', + '--runtime-v1-workload-fingerprint',environment['SYNAPTIC_WORKLOAD_FINGERPRINT'], + '--runtime-v1-configuration-revision',workload['configuration']['revision'], + '--runtime-v1-tokenizer-revision',model['tokenizer_revision'], + '--runtime-v1-dataset-revision',dataset['revision'], + '--runtime-v1-dataset-digest',dataset['content_digest']] + runtime_module['_append_sft_arguments'](arguments,config['sft'],model, + dataset_schema=dataset['format'] if dataset['format']=='synaptic-conversation-sft-row/v1' else None) + trainer_argv=['/source/engine/Trainers/sft/train_sft.py',*arguments[1:]] + argv_sha=hashlib.sha256(json.dumps(trainer_argv,sort_keys=True,separators=(',',':')).encode()).hexdigest() + token='EXPECTED'+'_'+'TRAINER_ARGV_SHA256' + child_text=child.decode('utf-8').replace(token,argv_sha) + if token in child_text: + raise ValueError() + stdout_path=Path(roots['tracking'])/'worker-entry.stdout' + stderr_path=Path(roots['tracking'])/'worker-entry.stderr' + with stdout_path.open('xb') as stdout,stderr_path.open('xb') as stderr: + process=subprocess.Popen([sys.executable,'-I','-c',child_text,*arguments],cwd=roots['tmp'],env=environment,stdin=subprocess.DEVNULL,stdout=stdout,stderr=stderr,pass_fds=(descriptor,)) + if process.wait()!=0: + raise ValueError() + with stdout_path.open('rb') as stream: + payload=stream.read(4097) + if len(payload)>4096: + raise ValueError() + finally: + os.close(descriptor) + else: + supplied_raw=json.dumps(environment,sort_keys=True,separators=(',',':')).encode() + if len(supplied_raw)>8192: + raise ValueError() + process=subprocess.Popen([sys.executable,'-I','-c',child.decode('utf-8')],cwd=roots['tmp'],env=environment,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL) + process.stdin.write(supplied_raw) + process.stdin.close() + payload=process.stdout.read(4097) + if len(payload)>4096: + process.kill();process.wait() + raise ValueError() + if process.wait()!=0: + raise ValueError() + result=json.loads(payload) + if set(result)!={'schema_version','category','stage','context','boundary'} or result['schema_version']!='synaptic-docker-torch-import-child/v1' or result['boundary']!='BEFORE_UNSLOTH' or result['category'] not in ('PASS','MODULE_NOT_FOUND','IMPORT','OS','RUNTIME','OTHER') or result['stage'] not in ('WORKER_IMPORT','WORKER_CLOSURE','WORKER_GUARD','TRAINER_PREFIX','BOOTSTRAP_ENV','TORCH_IMPORT','ENTRY_ADMISSION','WORKER_MAIN','WORKER_ORIGIN','WORKER_ARGUMENTS','WORKER_POSTCHECK') or not isinstance(result['context'],dict): + raise ValueError() + if result['category']=='PASS': + expected_context={'environment_supplied_sha256','environment_observed_sha256','startup_env_delta','python_isolated','cwd_sha256','file_sha256','sys_path_sha256'} + if ENTRY_MODE: + expected_context.update(('argv_sha256','runpy_main_bound','environment_before_torch_sha256','environment_after_torch_sha256')) + if result['stage']!='TORCH_IMPORT' or set(result['context'])!=expected_context or result['context']['python_isolated'] is not True or result['context']['startup_env_delta'] not in ('NONE','LC_CTYPE_C_UTF8') or (ENTRY_MODE and (result['context']['runpy_main_bound'] is not True or result['context']['argv_sha256']!=argv_sha)) or any(not isinstance(value,str) or not __import__('re').fullmatch('[0-9a-f]{64}',value) for key,value in result['context'].items() if key.endswith('_sha256')): + raise ValueError() + elif result['context']!={}: + raise ValueError() + category=result['category'];stage=result['stage'];context=result['context'] +except BaseException: + context={} +os.write(report_fd,(json.dumps({'schema_version':'synaptic-docker-torch-import-child/v1','category':category,'stage':stage,'context':context,'boundary':'BEFORE_UNSLOTH'},sort_keys=True,separators=(',',':'))+'\\n').encode('ascii')) +os.close(report_fd) +""" + + +def _read(path: Path, maximum: int) -> bytes: + if path.is_symlink() or not path.is_file() or path.stat().st_size > maximum: + raise ValueError("INPUT_INVALID") + with path.open("rb") as stream: + raw = stream.read(maximum + 1) + if not raw or len(raw) > maximum: + raise ValueError("INPUT_INVALID") + return raw + + +def _identity(path: Path): + value = path.stat() + return value.st_dev, value.st_ino, value.st_mode, value.st_size, value.st_mtime_ns + + +def _authority(docker: Path, docker_config: Path): + if (not docker.is_absolute() or docker.is_symlink() or not docker.is_file() + or not docker_config.is_absolute() or docker_config.is_symlink() + or not docker_config.is_dir() or any(docker_config.iterdir())): + raise ValueError("DOCKER_INVALID") + return (docker, docker_config, _identity(docker), _identity(docker_config)) + + +def _run(authority, args: tuple[str, ...], *, timeout: float = 15): + """Bound both CLI lifetime and combined bytes before they enter memory.""" + docker, config, exe_stat, config_stat = authority + if (_identity(docker) != exe_stat or _identity(config) != config_stat): + raise ValueError("DOCKER_CHANGED") + process = subprocess.Popen((str(docker), "--config", str(config), + "--host", "unix:///var/run/docker.sock", *args), + env={"PATH": os.defpath, "DOCKER_CONTENT_TRUST": "1"}, + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + start_new_session=True) + output = {process.stdout: bytearray(), process.stderr: bytearray()} + deadline = time.monotonic() + timeout + try: + with selectors.DefaultSelector() as selector: + for stream in output: + os.set_blocking(stream.fileno(), False) + selector.register(stream, selectors.EVENT_READ) + while selector.get_map() or process.poll() is None: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise ValueError("DOCKER_COMMAND_REJECTED") + for key, _ in selector.select(min(remaining, 0.1)): + chunk = os.read(key.fileobj.fileno(), _CLI_BUDGET + 1) + if not chunk: + selector.unregister(key.fileobj) + elif sum(map(len, output.values())) + len(chunk) > _CLI_BUDGET: + raise ValueError("DOCKER_COMMAND_REJECTED") + else: + output[key.fileobj].extend(chunk) + if process.wait(timeout=max(0.01, deadline - time.monotonic())) != 0: + raise ValueError("DOCKER_COMMAND_REJECTED") + if (_identity(docker) != exe_stat or _identity(config) != config_stat): + raise ValueError("DOCKER_CHANGED") + return subprocess.CompletedProcess(args, 0, bytes(output[process.stdout]), + bytes(output[process.stderr])) + finally: + if process.poll() is None: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + pass + for stream in output: + stream.close() + + +def _one_line(raw: bytes) -> str: + value = raw.decode("ascii") + if value != value.strip() and value != value.strip() + "\n": + raise ValueError("DIAGNOSTIC_OUTPUT_INVALID") + if not value.strip() or "\n" in value.strip(): + raise ValueError("DIAGNOSTIC_OUTPUT_INVALID") + return value.strip() + + +def _append(output_dir: Path, event: dict[str, object]) -> None: + with (output_dir / "events.jsonl").open("ab") as stream: + stream.write(json.dumps(event, sort_keys=True, separators=(",", ":")).encode("ascii") + b"\n") + stream.flush() + os.fsync(stream.fileno()) + + +def _source_identity(source_root: Path, source_lock: Path, expected_lock: str, + expected_manifest: str, expected_closure: str) -> dict[str, str]: + if (not source_root.is_absolute() or source_root.name != "engine" + or source_root.is_symlink() or not source_root.is_dir() + or not source_lock.is_absolute() or not re.fullmatch(r"[0-9a-f]{64}", expected_lock) + or not re.fullmatch(r"[0-9a-f]{64}", expected_manifest) + or not re.fullmatch(r"[0-9a-f]{64}", expected_closure)): + raise ValueError("SOURCE_IDENTITY_INVALID") + lock_raw = _read(source_lock, 128 * 1024) + manifest = source_root.parent / "control/offline-sft-worker-v1.json" + manifest_raw = _read(manifest, 128 * 1024) + if (hashlib.sha256(lock_raw).hexdigest() != expected_lock + or hashlib.sha256(manifest_raw).hexdigest() != expected_manifest): + raise ValueError("SOURCE_IDENTITY_INVALID") + lock = json.loads(lock_raw) + if (lock.get("schema_version") != "synaptic-source-lock/v1" + or lock.get("sources", {}).get("engine", {}).get("commit") != + "0d515ae1ae235d0858802228ce4af8dba8fccd03"): + raise ValueError("SOURCE_IDENTITY_INVALID") + # This trusted, checked-in verifier hashes every mounted worker member. + # It is imported from this checkout, never from the staged source. + checkout = Path(__file__).resolve().parents[1] + sys.path.insert(0, str(checkout)) + from tuner.runtime import offline_sft_worker + if Path(offline_sft_worker.__file__).resolve() != ( + checkout / "tuner/runtime/offline_sft_worker.py").resolve(): + raise ValueError("SOURCE_IDENTITY_INVALID") + try: + offline_sft_worker.load_offline_sft_worker_closure( + manifest, expected_digest=expected_closure, engine_root=source_root) + except Exception: + raise ValueError("SOURCE_IDENTITY_INVALID") from None + return {"source_lock_sha256": expected_lock, "manifest_sha256": expected_manifest, + "closure_digest": expected_closure, + "engine_commit": "0d515ae1ae235d0858802228ce4af8dba8fccd03"} + + +def _trainer_prefix_boundary(raw: bytes) -> int: + """Validate the fixed marker sequence and return the first excluded line.""" + lines = raw.decode("utf-8").splitlines(keepends=True) + markers = tuple(f'_mark_packaged_runtime_phase("{name}")' for name in + ("BOOTSTRAP_ENV", "TORCH_IMPORT", "UNSLOTH_IMPORT")) + positions = [] + for marker in markers: + matches = [i for i, line in enumerate(lines) if line.strip() == marker] + if len(matches) != 1: + raise ValueError("SOURCE_IDENTITY_INVALID") + positions.append(matches[0]) + if (not positions[0] < positions[1] < positions[2] + or lines[positions[1] + 1].strip() != "import torch # noqa: E402" + or lines[positions[2] + 1].strip() != + "from unsloth import is_bfloat16_supported # noqa: E402"): + raise ValueError("SOURCE_IDENTITY_INVALID") + prefix = "".join(lines[:positions[2]]) + tree = ast.parse(prefix) + if (not isinstance(tree.body[-1], ast.Import) + or [alias.name for alias in tree.body[-1].names] != ["torch"] + or any(isinstance(node, ast.ImportFrom) and node.module is not None and + node.module.split(".")[0] in {"unsloth", "transformers", "trl"} + for node in ast.walk(tree)) + or any(isinstance(node, ast.Import) and + any(alias.name.split(".")[0] in {"unsloth", "transformers", "trl"} + for alias in node.names) for node in ast.walk(tree))): + raise ValueError("SOURCE_IDENTITY_INVALID") + return positions[2] + + +def _worker_context_identity(source_root: Path, expected_workload: str, + expected_runtime: str, expected_trainer: str, + expected_closure: str) -> dict[str, str]: + if any(not re.fullmatch(r"[0-9a-f]{64}", value) for value in + (expected_workload, expected_runtime, expected_trainer, expected_closure)): + raise ValueError("SOURCE_IDENTITY_INVALID") + workload_raw = _read(source_root.parent / "control/workload.json", 1024 * 1024) + runtime_raw = _read(source_root / "Trainers/sft/runtime_v1.py", 1024 * 1024) + trainer_raw = _read(source_root / "Trainers/sft/train_sft.py", 1024 * 1024) + worker_raw = _read(source_root / "tuner/runtime/offline_sft_worker.py", 1024 * 1024) + if (hashlib.sha256(workload_raw).hexdigest() != expected_workload + or hashlib.sha256(runtime_raw).hexdigest() != expected_runtime + or hashlib.sha256(trainer_raw).hexdigest() != expected_trainer): + raise ValueError("SOURCE_IDENTITY_INVALID") + workload = json.loads(workload_raw) + runtime = workload["execution_source"]["runtime"] + planned = runtime["environment"]["variables"] + if (runtime["roots"] != _R2_ROOTS + or runtime["environment"]["clear_inherited"] is not True + or set(planned) != _R2_ENV_KEYS + or any(type(value) is not str for value in planned.values()) + or any(planned[_ROOT_ENV[name]] != path for name, path in _R2_ROOTS.items()) + or planned["PYTHONPATH"] != _R2_ROOTS["engine"] + or planned["HF_HUB_OFFLINE"] != "1" + or planned["TRANSFORMERS_OFFLINE"] != "1" + or runtime["interpreter"]["executable"] != "/opt/unsloth-venv/bin/python3"): + raise ValueError("SOURCE_IDENTITY_INVALID") + _trainer_prefix_boundary(trainer_raw) + model = workload["configuration"]["document"]["model"] + ref, revision = model["ref"], model["revision"] + if (type(ref) is not str or type(revision) is not str + or len(ref.split("/")) not in (1, 2) + or any(not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", part) + for part in ref.split("/")) + or not re.fullmatch(r"[0-9a-f]{40}(?:[0-9a-f]{24})?", revision)): + raise ValueError("SOURCE_IDENTITY_INVALID") + child_env = dict(planned) + child_env.pop("PYTHONPATH") + child_env.update({"SYNAPTIC_WORKER_CLOSURE_MANIFEST": + "/source/control/offline-sft-worker-v1.json"}) + child_env["SYNAPTIC_WORKER_CLOSURE_DIGEST"] = expected_closure + child_env.update({_ROOT_ENV[name]: value for name, value in _R2_ROOTS.items()}) + child_env.update({"SYNAPTIC_WORKLOAD_FINGERPRINT": hashlib.sha256( + b"synaptic-training-workload/v1\0" + workload_raw).hexdigest(), + "PYTHONNOUSERSITE": "1", "PYTHONSAFEPATH": "1", + "HF_HOME": _R2_ROOTS["cache"] + "/huggingface", + "TRANSFORMERS_CACHE": _R2_ROOTS["cache"] + "/transformers", + "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", + "SYNAPTIC_MODEL_SNAPSHOT": _R2_ROOTS["cache"] + "/model/models--" + + "--".join(ref.split("/")) + "/snapshots/" + revision, + "WANDB_DISABLED": "true"}) + child_env_lc_ctype = dict(child_env) + child_env_lc_ctype["LC_CTYPE"] = "C.UTF-8" + return {"workload_sha256": expected_workload, + "runtime_sha256": expected_runtime, + "trainer_sha256": expected_trainer, + "worker_sha256": hashlib.sha256(worker_raw).hexdigest(), + "child_env_sha256": hashlib.sha256(json.dumps( + child_env, sort_keys=True, separators=(",", ":")).encode()).hexdigest(), + "child_env_lc_ctype_sha256": hashlib.sha256(json.dumps( + child_env_lc_ctype, sort_keys=True, separators=(",", ":")).encode()).hexdigest()} + + +def _worker_probe(identity: dict[str, str], closure: str, *, entry_mode: bool = False) -> str: + child = (_WORKER_ENTRY_CHILD if entry_mode else _WORKER_PREFIX_CHILD) + child = child.replace("EXPECTED_TRAINER_SHA256", identity["trainer_sha256"]) + child = child.replace("EXPECTED_SUPPLIED_ENV_SHA256", + identity["child_env_sha256"]) + if entry_mode: + child = child.replace("EXPECTED_LC_ENV_SHA256", + identity["child_env_lc_ctype_sha256"]) + child = child.replace("EXPECTED_WORKER_SHA256", identity["worker_sha256"]) + child = child.replace("DIAGNOSTIC_ENGINE_ROOT", "/source/engine") + child = child.replace("DIAGNOSTIC_STDERR_PATH", + "/artifacts/run/tracking/worker-entry.stderr") + probe = _WORKER_CONTEXT_PROBE + replacements = {"EXPECTED_WORKLOAD_SHA256": identity["workload_sha256"], + "EXPECTED_CLOSURE_DIGEST": closure, + "EXPECTED_RUNTIME_SHA256": identity["runtime_sha256"], + "EXPECTED_ROOTS": repr(_R2_ROOTS), + "EXPECTED_ENV_KEYS": repr(sorted(_R2_ENV_KEYS)), + "ROOT_BINDINGS": repr({_ROOT_ENV[name]: value + for name, value in _R2_ROOTS.items()}), + "CHILD_SOURCE_B64": base64.b64encode(child.encode()).decode()} + for key, value in replacements.items(): + if probe.count(key) != 1: + raise ValueError("SOURCE_IDENTITY_INVALID") + probe = probe.replace(key, value) + probe = probe.replace("ENTRY_MODE", "True" if entry_mode else "False") + return probe + + +def _path_custody(path: Path) -> tuple[tuple[int, int, int], ...]: + if not path.is_absolute(): + raise ValueError("SOURCE_IDENTITY_INVALID") + observed = [] + for member in (path, *path.parents): + info = member.lstat() + if stat.S_ISLNK(info.st_mode): + raise ValueError("SOURCE_IDENTITY_INVALID") + observed.append((info.st_dev, info.st_ino, info.st_mode)) + return tuple(observed) + + +def _require_daemon(authority, expected: str, *, timeout: float = 5) -> None: + result = _run(authority, ("info", "--format", "{{.ID}}"), timeout=timeout) + if result.stderr or _one_line(result.stdout) != expected: + raise ValueError("DAEMON_INVALID") + + +def _inspect_owned(authority, identifier: str, *, name: str, label: str, image: str, + daemon_id: str | None = None, timeout: float = 15): + if not _CID.fullmatch(identifier): + raise ValueError("CONTAINER_ID_INVALID") + deadline = time.monotonic() + timeout + if daemon_id is not None: + _require_daemon(authority, daemon_id, timeout=max(0.05, min(5, timeout / 2))) + result = _run(authority, ("container", "inspect", "--format", _INSPECT, identifier), + timeout=max(0.05, deadline - time.monotonic())) + if result.stderr: + raise ValueError("CONTAINER_STATE_INVALID") + rows = result.stdout.decode("ascii").splitlines() + if len(rows) != 6: + raise ValueError("CONTAINER_STATE_INVALID") + cid, actual_name, actual_image, labels, status, code = map(json.loads, rows) + if (cid != identifier or actual_name != "/" + name or actual_image != image + or type(labels) is not dict or labels.get("ai.synapticlabs.torch-import.claim") != label + or status not in {"created", "running", "exited", "dead"} or type(code) is not int): + raise ValueError("CONTAINER_STATE_INVALID") + return status, code + + +def _require_gpu_zero(authority, identifier: str) -> None: + """Confirm the fixed device request before any GPU-exposed code starts.""" + result = _run(authority, ("container", "inspect", "--format", + "{{json .HostConfig.DeviceRequests}}", identifier)) + if result.stderr: + raise ValueError("GPU_SCOPE_INVALID") + try: + requests = json.loads(_one_line(result.stdout)) + except Exception: + raise ValueError("GPU_SCOPE_INVALID") from None + if (type(requests) is not list or len(requests) != 1 + or type(requests[0]) is not dict + or requests[0].get("DeviceIDs") != ["0"] + or requests[0].get("Capabilities") != [["gpu"]] + or requests[0].get("Count") != 0 + or requests[0].get("Driver") != "" + or requests[0].get("Options") != {}): + raise ValueError("GPU_SCOPE_INVALID") + + +def _stop_once(authority, identifier: str, *, name: str, label: str, + image: str, daemon_id: str, output_dir: Path) -> bool: + """One stop request; only a separate exact inspection can prove terminal state.""" + evidence_retained = True + + def record(phase: str) -> None: + nonlocal evidence_retained + try: + _append(output_dir, {"phase": phase, "container_id": identifier}) + except Exception: + # Storage failure cannot suppress cleanup of the already claimed + # container. It does prevent claiming durable terminal evidence. + evidence_retained = False + + try: + status, _ = _inspect_owned(authority, identifier, name=name, label=label, + image=image, daemon_id=daemon_id) + if status in {"exited", "dead"}: + record("TERMINAL_OBSERVED") + return evidence_retained + if status != "running": + return False + record("STOP_CLAIMED") + try: + _run(authority, ("container", "stop", "--timeout", "2", identifier), timeout=10) + except Exception: + # A lost CLI acknowledgement does not establish daemon state. + # Inspect independently without issuing a second stop. + pass + status, _ = _inspect_owned(authority, identifier, name=name, label=label, + image=image, daemon_id=daemon_id) + if status in {"exited", "dead"}: + record("TERMINAL_OBSERVED") + return evidence_retained + except Exception: + pass + record("STOP_UNCONFIRMED") + return False + + +def diagnose(*, docker: Path, docker_config: Path, release_path: Path, + daemon_id: str, output_dir: Path, expected_release_sha256: str, + source_root: Path | None = None, source_lock: Path | None = None, + expected_source_lock_sha256: str | None = None, + expected_manifest_sha256: str | None = None, + expected_closure_digest: str | None = None, + gpu_device: str | None = None, worker_context: bool = False, + worker_entry: bool = False, + expected_workload_sha256: str | None = None, + expected_runtime_sha256: str | None = None, + expected_trainer_sha256: str | None = None) -> dict[str, object]: + worker_values = (expected_workload_sha256, expected_runtime_sha256, + expected_trainer_sha256) + if (worker_entry and not worker_context) or ((worker_context and (source_root is None or any(value is None for value in worker_values))) + or (not worker_context and any(value is not None for value in worker_values))): + raise ValueError("SOURCE_IDENTITY_INVALID") + if gpu_device not in (None, "0") or (gpu_device is not None and source_root is None): + raise ValueError("GPU_SCOPE_INVALID") + if not re.fullmatch(r"[0-9a-f-]{36}", daemon_id): + raise ValueError("DAEMON_ID_INVALID") + release_raw = _read(release_path, 128 * 1024) + if (not re.fullmatch(r"[0-9a-f]{64}", expected_release_sha256) + or hashlib.sha256(release_raw).hexdigest() != expected_release_sha256): + raise ValueError("RELEASE_HASH_INVALID") + release = json.loads(release_raw) + if (type(release) is not dict or + json.dumps(release, sort_keys=True, separators=(",", ":"), + ensure_ascii=True).encode("ascii") != release_raw): + raise ValueError("RELEASE_INVALID") + image = "sha256:" + release["image"]["digest"] + python = release["python"]["executable"] + digest = release["manifest_digest"] + if (not _SHA.fullmatch(image) or type(python) is not str or not python.startswith("/") + or type(digest) is not str or not re.fullmatch(r"[0-9a-f]{64}", digest) + or release["image"]["reference"].split("@")[-1] != image): + raise ValueError("RELEASE_INVALID") + parent_info = output_dir.parent.stat() if output_dir.parent.is_dir() else None + if (output_dir.exists() or output_dir.is_symlink() or parent_info is None + or output_dir.parent.is_symlink() or parent_info.st_uid != os.getuid() + or stat.S_IMODE(parent_info.st_mode) != 0o700): + raise ValueError("OUTPUT_INVALID") + try: + _path_custody(output_dir.parent) + except Exception: + raise ValueError("OUTPUT_INVALID") from None + source_values = (source_lock, expected_source_lock_sha256, + expected_manifest_sha256, expected_closure_digest) + if (source_root is None and any(value is not None for value in source_values)) or ( + source_root is not None and any(value is None for value in source_values)): + raise ValueError("SOURCE_INVALID") + source_identity = ({} if source_root is None else _source_identity( + source_root, source_lock, expected_source_lock_sha256, + expected_manifest_sha256, expected_closure_digest)) + context_identity = ({} if not worker_context else _worker_context_identity( + source_root, *worker_values, expected_closure_digest)) + if worker_entry: + try: + workload = json.loads(_read(source_root.parent / "control/workload.json", 1024 * 1024)) + dataset_ref = workload["configuration"]["document"]["dataset"]["ref"] + except (KeyError, TypeError, ValueError): + raise ValueError("SOURCE_IDENTITY_INVALID") from None + if (type(dataset_ref) is not str or + not re.fullmatch(r"prepared://sha256/[0-9a-f]{64}", dataset_ref)): + raise ValueError("SOURCE_IDENTITY_INVALID") + custody = (() if source_root is None else tuple(_path_custody(path) for path in + (source_root, source_root.parent / "control/offline-sft-worker-v1.json", source_lock, + *((source_root.parent / "control/workload.json",) if worker_context else ())))) + authority = _authority(docker, docker_config) + info = _run(authority, ("info", "--format", "{{.ID}}")) + if info.stderr or _one_line(info.stdout) != daemon_id: + raise ValueError("DAEMON_INVALID") + before = _run(authority, ("image", "inspect", "--format", "{{.Id}}", image)) + if before.stderr or _one_line(before.stdout) != image: + raise ValueError("IMAGE_INVALID") + output_dir.mkdir(mode=0o700) + nonce = secrets.token_hex(16) + name = "synaptic-torch-import-" + nonce + probe = (_worker_probe(context_identity, expected_closure_digest, entry_mode=worker_entry) + if worker_context else (_PROBE if source_root is None else + _GUARDED_PROBE.replace("EXPECTED_CLOSURE_DIGEST", expected_closure_digest))) + if gpu_device is not None: + if not worker_context and probe.count(_HIDDEN_GPU_ENV) != 1: + raise ValueError("GPU_SCOPE_INVALID") + # The r2 isolated child did not inherit either visibility variable. + # The host device request is separately inspected before start. + if not worker_context: + probe = probe.replace(_HIDDEN_GPU_ENV, "") + claim = {"schema_version": SCHEMA, "daemon_id": daemon_id, "image_id": image, + "release_sha256": expected_release_sha256, "source_identity": source_identity, + "context_identity": context_identity, + "name": name, "scope": ("worker_entry" if worker_entry else "worker_context") if worker_context else + ("bare" if source_root is None else "guarded"), + "helper_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "probe_sha256": hashlib.sha256(probe.encode("utf-8")).hexdigest(), + "docker_file_identity": list(authority[2]), + "docker_config_identity": list(authority[3])} + if gpu_device is not None: + claim["gpu_device"] = gpu_device + claim_raw = json.dumps(claim, sort_keys=True, separators=(",", ":")).encode("ascii") + label = hashlib.sha256(claim_raw).hexdigest() + with (output_dir / "claim.json").open("xb") as stream: + stream.write(claim_raw + b"\n") + stream.flush() + os.fsync(stream.fileno()) + _append(output_dir, {"phase": "CREATE_CLAIMED", "claim_sha256": label}) + mount = (() if source_root is None else + ("--mount", f"type=bind,src={source_root.parent},dst=/source,readonly")) + user = (() if source_root is None else + ("--user", f"{source_root.stat().st_uid}:{source_root.stat().st_gid}")) + visibility = (("--env=NVIDIA_VISIBLE_DEVICES=void", "--env=CUDA_VISIBLE_DEVICES=") + if gpu_device is None else ("--gpus", "device=0")) + artifact_tmpfs = (("--tmpfs", "/artifacts:rw,nosuid,nodev,size=64m,mode=1777") + if worker_context else ()) + command = ( + "container", "create", "--name", name, "--label", "ai.synapticlabs.torch-import.claim=" + label, + "--pull=never", "--network=none", "--log-driver", "json-file", + "--log-opt", "max-size=1m", "--log-opt", "max-file=1", + "--read-only", "--cap-drop=ALL", "--runtime=runc", + "--security-opt=no-new-privileges", "--pids-limit=64", "--memory=4g", + "--cpus=2", "--tmpfs", "/tmp:rw,nosuid,nodev,size=64m,mode=1777", + *artifact_tmpfs, + *mount, *user, + *visibility, + "--workdir=/tmp", "--entrypoint", python, + image, "-I", "-c", probe, + ) + if source_root is not None and tuple(_path_custody(path) for path in + (source_root, source_root.parent / "control/offline-sft-worker-v1.json", + source_lock, *((source_root.parent / "control/workload.json",) + if worker_context else ()))) != custody: + raise ValueError("SOURCE_IDENTITY_INVALID") + _require_daemon(authority, daemon_id) + try: + created = _run(authority, command) + except Exception: + _append(output_dir, {"phase": "CREATE_UNCERTAIN"}) + raise ValueError("CREATE_UNCERTAIN") from None + if created.stderr: + _append(output_dir, {"phase": "CREATE_UNCERTAIN"}) + raise ValueError("CREATE_UNCERTAIN") + cid = _one_line(created.stdout) + if not _CID.fullmatch(cid): + _append(output_dir, {"phase": "CREATE_UNCERTAIN"}) + raise ValueError("CREATE_UNCERTAIN") + _append(output_dir, {"phase": "CREATED", "container_id": cid}) + if _inspect_owned(authority, cid, name=name, label=label, image=image, + daemon_id=daemon_id)[0] != "created": + raise ValueError("CONTAINER_STATE_INVALID") + if gpu_device is not None: + _require_gpu_zero(authority, cid) + # Recheck the exact source bytes immediately before starting mounted code. + if source_root is not None and _source_identity(source_root, source_lock, + expected_source_lock_sha256, expected_manifest_sha256, + expected_closure_digest) != source_identity: + raise ValueError("SOURCE_IDENTITY_INVALID") + if worker_context and _worker_context_identity( + source_root, *worker_values, expected_closure_digest) != context_identity: + raise ValueError("SOURCE_IDENTITY_INVALID") + if source_root is not None and tuple(_path_custody(path) for path in + (source_root, source_root.parent / "control/offline-sft-worker-v1.json", + source_lock, *((source_root.parent / "control/workload.json",) + if worker_context else ()))) != custody: + raise ValueError("SOURCE_IDENTITY_INVALID") + deadline = time.monotonic() + _RUN_SECONDS + _append(output_dir, {"phase": "START_CLAIMED", "container_id": cid}) + cleanup_attempted = False + start_acknowledged = False + try: + started = _run(authority, ("container", "start", cid), timeout=15) + if started.stderr or _one_line(started.stdout) != cid: + raise ValueError("START_UNCERTAIN") + start_acknowledged = True + _append(output_dir, {"phase": "STARTED", "container_id": cid}) + while True: + if time.monotonic() >= deadline: + cleanup_attempted = True + confirmed = _stop_once(authority, cid, name=name, label=label, + image=image, daemon_id=daemon_id, output_dir=output_dir) + raise ValueError("DEADLINE_STOPPED" if confirmed else "DEADLINE_UNCERTAIN") + status, exit_code = _inspect_owned(authority, cid, name=name, label=label, + image=image, daemon_id=daemon_id, + timeout=max(0.1, min(5, deadline - time.monotonic()))) + if status in {"exited", "dead"}: + _append(output_dir, {"phase": "TERMINAL_OBSERVED", "container_id": cid, + "exit_code": exit_code}) + break + if time.monotonic() >= deadline: + cleanup_attempted = True + confirmed = _stop_once(authority, cid, name=name, label=label, + image=image, daemon_id=daemon_id, output_dir=output_dir) + raise ValueError("DEADLINE_STOPPED" if confirmed else "DEADLINE_UNCERTAIN") + time.sleep(min(0.5, max(0, deadline - time.monotonic()))) + except BaseException as error: + if not cleanup_attempted: + cleanup_attempted = True + confirmed = _stop_once(authority, cid, name=name, label=label, + image=image, daemon_id=daemon_id, output_dir=output_dir) + if not confirmed: + raise ValueError("DEADLINE_UNCERTAIN") from None + if isinstance(error, KeyboardInterrupt): + raise + if isinstance(error, ValueError) and error.args and error.args[0] in {"DEADLINE_UNCERTAIN", "DEADLINE_STOPPED"}: + raise + raise ValueError("DEADLINE_UNCERTAIN" if start_acknowledged else "START_UNCERTAIN") from None + if exit_code != 0: + raise ValueError("CHILD_OUTPUT_INVALID") + result = _run(authority, ("container", "logs", cid)) + if result.stderr: + raise ValueError("CHILD_OUTPUT_INVALID") + child = json.loads(_one_line(result.stdout)) + fields = ({"schema_version", "category", "stage", "context", "boundary"} + if worker_context else ({"schema_version", "category"} if source_root is None + else {"schema_version", "category", "stage"})) + if (type(child) is not dict or set(child) != fields + or child["schema_version"] != "synaptic-docker-torch-import-child/v1" + or child["category"] not in _CATEGORIES + or (worker_context and (child["boundary"] != "BEFORE_UNSLOTH" + or child["stage"] not in {"READ_WORKLOAD", "ISOLATED_CHILD", + "WORKER_IMPORT", "WORKER_CLOSURE", "WORKER_GUARD", + "TRAINER_PREFIX", "BOOTSTRAP_ENV", "TORCH_IMPORT", + "ENTRY_ADMISSION", "WORKER_MAIN", "WORKER_ORIGIN", + "WORKER_ARGUMENTS", "WORKER_POSTCHECK"} + or type(child["context"]) is not dict + or (child["category"] == "PASS" and + (child["stage"] != "TORCH_IMPORT" or + set(child["context"]) != ({"environment_supplied_sha256", + "environment_observed_sha256", "startup_env_delta", + "python_isolated", "cwd_sha256", "file_sha256", "sys_path_sha256"} + | ({"argv_sha256", "runpy_main_bound", + "environment_before_torch_sha256", + "environment_after_torch_sha256"} if worker_entry else set())) + or child["context"]["python_isolated"] is not True + or (worker_entry and child["context"]["runpy_main_bound"] is not True) + or child["context"]["startup_env_delta"] not in + {"NONE", "LC_CTYPE_C_UTF8"})) + or (child["category"] != "PASS" and child["context"] != {}) + or any(type(value) is not str or not re.fullmatch(r"[0-9a-f]{64}", value) + for key, value in child["context"].items() if key.endswith("_sha256")))) + or (source_root is not None and not worker_context and child["stage"] not in {"IMPORT_WORKER", "READ_MANIFEST", + "LOAD_CLOSURE", "VERIFY_ORIGINS", "INSTALL_GUARD", "BOOTSTRAP_ENV", "TORCH_IMPORT"})): + raise ValueError("CHILD_OUTPUT_INVALID") + if worker_context and child["category"] == "PASS": + context = child["context"] + expected_observed = (context_identity["child_env_sha256"] + if context["startup_env_delta"] == "NONE" + else context_identity["child_env_lc_ctype_sha256"]) + if (context["environment_supplied_sha256"] != context_identity["child_env_sha256"] + or context["environment_observed_sha256"] != expected_observed): + raise ValueError("CHILD_OUTPUT_INVALID") + if _inspect_owned(authority, cid, name=name, label=label, image=image, + daemon_id=daemon_id)[0] not in {"exited", "dead"}: + raise ValueError("CONTAINER_STATE_INVALID") + after = _run(authority, ("image", "inspect", "--format", "{{.Id}}", image)) + if after.stderr or _one_line(after.stdout) != image: + raise ValueError("IMAGE_CHANGED") + report = {"schema_version": SCHEMA, "category": child["category"], + "container_id": cid, "image_id": image, "daemon_id": daemon_id, + "runtime_release_digest": digest, + "training_executed": False, "gpu_used": None if gpu_device is not None else False, + "scope": ("worker_entry" if worker_entry else "worker_context") if worker_context else + ("bare" if source_root is None else "guarded"), + "claim_sha256": label, "source_identity": source_identity, + "context_identity": context_identity, + "terminal_observed": True} + if source_root is not None: + report["stage"] = child["stage"] + if worker_context: + report["context"] = child["context"] + report["boundary"] = child["boundary"] + if gpu_device is not None: + report["gpu_device"] = gpu_device + with (output_dir / "result.json").open("x", encoding="ascii") as stream: + json.dump(report, stream, sort_keys=True, separators=(",", ":")) + stream.write("\n") + return report + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + for name in ("docker", "docker-config", "release", "output-dir"): + parser.add_argument("--" + name, required=True, type=Path) + parser.add_argument("--daemon-id", required=True) + parser.add_argument("--expected-release-sha256", required=True) + parser.add_argument("--source-root", type=Path) + parser.add_argument("--source-lock", type=Path) + parser.add_argument("--expected-source-lock-sha256") + parser.add_argument("--expected-manifest-sha256") + parser.add_argument("--expected-closure-digest") + parser.add_argument("--gpu-device") + parser.add_argument("--worker-context", action="store_true") + parser.add_argument("--worker-entry", action="store_true") + parser.add_argument("--expected-workload-sha256") + parser.add_argument("--expected-runtime-sha256") + parser.add_argument("--expected-trainer-sha256") + args = parser.parse_args(argv) + try: + report = diagnose(docker=args.docker, docker_config=args.docker_config, + release_path=args.release, daemon_id=args.daemon_id, + output_dir=args.output_dir, + expected_release_sha256=args.expected_release_sha256, + source_root=args.source_root, source_lock=args.source_lock, + expected_source_lock_sha256=args.expected_source_lock_sha256, + expected_manifest_sha256=args.expected_manifest_sha256, + expected_closure_digest=args.expected_closure_digest, + gpu_device=args.gpu_device, worker_context=args.worker_context, + worker_entry=args.worker_entry, + expected_workload_sha256=args.expected_workload_sha256, + expected_runtime_sha256=args.expected_runtime_sha256, + expected_trainer_sha256=args.expected_trainer_sha256) + except BaseException as error: + reason = error.args[0] if type(error) is ValueError and len(error.args) == 1 else None + if type(reason) is not str or reason not in _REASONS: + reason = "OTHER" + print(json.dumps({"schema_version": SCHEMA, "status": "REJECTED", + "reason_code": reason}, sort_keys=True, separators=(",", ":")), file=sys.stderr) + return 2 + print(json.dumps(report, sort_keys=True, separators=(",", ":"))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/examples/test_docker_material_preparation.py b/tests/examples/test_docker_material_preparation.py index ff47087c..aec1c6da 100644 --- a/tests/examples/test_docker_material_preparation.py +++ b/tests/examples/test_docker_material_preparation.py @@ -2,8 +2,13 @@ """Genuine Git/publication/compiler and no-daemon Docker preparation controls.""" import importlib.util +import hashlib import json +import os from pathlib import Path +import sys +from types import SimpleNamespace +from contextlib import contextmanager import pytest from tests.examples.test_hf_material_preparation import configured as source_configured # noqa: F401 @@ -11,6 +16,9 @@ from tests.runtime.test_packaged_runtime_releases import _release from tuner.training.coordinator_material import CoordinatorResolvedMaterial from tuner.training import default_recipe_registry +from tuner.execution.providers.modal.model_snapshot import ( + ModelSnapshotPreparationError, prepare_model_snapshot, +) ROOT = Path(__file__).resolve().parents[2] spec = importlib.util.spec_from_file_location( @@ -183,6 +191,218 @@ def model_preparer(**kwargs): assert len(calls) == 1 +def test_model_first_claim_precedes_source_and_receipt_binds_fresh_prepare(configured): + c, verifier, auth, clock, _ = configured + c["model_output_root"] = str(Path(c["output_root"]).parent / "model-attempt") + selected = c["consumer"]["docker"] + source = Path(selected["source_host_root"]) + artifact = Path(selected["artifact_host_root"]) + calls = [] + + def model_preparer(**kwargs): + assert not source.exists() + assert not Path(c["output_root"]).exists() + assert (Path(c["model_output_root"]) / "model-preparation-attempt.json").exists() + calls.append(kwargs) + target = (kwargs["destination_root"] / "model" + / ("models--" + kwargs["model_ref"].replace("/", "--")) + / "snapshots" / kwargs["revision"]) + target.mkdir(parents=True, mode=0o700) + (target / "synthetic-config.json").write_text('{"fixture":true}') + return target + + receipt = helper.prepare_model( + c, model_preparer=model_preparer, sdk_preflight=lambda: None, + ) + assert receipt["schema_version"] == "synaptic-docker-model-preparation/v1" + assert not source.exists() + assert len(calls) == 1 + with pytest.raises(ValueError): + helper.prepare_model(c, model_preparer=model_preparer, sdk_preflight=lambda: None) + assert len(calls) == 1 + member = (artifact / "run/cache/model" + / ("models--" + calls[0]["model_ref"].replace("/", "--")) + / "snapshots" / calls[0]["revision"] / "synthetic-config.json") + original = member.read_bytes() + member.write_bytes(b"changed before source") + with pytest.raises(ValueError): + helper.prepare(c, pushed_sources=verifier, authenticator=auth, clock=clock) + assert not Path(c["output_root"]).exists() + member.write_bytes(original) + helper.prepare(c, pushed_sources=verifier, authenticator=auth, clock=clock) + consumer = helper.launch.load_config(Path(c["output_root"]) / "consumer.json") + assert "model_preparation_file" in consumer["docker"] + assert not source.exists() + member.write_text("changed") + assert helper.docker_host.preview(consumer)["run"] + with pytest.raises(ValueError): + helper.docker_host._components(consumer, live_model_bytes=True) + + +def test_model_first_failure_consumes_only_fresh_model_claim(configured): + c, _, _, _, _ = configured + c["model_output_root"] = str(Path(c["output_root"]).parent / "model-attempt") + selected = c["consumer"]["docker"] + + def fail(**kwargs): + assert (Path(c["model_output_root"]) / "model-preparation-attempt.json").exists() + raise RuntimeError("private fixture failure") + + with pytest.raises(RuntimeError): + helper.prepare_model(c, model_preparer=fail, sdk_preflight=lambda: None) + assert not Path(c["output_root"]).exists() + assert not Path(selected["source_host_root"]).exists() + with pytest.raises(ValueError): + helper.prepare_model(c, model_preparer=fail, sdk_preflight=lambda: None) + + +def test_model_first_rejects_source_created_during_expensive_preparation(configured): + c, _, _, _, _ = configured + c["model_output_root"] = str(Path(c["output_root"]).parent / "model-attempt") + source = Path(c["consumer"]["docker"]["source_host_root"]) + + def drift(**kwargs): + source.mkdir(mode=0o700) + target = (kwargs["destination_root"] / "model" + / ("models--" + kwargs["model_ref"].replace("/", "--")) + / "snapshots" / kwargs["revision"]) + target.mkdir(parents=True, mode=0o700) + (target / "synthetic-config.json").write_text('{"fixture":true}') + return target + + with pytest.raises(ValueError, match="docker_source_changed"): + helper.prepare_model(c, model_preparer=drift, sdk_preflight=lambda: None) + assert (Path(c["model_output_root"]) / "model-preparation-attempt.json").exists() + assert not (Path(c["model_output_root"]) / "model-preparation.json").exists() + + +@pytest.mark.parametrize("seed_state", [ + "verified", "corrupt", "symlink", "revision_symlink", "mutated", + "absent_revision", "cold", +]) +def test_model_first_seed_is_independently_verified(configured, monkeypatch, seed_state): + c, _, _, _, _ = configured + c["model_output_root"] = str(Path(c["output_root"]).parent / "model-attempt") + model = json.loads(Path(c["resolved_config"]).read_bytes())["model"] + content = b'{"fixture":true}' + name = "synthetic-config.json" + seed = Path(c["output_root"]).parent / "retained-cache" + relative = (Path("models--" + model["ref"].replace("/", "--")) + / "snapshots" / model["revision"]) + if seed_state != "cold": + seed.mkdir(mode=0o700) + (seed / "old-attempt-claim.json").write_bytes(b"retained claim") + member = seed / relative + if seed_state == "revision_symlink": + member.parent.mkdir(parents=True, mode=0o700) + member.symlink_to(Path(c["resolved_config"]), target_is_directory=True) + elif seed_state != "absent_revision": + member.mkdir(parents=True, mode=0o700) + (member / name).write_bytes(content) + if seed_state == "corrupt": + (member / name).write_bytes(b"wrong fixture") + elif seed_state == "symlink": + (member / name).unlink() + (member / name).symlink_to(Path(c["resolved_config"])) + c["model_seed_root"] = str(seed) + calls = [] + + class API: + def __init__(self, *, endpoint, token): + pass + + def model_info(self, repo_id, *, revision, files_metadata, token): + assert (repo_id, revision) == (model["ref"], model["revision"]) + calls.append("metadata") + return SimpleNamespace( + sha=revision, + siblings=[SimpleNamespace( + rfilename=name, size=len(content), lfs=None, + blob_id=hashlib.sha1( + f"blob {len(content)}\0".encode() + content + ).hexdigest(), + )], + ) + + def download(**kwargs): + calls.append("download") + (kwargs["local_dir"] / name).write_bytes(content) + return str(kwargs["local_dir"]) + + monkeypatch.setitem(sys.modules, "huggingface_hub", SimpleNamespace( + HfApi=API, snapshot_download=download, + )) + if seed_state == "mutated": + original_copy = helper.copy_regular + + def mutate_after_copy(*args, **kwargs): + result = original_copy(*args, **kwargs) + (seed / relative / name).write_bytes(b"x" * len(content)) + return result + + monkeypatch.setattr(helper, "copy_regular", mutate_after_copy) + if seed_state in {"corrupt", "symlink", "revision_symlink", "mutated"}: + with pytest.raises((ValueError, ModelSnapshotPreparationError)): + helper.prepare_model( + c, model_preparer=prepare_model_snapshot, sdk_preflight=lambda: None, + ) + assert "download" not in calls + assert not Path(c["output_root"]).exists() + else: + receipt = helper.prepare_model( + c, model_preparer=prepare_model_snapshot, sdk_preflight=lambda: None, + ) + assert receipt["files"] == [{ + "path": name, "size_bytes": len(content), + "sha256": hashlib.sha256(content).hexdigest(), + }] + assert ("download" in calls) == (seed_state in {"cold", "absent_revision"}) + if seed_state == "verified": + assert (seed / relative / name).read_bytes() == content + if seed_state != "cold": + assert (seed / "old-attempt-claim.json").read_bytes() == b"retained claim" + assert Path(c["model_output_root"]).exists() + + +def test_model_inventory_hashes_held_scan_during_transient_ancestor_swap( + tmp_path, monkeypatch +): + root, alternate, held = (tmp_path / name for name in ("model", "alternate", "held")) + for directory, content in ((root, b"trusted"), (alternate, b"hostile")): + directory.mkdir(mode=0o700) + snapshot = directory / "snapshot" + snapshot.mkdir(mode=0o700) + (snapshot / "member.bin").write_bytes(content) + original_scandir = os.scandir + swapped = False + + @contextmanager + def swap_while_scanning(directory_fd): + nonlocal swapped + with original_scandir(directory_fd) as entries: + if not swapped: + swapped = True + os.rename(root, held) + os.rename(alternate, root) + try: + yield entries + finally: + os.rename(root, alternate) + os.rename(held, root) + else: + yield entries + + monkeypatch.setattr(helper.docker_host.os, "scandir", swap_while_scanning) + rows = helper.docker_host.model_inventory(root, root / "snapshot") + assert swapped + assert rows == [{ + "path": "member.bin", "size_bytes": len(b"trusted"), + "sha256": hashlib.sha256(b"trusted").hexdigest(), + }] + assert (root / "snapshot/member.bin").read_bytes() == b"trusted" + assert (alternate / "snapshot/member.bin").read_bytes() == b"hostile" + + @pytest.mark.parametrize("kind", ["source_member", "artifact_symlink", "artifact_mode"]) def test_hostile_model_roots_refuse_before_model_calls(configured, kind): c, verifier, auth, clock, _ = configured diff --git a/tests/examples/test_docker_training_qualification.py b/tests/examples/test_docker_training_qualification.py index 14bd532f..ff8a8000 100644 --- a/tests/examples/test_docker_training_qualification.py +++ b/tests/examples/test_docker_training_qualification.py @@ -304,6 +304,100 @@ def no_resolution(*args, **kwargs): assert run.to_dict() == preview["run"] +def bind_tiny_model_receipt(config, kwargs): + _, rich, _, _, _ = docker_host._components(config) + model = rich.resolved_config.to_dict()["model"] + artifact = kwargs["artifact_root"] + snapshot = (artifact / "run/cache/model" + / ("models--" + model["ref"].replace("/", "--")) + / "snapshots" / model["revision"]) + current = artifact + for part in snapshot.relative_to(artifact).parts: + current /= part + current.mkdir(mode=0o700) + member = snapshot / "synthetic-config.json" + member.write_bytes(b'{"fixture":true}') + receipt = { + "schema_version": "synaptic-docker-model-preparation/v1", + "model": model, + "configuration_sha256": hashlib.sha256( + rich.resolved_config.canonical_json.encode() + ).hexdigest(), + "runtime_declaration_sha256": hashlib.sha256( + rich.execution_context.canonical_json.encode() + ).hexdigest(), + "artifact_host_root": str(artifact), + "snapshot": str(snapshot), + "files": docker_host.model_inventory(artifact, snapshot), + } + receipt_path = artifact.parent / "model-preparation.json" + raw = json.dumps(receipt, sort_keys=True).encode() + config["docker"]["model_preparation_file"] = committed(receipt_path, raw) + return artifact, snapshot, member, receipt_path, raw + + +def test_model_first_public_start_rejects_changed_bytes_before_create(configured): + config, kwargs, commands, _, _, options = configured + _, _, member, _, _ = bind_tiny_model_receipt(config, kwargs) + member.write_bytes(b"changed bytes") + assert launch.preview(config)["run"] + with pytest.raises(ValueError): + launch.launch(config, adapter_options=options) + assert not [c for c in commands if c[:2] == ("container", "create")] + assert Path(config["state_root"]).exists() + with pytest.raises(FileExistsError): + launch.launch(config, adapter_options=options) + + +def test_model_first_public_start_and_cold_reads_without_weights( + configured, monkeypatch +): + config, kwargs, commands, state, roles, options = configured + artifact, snapshot, member, receipt_path, raw = bind_tiny_model_receipt(config, kwargs) + assert launch.preview(config)["run"] + receipt_path.write_bytes(b"changed receipt") + with pytest.raises(ValueError): + launch.launch(config, adapter_options=options) + assert not [c for c in commands if c[:2] == ("container", "create")] + assert not Path(config["state_root"]).exists() + receipt_path.write_bytes(raw) + result = launch.launch(config, adapter_options=options) + assert result["start"]["accepted"] + assert len([c for c in commands if c[:2] == ("container", "create")]) == 1 + retained = launch.load_config(Path(config["state_root"]) / "config.json") + retained_receipt = Path(retained["docker"]["model_preparation_file"]["path"]) + assert retained_receipt != receipt_path and retained_receipt.read_bytes() == raw + receipt_path.unlink() + member.unlink() + snapshot.rmdir() + assert launch.preview(retained)["run"] == result["start"]["run"] + assert launch.known(retained, "observe", adapter_options=options)["state"] == "running" + state["status"] = "exited" + (artifact / "native").mkdir(mode=0o700) + values = native_values(kwargs, monkeypatch) + for role, relative in roles.items(): + (artifact / relative).write_bytes(values[role]) + assert launch.known(retained, "observe", adapter_options=options)["state"] == "succeeded" + assert launch.known(retained, "verify", adapter_options=options)["verified"] + output = Path(config["state_root"]) / "downloads" + downloaded = launch.known( + retained, "download", destination=output, adapter_options=options + ) + assert len(downloaded["paths"]) == 5 + for role in roles: + assert (output / (role + ".artifact")).read_bytes() == values[role] + wrong_run = json.loads(json.dumps(retained)) + wrong_run["run"]["run_id"] = "another-run" + with pytest.raises(ValueError): + launch.known(wrong_run, "observe", adapter_options=options) + retained_receipt.write_bytes(b"changed receipt") + with pytest.raises(ValueError): + launch.known(retained, "observe", adapter_options=options) + with pytest.raises(ValueError): + launch.preview(retained) + assert len([c for c in commands if c[:2] == ("container", "create")]) == 1 + + def test_unknown_submit_and_existing_attempt_never_create_again(configured): config, _, commands, state, _, options = configured state["unknown_create"] = True diff --git a/tests/scripts/test_diagnose_docker_torch_import.py b/tests/scripts/test_diagnose_docker_torch_import.py new file mode 100644 index 00000000..612ff6dc --- /dev/null +++ b/tests/scripts/test_diagnose_docker_torch_import.py @@ -0,0 +1,722 @@ +from __future__ import annotations + +import hashlib +import json +import base64 +import re +import shutil +import subprocess +import sys +import ast +from pathlib import Path + +import pytest + +from scripts import diagnose_docker_torch_import as diagnostic + + +DAEMON = "24805003-b125-469b-8080-9da4daacd364" +IMAGE = "sha256:" + "a" * 64 +CID = "b" * 64 +ROOT = Path(__file__).parents[2] + + +def _release(root: Path) -> tuple[Path, str]: + path = root / "release.json" + path.write_text(json.dumps({"image": {"digest": "a" * 64, + "reference": "example/image@" + IMAGE}, "python": {"executable": "/usr/bin/python3"}, + "manifest_digest": "c" * 64}, sort_keys=True, separators=(",", ":"))) + return path, hashlib.sha256(path.read_bytes()).hexdigest() + + +def _fake_docker(root: Path) -> tuple[Path, Path, Path]: + executable = root / "docker" + state = root / "state.json" + state.write_text(json.dumps({"mode": "normal", "status": "absent", "stops": 0})) + config = root / "config" + config.mkdir() + executable.write_text(f"#!{sys.executable}\n" + r'''import json,sys,time +from pathlib import Path +state_path=Path(__file__).with_name('state.json') +s=json.loads(state_path.read_text()) +a=sys.argv[sys.argv.index('--host')+2:] +def save(): state_path.write_text(json.dumps(s)) +if a[0]=='info': print('24805003-b125-469b-8080-9da4daacd364') +elif a[:2]==['image','inspect']: print('sha256:'+'a'*64) +elif a[:2]==['container','create']: + s['status']='created'; s['name']=a[a.index('--name')+1] + s['guarded']='--mount' in a + s['gpu_requested']='--gpus' in a and a[a.index('--gpus')+1]=='device=0' if '--gpus' in a else False + s['gpu_probe_unhidden']=s['gpu_requested'] and "'NVIDIA_VISIBLE_DEVICES':'void'" not in a[-1] and "'CUDA_VISIBLE_DEVICES':''" not in a[-1] + s['label']=a[a.index('--label')+1].split('=',1)[1]; save() + if s['mode']=='create_hang': time.sleep(10) + if s['mode']=='bad_id': print('invalid'); sys.exit(0) + print('b'*64) +elif a[:2]==['container','start']: + s['status']='running'; save() + if s['mode']=='start_hang': time.sleep(10) + if s['mode']=='start_error': sys.exit(7) + if s['mode']=='start_stdout_flood': sys.stdout.write('x'*20000); sys.exit(0) + if s['mode']=='start_stderr_flood': sys.stderr.write('x'*20000); sys.exit(0) + print('b'*64) +elif a[:2]==['container','inspect']: + if 'DeviceRequests' in a[a.index('--format')+1]: + if s['mode']=='wrong_gpu_request': print('null') + else: print(json.dumps([{'Capabilities':[['gpu']],'Count':0,'DeviceIDs':['0'],'Driver':'','Options':{}}] if s.get('gpu_requested') else None)) + sys.exit(0) + name='/' + s.get('name','missing') + if s['mode']=='wrong_identity' or (s['mode']=='wrong_running_identity' and s['status']=='running'): + name='/another-container' + for value in ('b'*64,name,'sha256:'+'a'*64, + {'ai.synapticlabs.torch-import.claim':s.get('label','')}, + s['status'],0): print(json.dumps(value)) +elif a[:2]==['container','stop']: + s['stops']+=1 + if s['mode']!='stop_unconfirmed': s['status']='exited' + save(); print('b'*64) +elif a[:2]==['container','logs']: + if s['mode']=='stdout_flood': sys.stdout.write('x'*20000) + elif s['mode']=='stderr_flood': sys.stderr.write('x'*20000) + elif s['mode']=='both_flood': + sys.stdout.write('x'*5000); sys.stderr.write('x'*5000) + elif s.get('guarded'): print('{"category":"PASS","schema_version":"synaptic-docker-torch-import-child/v1","stage":"TORCH_IMPORT"}') + else: print('{"category":"PASS","schema_version":"synaptic-docker-torch-import-child/v1"}') +else: sys.exit(8) +''') + executable.chmod(0o700) + return executable, config, state + + +def _call(root: Path, executable: Path, config: Path, *, output: str = "out", **extra): + release, sha = _release(root) + return diagnostic.diagnose(docker=executable, docker_config=config, + release_path=release, daemon_id=DAEMON, output_dir=root / output, + expected_release_sha256=sha, **extra) + + +def _state(path: Path, **changes): + value = json.loads(path.read_text()) + value.update(changes) + path.write_text(json.dumps(value)) + return value + + +def _stage_source(root: Path): + source = root / "source" + engine = source / "engine" + document = json.loads((ROOT / "tuner/runtime/manifests/offline-sft-worker-v1.json").read_text()) + for member in document["members"]: + target = engine / member["path"] + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(ROOT / member["path"], target) + (source / "control").mkdir() + manifest = source / "control/offline-sft-worker-v1.json" + shutil.copy2(ROOT / "tuner/runtime/manifests/offline-sft-worker-v1.json", manifest) + lock = root / "source-lock.json" + lock.write_text(json.dumps({"schema_version": "synaptic-source-lock/v1", "sources": { + "engine": {"commit": "0d515ae1ae235d0858802228ce4af8dba8fccd03"}}})) + flags = {"source_root": engine, "source_lock": lock, + "expected_source_lock_sha256": hashlib.sha256(lock.read_bytes()).hexdigest(), + "expected_manifest_sha256": hashlib.sha256(manifest.read_bytes()).hexdigest(), + "expected_closure_digest": document["closure_digest"]} + return flags, document, manifest + + +def _stage_worker_context(root: Path): + flags, _document, _manifest = _stage_source(root) + engine = flags["source_root"] + variables = {name: "1" for name in diagnostic._R2_ENV_KEYS} + variables.update({diagnostic._ROOT_ENV[name]: value + for name, value in diagnostic._R2_ROOTS.items()}) + variables.update({"PYTHONPATH": "/source/engine", + "PATH": "/opt/unsloth-venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "HF_HOME": "/artifacts/run/cache/huggingface", + "TRANSFORMERS_CACHE": "/artifacts/run/cache/transformers", + "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1"}) + workload = {"execution_source": {"runtime": { + "roots": diagnostic._R2_ROOTS, + "environment": {"clear_inherited": True, "variables": variables}, + "interpreter": {"executable": "/opt/unsloth-venv/bin/python3"}}}, + "configuration": {"document": {"model": { + "ref": "Example/Model", "revision": "a" * 40}}}} + path = engine.parent / "control/workload.json" + path.write_text(json.dumps(workload, sort_keys=True, separators=(",", ":"))) + flags.update({"worker_context": True, + "expected_workload_sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "expected_runtime_sha256": hashlib.sha256( + (engine / "Trainers/sft/runtime_v1.py").read_bytes()).hexdigest(), + "expected_trainer_sha256": hashlib.sha256( + (engine / "Trainers/sft/train_sft.py").read_bytes()).hexdigest()}) + return flags, path + + +def test_worker_prefix_excludes_unsloth_and_all_trainer_calls() -> None: + raw = (ROOT / "Trainers/sft/train_sft.py").read_bytes() + boundary = diagnostic._trainer_prefix_boundary(raw) + prefix = "".join(raw.decode().splitlines(keepends=True)[:boundary]) + tree = ast.parse(prefix) + assert isinstance(tree.body[-1], ast.Import) + assert [alias.name for alias in tree.body[-1].names] == ["torch"] + assert "UNSLOTH_IMPORT" not in prefix + assert "from unsloth import" not in prefix + assert "TRAIN_CALL" not in prefix + assert "MODEL_SNAPSHOT" not in prefix + assert "if __name__" not in prefix + for old, new in (('"TORCH_IMPORT"', '"UNSLOTH_IMPORT"'), + ('import torch # noqa: E402', 'from unsloth import x')): + with pytest.raises(ValueError, match="SOURCE_IDENTITY_INVALID"): + diagnostic._trainer_prefix_boundary(raw.replace(old.encode(), new.encode(), 1)) + + +def test_worker_context_source_and_workload_drift_reject_before_effect(tmp_path: Path, + monkeypatch) -> None: + flags, workload = _stage_worker_context(tmp_path) + docker, config, state = _fake_docker(tmp_path) + monkeypatch.setattr(diagnostic, "_run", lambda *args, **kwargs: pytest.fail("Docker touched")) + workload.write_bytes(workload.read_bytes() + b" ") + with pytest.raises(ValueError, match="SOURCE_IDENTITY_INVALID"): + _call(tmp_path, docker, config, **flags) + assert json.loads(state.read_text())["status"] == "absent" + assert not (tmp_path / "out").exists() + + +def test_worker_context_program_is_isolated_and_excludes_trainer_main(tmp_path: Path) -> None: + flags, _workload = _stage_worker_context(tmp_path) + identity = diagnostic._worker_context_identity( + flags["source_root"], flags["expected_workload_sha256"], + flags["expected_runtime_sha256"], flags["expected_trainer_sha256"], + flags["expected_closure_digest"]) + probe = diagnostic._worker_probe(identity, flags["expected_closure_digest"]) + ast.parse(probe) + ast.parse(diagnostic._WORKER_PREFIX_CHILD.replace( + "EXPECTED_TRAINER_SHA256", flags["expected_trainer_sha256"]).replace( + "EXPECTED_SUPPLIED_ENV_SHA256", identity["child_env_sha256"])) + assert "EXPECTED_" not in probe + assert "CHILD_SOURCE_B64" not in probe + assert "process=subprocess.Popen([sys.executable,'-I','-c'" in probe + assert "runpy.run_path(str(root/'tuner/runtime/offline_sft_worker.py')" in diagnostic._WORKER_PREFIX_CHILD + assert "runpy.run_path(str(trainer)" not in diagnostic._WORKER_PREFIX_CHILD + assert "exec(compile(tree,str(trainer),'exec'),namespace)" in diagnostic._WORKER_PREFIX_CHILD + assert "lines[:positions[2]]" in diagnostic._WORKER_PREFIX_CHILD + assert identity["child_env_sha256"] != hashlib.sha256(b"").hexdigest() + assert identity["child_env_lc_ctype_sha256"] != identity["child_env_sha256"] + + +def _entry_fixture(tmp_path: Path, *, disable_trace: bool = False, + import_error: bool = False): + """Use real isolated runpy and a non-ML Torch stub at the stop boundary.""" + engine = tmp_path / "engine" + trainer = engine / "Trainers/sft/train_sft.py" + trainer.parent.mkdir(parents=True) + worker = engine / "tuner/runtime/offline_sft_worker.py" + worker.parent.mkdir(parents=True) + worker.write_text("""import runpy,sys +from pathlib import Path +def mark(value): + pass +def main(): + if sys.argv[1:]!=['--','--max-steps','1','--no-load-in-4bit']: + print('OFFLINE_SFT_WORKER_REJECTED phase=ARGUMENTS category=REJECTION',file=sys.stderr) + return 2 + root=Path(__file__).parents[2] + trainer=root/'Trainers/sft/train_sft.py' + sys.path[:0]=[str(trainer.parent),str(root)] + sys.argv=[str(trainer),*sys.argv[2:]] + try: + runpy.run_path(str(trainer),run_name='__main__',init_globals={'__packaged_runtime_phase__':mark}) + except ImportError: + print('OFFLINE_SFT_WORKER_REJECTED phase=TRAINER category=IMPORT milestone=TORCH_IMPORT',file=sys.stderr) + return 2 + except BaseException: + print('OFFLINE_SFT_WORKER_REJECTED phase=TRAINER category=OTHER milestone=TORCH_IMPORT',file=sys.stderr) + return 2 + return 0 +if __name__=='__main__': + raise SystemExit(main()) +""") + trainer.write_text("""import os,sys +from pathlib import Path +def _mark_packaged_runtime_phase(value): + marker=globals().get('__packaged_runtime_phase__') + if marker is not None: marker(value) +assert __name__=='__main__' +assert sys.modules['__main__'].__dict__ is globals() +_mark_packaged_runtime_phase("BOOTSTRAP_ENV") +os.environ['TORCH_COMPILE_DISABLE']='1' +_mark_packaged_runtime_phase("TORCH_IMPORT") +import torch # noqa: E402 +_mark_packaged_runtime_phase("UNSLOTH_IMPORT") +from unsloth import is_bfloat16_supported # noqa: E402 +Path(__file__).with_name('TRAINER_FELL_THROUGH').write_text('bad') +""") + (trainer.parent / "torch.py").write_text( + "from pathlib import Path\nimport sys\n" + + ("import types\nsys.settrace(None)\n" + "sys.modules['unsloth']=types.ModuleType('unsloth')\n" + "sys.modules['unsloth'].is_bfloat16_supported=True\n" + if disable_trace else "") + + "Path(__file__).with_name('TORCH_IMPORTED').write_text('yes')\n" + + ("raise ImportError('private detail')\n" if import_error else "")) + (trainer.parent / "unsloth.py").write_text( + "from pathlib import Path\n" + "Path(__file__).with_name('UNSLOTH_IMPORTED').write_text('bad')\n" + "is_bfloat16_supported=True\n") + environment = {"PATH": "/usr/bin:/bin", "PYTHONNOUSERSITE": "1", + "PYTHONSAFEPATH": "1", "PYTHONDONTWRITEBYTECODE": "1"} + observed_locale = {**environment, "LC_CTYPE": "C.UTF-8"} + arguments = ["--", "--max-steps", "1", "--no-load-in-4bit"] + digest = lambda obj: hashlib.sha256(json.dumps( + obj, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + source = diagnostic._WORKER_ENTRY_CHILD + for key, value in { + "DIAGNOSTIC_ENGINE_ROOT": str(engine), + "DIAGNOSTIC_STDERR_PATH": str(tmp_path / "worker-entry.stderr"), + "EXPECTED_TRAINER_SHA256": hashlib.sha256(trainer.read_bytes()).hexdigest(), + "EXPECTED_WORKER_SHA256": hashlib.sha256(worker.read_bytes()).hexdigest(), + "EXPECTED_SUPPLIED_ENV_SHA256": digest(environment), + "EXPECTED_LC_ENV_SHA256": digest(observed_locale), + "EXPECTED_TRAINER_ARGV_SHA256": digest([str(trainer), *arguments[1:]]), + }.items(): + source = source.replace(key, value) + return trainer, source, environment, arguments + + +@pytest.mark.parametrize("disable_trace", [False, True]) +def test_real_isolated_worker_entry_stops_before_unsloth(tmp_path: Path, + disable_trace: bool) -> None: + trainer, source, environment, arguments = _entry_fixture( + tmp_path, disable_trace=disable_trace) + with (tmp_path / "worker-entry.stderr").open("xb") as stderr: + completed = subprocess.run([sys.executable, "-I", "-c", source, *arguments], + cwd=tmp_path, env=environment, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=stderr, timeout=20) + assert completed.returncode == 0 + result = json.loads(completed.stdout) + assert result["boundary"] == "BEFORE_UNSLOTH" + if disable_trace: + assert result["category"] == "OTHER" + else: + assert result["category"] == "PASS" + assert result["stage"] == "TORCH_IMPORT" + assert result["context"]["runpy_main_bound"] is True + assert (result["context"]["environment_before_torch_sha256"] == + result["context"]["environment_after_torch_sha256"]) + assert (result["context"]["environment_before_torch_sha256"] != + result["context"]["environment_observed_sha256"]) + assert (trainer.parent / "TORCH_IMPORTED").exists() + assert not (trainer.parent / "UNSLOTH_IMPORTED").exists() + assert not (trainer.parent / "TRAINER_FELL_THROUGH").exists() + + +def test_real_isolated_worker_entry_reports_closed_torch_import_failure( + tmp_path: Path) -> None: + trainer, source, environment, arguments = _entry_fixture( + tmp_path, import_error=True) + with (tmp_path / "worker-entry.stderr").open("xb") as stderr: + completed = subprocess.run([sys.executable, "-I", "-c", source, *arguments], + cwd=tmp_path, env=environment, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=stderr, timeout=20) + assert completed.returncode == 0 + result = json.loads(completed.stdout) + assert result == {"schema_version": "synaptic-docker-torch-import-child/v1", + "category": "IMPORT", "stage": "TORCH_IMPORT", "context": {}, + "boundary": "BEFORE_UNSLOTH"} + assert (trainer.parent / "TORCH_IMPORTED").exists() + assert not (trainer.parent / "UNSLOTH_IMPORTED").exists() + + +def test_isolated_worker_entry_rejects_trainer_drift_before_worker( + tmp_path: Path) -> None: + trainer, source, environment, arguments = _entry_fixture(tmp_path) + trainer.write_bytes(trainer.read_bytes() + b"\n") + with (tmp_path / "worker-entry.stderr").open("xb") as stderr: + completed = subprocess.run([sys.executable, "-I", "-c", source, *arguments], + cwd=tmp_path, env=environment, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=stderr, timeout=20) + assert completed.returncode == 0 + result = json.loads(completed.stdout) + assert result["category"] == "OTHER" and result["stage"] == "ENTRY_ADMISSION" + assert result["context"] == {} + assert not (trainer.parent / "TORCH_IMPORTED").exists() + + +def test_worker_entry_source_and_mode_bind_before_docker(tmp_path: Path, + monkeypatch) -> None: + flags, workload_path = _stage_worker_context(tmp_path) + docker, config, state = _fake_docker(tmp_path) + monkeypatch.setattr(diagnostic, "_run", lambda *args, **kwargs: + pytest.fail("Docker touched")) + with pytest.raises(ValueError, match="SOURCE_IDENTITY_INVALID"): + _call(tmp_path, docker, config, worker_entry=True, **flags) + assert json.loads(state.read_text())["status"] == "absent" + workload = json.loads(workload_path.read_text()) + workload["configuration"]["document"]["dataset"] = { + "ref": "prepared://sha256/" + "a" * 64} + workload_path.write_text(json.dumps(workload, sort_keys=True, separators=(",", ":"))) + flags["expected_workload_sha256"] = hashlib.sha256(workload_path.read_bytes()).hexdigest() + identity = diagnostic._worker_context_identity( + flags["source_root"], flags["expected_workload_sha256"], + flags["expected_runtime_sha256"], flags["expected_trainer_sha256"], + flags["expected_closure_digest"]) + probe = diagnostic._worker_probe(identity, flags["expected_closure_digest"], + entry_mode=True) + ast.parse(probe) + assert "EXPECTED_" not in probe and "ENTRY_MODE" not in probe + match = re.search(r"base64\.b64decode\('([A-Za-z0-9+/=]+)'\)", probe) + assert match is not None + child = base64.b64decode(match.group(1)).decode() + ast.parse(child) + assert "DIAGNOSTIC_ENGINE_ROOT" not in child + assert "DIAGNOSTIC_STDERR_PATH" not in child + assert "runpy.run_path(str(worker),run_name='__main__')" in child + assert "sys.settrace(trace)" in child and "_UnslothBarrier" in child + trainer = flags["source_root"] / "Trainers/sft/train_sft.py" + trainer.write_bytes(trainer.read_bytes() + b"\n") + with pytest.raises(ValueError, match="SOURCE_IDENTITY_INVALID"): + _call(tmp_path, docker, config, worker_entry=True, **flags) + assert json.loads(state.read_text())["status"] == "absent" + assert not (tmp_path / "out").exists() + + +@pytest.mark.parametrize("mutation,expected", [ + ("none", "LC_CTYPE_C_UTF8"), + ("no_coercion", "NONE"), + ("extra", "REJECTED"), + ("changed", "REJECTED"), + ("locale_value", "REJECTED"), +]) +def test_real_isolated_python_startup_environment_evidence(mutation: str, + expected: str) -> None: + """Run the exact child classifier after a real -I interpreter startup.""" + source = diagnostic._WORKER_PREFIX_CHILD + tree = ast.parse(source) + function = next(node for node in tree.body + if isinstance(node, ast.FunctionDef) and + node.name == "_environment_evidence") + lines = source.splitlines(keepends=True) + classifier = "".join(lines[function.lineno - 1:function.end_lineno]) + program = ("import hashlib,json,os,sys\n" + classifier + + "\nsupplied=json.loads(sys.stdin.buffer.read(8193))\n" + + ("os.environ['UNEXPECTED_DIAGNOSTIC_KEY']='x'\n" if mutation == "extra" else "") + + ("os.environ['PATH']='/tampered'\n" if mutation == "changed" else "") + + ("os.environ['LC_CTYPE']='C.utf8'\n" if mutation == "locale_value" else "") + + "try:\n evidence=_environment_evidence(supplied,dict(os.environ))\n" + "except BaseException:\n print(json.dumps({'status':'REJECTED'}))\n" + "else:\n print(json.dumps({'status':'PASS','delta':evidence['startup_env_delta']," + "'supplied_sha256':evidence['environment_supplied_sha256']," + "'observed_sha256':evidence['environment_observed_sha256']}))\n") + supplied = {"PATH": "/usr/bin:/bin", "PYTHONNOUSERSITE": "1", + "PYTHONSAFEPATH": "1"} + if mutation == "no_coercion": + supplied["LC_ALL"] = "C" + result = subprocess.run([sys.executable, "-I", "-c", program], + input=json.dumps(supplied, sort_keys=True, separators=(",", ":")), + text=True, capture_output=True, env=supplied, timeout=5, check=True) + assert result.stderr == "" + report = json.loads(result.stdout) + if expected == "REJECTED": + assert report == {"status": "REJECTED"} + else: + assert report["status"] == "PASS" + assert report["delta"] == expected + expected_sha = hashlib.sha256(json.dumps( + supplied, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + assert report["supplied_sha256"] == expected_sha + assert (report["observed_sha256"] == expected_sha) == (expected == "NONE") + + +def test_worker_context_requires_independent_r2_hashes_before_docker(tmp_path: Path, + monkeypatch) -> None: + flags, _workload = _stage_worker_context(tmp_path) + flags.pop("expected_workload_sha256") + docker, config, state = _fake_docker(tmp_path) + monkeypatch.setattr(diagnostic, "_run", lambda *args, **kwargs: pytest.fail("Docker touched")) + with pytest.raises(ValueError, match="SOURCE_IDENTITY_INVALID"): + _call(tmp_path, docker, config, **flags) + assert json.loads(state.read_text())["status"] == "absent" + assert not (tmp_path / "out").exists() + + +def test_worker_context_rechecks_drift_after_create_without_start(tmp_path: Path, + monkeypatch) -> None: + flags, workload = _stage_worker_context(tmp_path) + docker, config, state = _fake_docker(tmp_path) + original = diagnostic._inspect_owned + changed = False + + def inspect(*args, **kwargs): + nonlocal changed + result = original(*args, **kwargs) + if not changed and result[0] == "created": + workload.write_bytes(workload.read_bytes() + b" ") + changed = True + return result + + monkeypatch.setattr(diagnostic, "_inspect_owned", inspect) + with pytest.raises(ValueError, match="SOURCE_IDENTITY_INVALID"): + _call(tmp_path, docker, config, **flags) + assert json.loads(state.read_text())["status"] == "created" + assert (tmp_path / "out/claim.json").exists() + assert not (tmp_path / "out/result.json").exists() + with pytest.raises(ValueError, match="OUTPUT_INVALID"): + _call(tmp_path, docker, config, **flags) + + +def test_real_cli_output_budget_and_timeout(tmp_path: Path) -> None: + docker, config, state = _fake_docker(tmp_path) + authority = diagnostic._authority(docker, config) + _state(state, mode="create_hang") + with pytest.raises(ValueError, match="DOCKER_COMMAND_REJECTED"): + diagnostic._run(authority, ("container", "create", "--name", "test-name", + "--label", "ai.synapticlabs.torch-import.claim=hash"), timeout=0.2) + # Killing the client did not stop the fake daemon's created container. + assert json.loads(state.read_text())["status"] == "created" + for mode in ("stdout_flood", "stderr_flood", "both_flood"): + _state(state, mode=mode) + with pytest.raises(ValueError, match="DOCKER_COMMAND_REJECTED"): + diagnostic._run(authority, ("container", "logs", CID), timeout=2) + + +def test_owned_lifecycle_and_closed_result(tmp_path: Path, monkeypatch) -> None: + docker, config, state = _fake_docker(tmp_path) + real = diagnostic._run + monkeypatch.setattr(diagnostic, "_run", lambda authority, args, *, timeout=15: + real(authority, args, timeout=min(timeout, 0.4))) + monkeypatch.setattr(diagnostic, "_RUN_SECONDS", 2) + # The fake daemon advances to terminal independently after start. + original_inspect = diagnostic._inspect_owned + def inspect(*args, **kwargs): + status, code = original_inspect(*args, **kwargs) + if status == "running": + _state(state, status="exited") + return status, code + monkeypatch.setattr(diagnostic, "_inspect_owned", inspect) + report = _call(tmp_path, docker, config) + assert report["category"] == "PASS" and report["terminal_observed"] is True + assert report["container_id"] == CID and report["source_identity"] == {} + assert json.loads((tmp_path / "out/result.json").read_text()) == report + assert json.loads(state.read_text())["stops"] == 0 + + +def test_guarded_happy_path_records_independent_source_identity(tmp_path: Path, monkeypatch) -> None: + flags, document, _manifest = _stage_source(tmp_path) + docker, config, state = _fake_docker(tmp_path) + monkeypatch.setattr(diagnostic, "_RUN_SECONDS", 2) + original_inspect = diagnostic._inspect_owned + def inspect(*args, **kwargs): + status, code = original_inspect(*args, **kwargs) + if status == "running": + _state(state, status="exited") + return status, code + monkeypatch.setattr(diagnostic, "_inspect_owned", inspect) + report = _call(tmp_path, docker, config, **flags) + assert report["scope"] == "guarded" and report["stage"] == "TORCH_IMPORT" + assert report["source_identity"]["closure_digest"] == document["closure_digest"] + assert report["source_identity"]["manifest_sha256"] == flags["expected_manifest_sha256"] + + +def test_gpu_visibility_is_fixed_guarded_import_only(tmp_path: Path, monkeypatch) -> None: + flags, _, _ = _stage_source(tmp_path) + docker, config, state = _fake_docker(tmp_path) + monkeypatch.setattr(diagnostic, "_RUN_SECONDS", 2) + original_inspect = diagnostic._inspect_owned + def inspect(*args, **kwargs): + status, code = original_inspect(*args, **kwargs) + if status == "running": + _state(state, status="exited") + return status, code + monkeypatch.setattr(diagnostic, "_inspect_owned", inspect) + report = _call(tmp_path, docker, config, gpu_device="0", **flags) + claim = json.loads((tmp_path / "out/claim.json").read_text()) + actual = json.loads(state.read_text()) + assert claim["gpu_device"] == "0" + assert actual["gpu_requested"] is True and actual["gpu_probe_unhidden"] is True + assert actual["stops"] == 0 + assert report["category"] == "PASS" and report["stage"] == "TORCH_IMPORT" + assert report["gpu_device"] == "0" and report["gpu_used"] is None + assert report["training_executed"] is False + + +def test_gpu_scope_rejects_before_effect_or_start(tmp_path: Path) -> None: + docker, config, state = _fake_docker(tmp_path) + with pytest.raises(ValueError, match="GPU_SCOPE_INVALID"): + _call(tmp_path, docker, config, gpu_device="0") + assert json.loads(state.read_text())["status"] == "absent" + assert not (tmp_path / "out").exists() + flags, _, _ = _stage_source(tmp_path) + with pytest.raises(ValueError, match="GPU_SCOPE_INVALID"): + _call(tmp_path, docker, config, gpu_device="1", **flags) + assert json.loads(state.read_text())["status"] == "absent" + _state(state, mode="wrong_gpu_request") + with pytest.raises(ValueError, match="GPU_SCOPE_INVALID"): + _call(tmp_path, docker, config, gpu_device="0", **flags) + assert json.loads(state.read_text())["status"] == "created" + assert json.loads(state.read_text())["stops"] == 0 + assert (tmp_path / "out/claim.json").exists() + assert not (tmp_path / "out/result.json").exists() + + +def test_invalid_gpu_cli_value_is_closed(tmp_path: Path, capsys) -> None: + exit_code = diagnostic.main([ + "--docker", str(tmp_path / "docker"), "--docker-config", str(tmp_path / "config"), + "--release", str(tmp_path / "release"), "--output-dir", str(tmp_path / "out"), + "--daemon-id", DAEMON, "--expected-release-sha256", "a" * 64, + "--gpu-device", "private-value", + ]) + captured = capsys.readouterr() + assert exit_code == 2 and captured.out == "" + assert json.loads(captured.err) == {"schema_version": diagnostic.SCHEMA, + "status": "REJECTED", "reason_code": "GPU_SCOPE_INVALID"} + + +@pytest.mark.parametrize("mode,reason,expected_stops", [ + ("start_hang", "START_UNCERTAIN", 1), + ("start_error", "START_UNCERTAIN", 1), + ("start_stdout_flood", "START_UNCERTAIN", 1), + ("start_stderr_flood", "START_UNCERTAIN", 1), + ("normal", "DEADLINE_STOPPED", 1), + ("stop_unconfirmed", "DEADLINE_UNCERTAIN", 1), + ("wrong_identity", "CONTAINER_STATE_INVALID", 0), + ("wrong_running_identity", "DEADLINE_UNCERTAIN", 0), +]) +def test_uncertain_start_and_exact_owned_cleanup(tmp_path: Path, monkeypatch, + mode, reason, expected_stops) -> None: + docker, config, state = _fake_docker(tmp_path) + _state(state, mode=mode) + real = diagnostic._run + monkeypatch.setattr(diagnostic, "_run", lambda authority, args, *, timeout=15: + real(authority, args, timeout=min(timeout, 0.3))) + monkeypatch.setattr(diagnostic, "_RUN_SECONDS", + 1 if mode in {"normal", "stop_unconfirmed"} else 0.1) + with pytest.raises(ValueError, match=reason): + _call(tmp_path, docker, config) + actual = json.loads(state.read_text()) + assert actual["stops"] == expected_stops + if expected_stops and mode != "stop_unconfirmed": + assert actual["status"] == "exited" + assert (tmp_path / "out/claim.json").exists() + assert not (tmp_path / "out/result.json").exists() + + +def test_uncertain_create_does_not_replay_or_start(tmp_path: Path, monkeypatch) -> None: + docker, config, state = _fake_docker(tmp_path) + _state(state, mode="create_hang") + real = diagnostic._run + monkeypatch.setattr(diagnostic, "_run", lambda authority, args, *, timeout=15: + real(authority, args, timeout=min(timeout, 0.2))) + with pytest.raises(ValueError, match="CREATE_UNCERTAIN"): + _call(tmp_path, docker, config) + assert json.loads(state.read_text())["status"] == "created" + assert json.loads(state.read_text())["stops"] == 0 + with pytest.raises(ValueError, match="OUTPUT_INVALID"): + _call(tmp_path, docker, config) + + +def test_invalid_created_id_is_consumed_without_start(tmp_path: Path) -> None: + docker, config, state = _fake_docker(tmp_path) + _state(state, mode="bad_id") + with pytest.raises(ValueError, match="CREATE_UNCERTAIN"): + _call(tmp_path, docker, config) + assert json.loads(state.read_text())["status"] == "created" + assert (tmp_path / "out/claim.json").exists() + + +def test_interrupt_after_start_stops_only_owned_container_once(tmp_path: Path, monkeypatch) -> None: + docker, config, state = _fake_docker(tmp_path) + monkeypatch.setattr(diagnostic, "time", type("Clock", (), { + "monotonic": staticmethod(__import__("time").monotonic), + "sleep": staticmethod(lambda _seconds: (_ for _ in ()).throw(KeyboardInterrupt())) + })()) + with pytest.raises(KeyboardInterrupt): + _call(tmp_path, docker, config) + actual = json.loads(state.read_text()) + assert actual["stops"] == 1 and actual["status"] == "exited" + + +def test_storage_failure_after_start_still_stops_owned_container(tmp_path: Path, monkeypatch) -> None: + docker, config, state = _fake_docker(tmp_path) + original_append = diagnostic._append + storage_failed = False + + def append(output_dir, event): + nonlocal storage_failed + if event["phase"] == "STARTED": + storage_failed = True + if storage_failed: + raise OSError("simulated full evidence filesystem") + original_append(output_dir, event) + + monkeypatch.setattr(diagnostic, "_append", append) + with pytest.raises(ValueError, match="DEADLINE_UNCERTAIN"): + _call(tmp_path, docker, config) + actual = json.loads(state.read_text()) + assert actual["stops"] == 1 and actual["status"] == "exited" + assert (tmp_path / "out/claim.json").exists() + assert not (tmp_path / "out/result.json").exists() + + +def test_uncertain_stop_acknowledgement_is_independently_inspected(tmp_path: Path, monkeypatch) -> None: + docker, config, state = _fake_docker(tmp_path) + original_run = diagnostic._run + inspected_after_stop = [] + stop_sent = False + + def run(authority, args, *, timeout=15): + nonlocal stop_sent + result = original_run(authority, args, timeout=timeout) + if args[:2] == ("container", "stop"): + stop_sent = True + raise ValueError("DOCKER_COMMAND_REJECTED") + if stop_sent and args[:2] == ("container", "inspect"): + inspected_after_stop.append(args[-1]) + return result + + monkeypatch.setattr(diagnostic, "_run", run) + monkeypatch.setattr(diagnostic, "_RUN_SECONDS", 1) + with pytest.raises(ValueError, match="DEADLINE_STOPPED"): + _call(tmp_path, docker, config) + actual = json.loads(state.read_text()) + assert actual["stops"] == 1 and actual["status"] == "exited" + assert inspected_after_stop == [CID] + events = [json.loads(line) for line in (tmp_path / "out/events.jsonl").read_text().splitlines()] + assert events[-1]["phase"] == "TERMINAL_OBSERVED" + + +def test_wrong_self_consistent_source_rejected_before_docker(tmp_path: Path, monkeypatch) -> None: + from tuner.runtime.offline_sft_worker import closure_digest + flags, document, manifest = _stage_source(tmp_path) + engine = flags["source_root"] + assert diagnostic._source_identity(engine, flags["source_lock"], + flags["expected_source_lock_sha256"], flags["expected_manifest_sha256"], + flags["expected_closure_digest"])["closure_digest"] == flags["expected_closure_digest"] + # Alter a staged member and rewrite all its internal commitments coherently. + target = engine / "shared/env_bootstrap.py" + target.write_bytes(target.read_bytes() + b"\n# alternate closure\n") + member = next(item for item in document["members"] if item["path"] == "shared/env_bootstrap.py") + member["size_bytes"] = target.stat().st_size + member["sha256"] = hashlib.sha256(target.read_bytes()).hexdigest() + document["payload_bytes"] += len(b"\n# alternate closure\n") + document["closure_digest"] = closure_digest(document) + manifest.write_bytes(json.dumps(document, sort_keys=True, separators=(",", ":")).encode() + b"\n") + from tuner.runtime.offline_sft_worker import load_offline_sft_worker_closure + assert load_offline_sft_worker_closure(manifest, + expected_digest=document["closure_digest"], engine_root=engine).closure_digest == document["closure_digest"] + docker, config, _state_path = _fake_docker(tmp_path) + monkeypatch.setattr(diagnostic, "_run", lambda *args, **kwargs: pytest.fail("Docker touched")) + with pytest.raises(ValueError, match="SOURCE_IDENTITY_INVALID"): + _call(tmp_path, docker, config, **flags) + assert not (tmp_path / "out").exists() + + +def test_cli_rejection_never_prints_exception_text(monkeypatch, capsys) -> None: + def fail(**_kwargs): + raise RuntimeError("private exception path /secret and credential") + monkeypatch.setattr(diagnostic, "diagnose", fail) + code = diagnostic.main(["--docker", "/usr/bin/docker", "--docker-config", "/private/config", + "--release", "/private/release.json", "--output-dir", "/private/output", + "--daemon-id", DAEMON, "--expected-release-sha256", "a" * 64]) + assert code == 2 + output = capsys.readouterr() + assert output.out == "" + assert json.loads(output.err) == {"schema_version": diagnostic.SCHEMA, + "status": "REJECTED", "reason_code": "OTHER"}