From 2762cd186751b2c42ded7bf7033d8e2075800d5d Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:50:53 -0400 Subject: [PATCH 01/12] Add restricted native MLX host consumer and close review gaps --- .agents/skills/fine-tuning/SKILL.md | 2 +- .../reference/native-local-training.md | 51 +- .claude/skills/fine-tuning/SKILL.md | 2 +- .../reference/native-local-training.md | 51 +- .skills/fine-tuning/SKILL.md | 2 +- .../reference/native-local-training.md | 51 +- .../training_qualification/native-host.md | 152 +++++ .../native_host.example.json | 46 ++ .../training_qualification/native_host.py | 624 ++++++++++++++++++ tests/cli/test_native_host.py | 98 +++ tests/runtime/test_native_mlx_child.py | 45 +- tests/runtime/test_native_mlx_execution.py | 71 +- tests/runtime/test_native_mlx_family.py | 76 ++- tests/runtime/test_native_mlx_publication.py | 51 +- tuner/runtime/native_mlx_child.py | 43 +- tuner/runtime/native_mlx_composition.py | 23 +- tuner/runtime/native_mlx_publication.py | 73 +- tuner/training/native_mlx_family.py | 31 +- tuner/training/native_mlx_planning.py | 79 ++- 19 files changed, 1458 insertions(+), 113 deletions(-) create mode 100644 examples/training_qualification/native-host.md create mode 100644 examples/training_qualification/native_host.example.json create mode 100644 examples/training_qualification/native_host.py create mode 100644 tests/cli/test_native_host.py diff --git a/.agents/skills/fine-tuning/SKILL.md b/.agents/skills/fine-tuning/SKILL.md index 7aae0806..751c1325 100644 --- a/.agents/skills/fine-tuning/SKILL.md +++ b/.agents/skills/fine-tuning/SKILL.md @@ -57,7 +57,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Check SFT loss masking (tokenizer only) | `python tuner.py doctor sft-mask --dataset-path --model ` | | Inspect native training host candidate | `python tuner.py doctor native-training --json` — read-only platform/Python/package metadata; see `reference/native-local-training.md` | | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | -| Native MLX prepared SFT candidate | Gated `ProviderFamilyV1` through existing TrainingAPI/RunsAPI; isolated worker, signed host catalog and native artifacts; no hardware qualification or launch CLI yet; see `reference/native-local-training.md` | +| Native MLX restricted host consumer | `python -B examples/training_qualification/native_host.py inspect ` is read-only; exact-run `start` requires an externally signed root admission and separately supplied public-key pin. No live Mac qualification yet; see `reference/native-local-training.md` | | Capture reviewed native runtime bytes | `python -B scripts/capture_native_mlx_runtime.py --config --dependency-lock --output ` — observation only; explicit roots, dependencies and original pins required; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.agents/skills/fine-tuning/reference/native-local-training.md b/.agents/skills/fine-tuning/reference/native-local-training.md index f7684383..2893ada9 100644 --- a/.agents/skills/fine-tuning/reference/native-local-training.md +++ b/.agents/skills/fine-tuning/reference/native-local-training.md @@ -60,10 +60,9 @@ original hash lock and use the existing `uv pip freeze`, `uv pip check`, `shasum -a 256` and `otool -L` tools for review evidence; do not install packages into the shared interpreter or treat a newly observed inventory as approved. -There is no native launch CLI yet. The internal worker may run only in an -independently admitted, host-owned bounded child. The desktop plan lists the -implemented runtime/family boundaries and remaining consumer/qualification work. -Do not call the worker as an ad hoc substitute for that composition. +The internal worker may run only in an independently admitted, host-owned +bounded child. Use the checked-in restricted host consumer described below; +do not call the worker directly as an ad hoc substitute. ## Execution-boundary continuation @@ -71,8 +70,8 @@ The internal runtime modules now include a read-only native material capture, retained-source/tokenizer join, explicit LoRA/constant-AdamW composition, independent safetensors delta verifier, and owned subprocess guardian. These are reviewable runtime boundaries. The gated family described below composes them -through the existing public coordinator; no global provider registration or -native launch CLI is supplied. +through the existing public coordinator; no global provider registration is +supplied. `scripts/capture_native_mlx_runtime.py` takes a JSON object with `capture` and `expectation`. `expectation` uses the existing `MLXRuntimeExpectation` fields. @@ -113,8 +112,11 @@ LoRA target/layer explicitly, checks the realized trainable names/shapes and use explicit AdamW parameters with constant schedule. Unsupported schedules reject. It checks the optimizer's actual step count. The native adapter verifier checks tensor headers, bounds, names/shapes/dtypes, finite values and a nonzero numeric -parameter delta. Final validation is a distinct pass **after reloading the saved -adapter**; upstream in-loop validation occurs before the last optimizer update. +parameter delta. Final validation loads a second clean pinned base, attaches the +same configured LoRA, loads the saved final adapter and compares every trainable +tensor against the saved bytes before evaluating that second model. A no-op or +partial reload rejects. Upstream in-loop validation occurs before the last +optimizer update. It saves native `adapter_config.json`, `input_binding.json` and `native_result.json` exclusively. These worker files become authenticated public artifact roles only through the family verifier described below. @@ -151,7 +153,11 @@ embedded in a launcher. `native_mlx_child.py` is an internal `-I -S -B` entrypoint. It restricts import roots, checks imported source/extension origins against capture members and compiles pinned Python source rather than ambient bytecode. It joins the exact -staged dataset, capture, lock and snapshots to the recompiled workload. It is +staged dataset, capture, lock and snapshots to the recompiled workload. Before +optimizer work, it builds the exact complete tokenizer role archive, extracts +those bytes into a private temporary directory and loads the tokenizer locally +with remote code disabled. The host later publishes that same deterministic +archive. It is not a public invocation or an independent source/qualification authority. `native_runtime_spec` represents the capture with the existing `RuntimeSpec` `native_environment` discriminator; it invents no OCI image. Legacy OCI @@ -166,9 +172,28 @@ must authenticate actual source evidence and current hardware before signing command material. `NativeCommandCatalogV1` retains create-only signed material through the existing record port. Signed terminal evidence must also match the retained process outcome on every read. It is not an engine database or replay grant. +Immediately before a submit process claim, the family recaptures the admitted +runtime/source closure and verifies complete model/tokenizer inventories. Drift +at this boundary rejects before `run_once`; later dispatch remains uncertain. +The CPU artifact verifier requires every configured target in the exact final +model layer set and complete tokenizer snapshot selection. + +No accepting **ordinary** hardware authority is shipped. An unadmitted +candidate has observe/artifact capabilities disabled and preflight is not +ready. A separately verified `NativeRestrictedAdmissionV1` enables only its +exact initial run, attempt, source, runtime, prepared bytes and model/tokenizer +snapshots, subject to a finite expiry; it is not qualification. The host +consumer in `examples/training_qualification/native_host.py` verifies the +external signature using a separately supplied reviewed public-key SHA-256. +It has no signing key or acceptance switch. The same host checks the actual +clean pushed Git ref, full runtime recapture, private prepared publication, +model snapshot inventories and M4 hardware before issuing command material. +The proposal remains parameters, not an admission. See +`examples/training_qualification/native-host.md` for exact config fields, +preparation/readback commands, and the root signing boundary. No Mac training, +ML import, model download or host qualification has been performed for this +consumer yet. -No accepting hardware authority is shipped. Candidate descriptors have -observe/artifact capabilities disabled and preflight is not ready. A `NativeQualificationV1` contains exact selected profile/source/runtime/hardware and two distinct retained attempt-evidence references; its independent host authority must authenticate the supporting real runs, selected wheel/install @@ -186,6 +211,6 @@ configured member archive from the verified snapshot. Lineage/metrics retain the exact worker bindings. Finite file/aggregate budgets and public 1 MiB stream chunks apply. CPU verification does not rerun the final evaluation. -See `docs/plans/native-desktop-training-api-v1.md` for remaining host assembly, -review and real-Mac acceptance. The CPU facade tests substitute the ML child; +See `docs/plans/native-desktop-training-api-v1.md` for remaining review and +real-Mac acceptance. The CPU facade tests substitute the ML child; they are not a hardware result and must never unlock the candidate. diff --git a/.claude/skills/fine-tuning/SKILL.md b/.claude/skills/fine-tuning/SKILL.md index 7aae0806..751c1325 100644 --- a/.claude/skills/fine-tuning/SKILL.md +++ b/.claude/skills/fine-tuning/SKILL.md @@ -57,7 +57,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Check SFT loss masking (tokenizer only) | `python tuner.py doctor sft-mask --dataset-path --model ` | | Inspect native training host candidate | `python tuner.py doctor native-training --json` — read-only platform/Python/package metadata; see `reference/native-local-training.md` | | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | -| Native MLX prepared SFT candidate | Gated `ProviderFamilyV1` through existing TrainingAPI/RunsAPI; isolated worker, signed host catalog and native artifacts; no hardware qualification or launch CLI yet; see `reference/native-local-training.md` | +| Native MLX restricted host consumer | `python -B examples/training_qualification/native_host.py inspect ` is read-only; exact-run `start` requires an externally signed root admission and separately supplied public-key pin. No live Mac qualification yet; see `reference/native-local-training.md` | | Capture reviewed native runtime bytes | `python -B scripts/capture_native_mlx_runtime.py --config --dependency-lock --output ` — observation only; explicit roots, dependencies and original pins required; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.claude/skills/fine-tuning/reference/native-local-training.md b/.claude/skills/fine-tuning/reference/native-local-training.md index f7684383..2893ada9 100644 --- a/.claude/skills/fine-tuning/reference/native-local-training.md +++ b/.claude/skills/fine-tuning/reference/native-local-training.md @@ -60,10 +60,9 @@ original hash lock and use the existing `uv pip freeze`, `uv pip check`, `shasum -a 256` and `otool -L` tools for review evidence; do not install packages into the shared interpreter or treat a newly observed inventory as approved. -There is no native launch CLI yet. The internal worker may run only in an -independently admitted, host-owned bounded child. The desktop plan lists the -implemented runtime/family boundaries and remaining consumer/qualification work. -Do not call the worker as an ad hoc substitute for that composition. +The internal worker may run only in an independently admitted, host-owned +bounded child. Use the checked-in restricted host consumer described below; +do not call the worker directly as an ad hoc substitute. ## Execution-boundary continuation @@ -71,8 +70,8 @@ The internal runtime modules now include a read-only native material capture, retained-source/tokenizer join, explicit LoRA/constant-AdamW composition, independent safetensors delta verifier, and owned subprocess guardian. These are reviewable runtime boundaries. The gated family described below composes them -through the existing public coordinator; no global provider registration or -native launch CLI is supplied. +through the existing public coordinator; no global provider registration is +supplied. `scripts/capture_native_mlx_runtime.py` takes a JSON object with `capture` and `expectation`. `expectation` uses the existing `MLXRuntimeExpectation` fields. @@ -113,8 +112,11 @@ LoRA target/layer explicitly, checks the realized trainable names/shapes and use explicit AdamW parameters with constant schedule. Unsupported schedules reject. It checks the optimizer's actual step count. The native adapter verifier checks tensor headers, bounds, names/shapes/dtypes, finite values and a nonzero numeric -parameter delta. Final validation is a distinct pass **after reloading the saved -adapter**; upstream in-loop validation occurs before the last optimizer update. +parameter delta. Final validation loads a second clean pinned base, attaches the +same configured LoRA, loads the saved final adapter and compares every trainable +tensor against the saved bytes before evaluating that second model. A no-op or +partial reload rejects. Upstream in-loop validation occurs before the last +optimizer update. It saves native `adapter_config.json`, `input_binding.json` and `native_result.json` exclusively. These worker files become authenticated public artifact roles only through the family verifier described below. @@ -151,7 +153,11 @@ embedded in a launcher. `native_mlx_child.py` is an internal `-I -S -B` entrypoint. It restricts import roots, checks imported source/extension origins against capture members and compiles pinned Python source rather than ambient bytecode. It joins the exact -staged dataset, capture, lock and snapshots to the recompiled workload. It is +staged dataset, capture, lock and snapshots to the recompiled workload. Before +optimizer work, it builds the exact complete tokenizer role archive, extracts +those bytes into a private temporary directory and loads the tokenizer locally +with remote code disabled. The host later publishes that same deterministic +archive. It is not a public invocation or an independent source/qualification authority. `native_runtime_spec` represents the capture with the existing `RuntimeSpec` `native_environment` discriminator; it invents no OCI image. Legacy OCI @@ -166,9 +172,28 @@ must authenticate actual source evidence and current hardware before signing command material. `NativeCommandCatalogV1` retains create-only signed material through the existing record port. Signed terminal evidence must also match the retained process outcome on every read. It is not an engine database or replay grant. +Immediately before a submit process claim, the family recaptures the admitted +runtime/source closure and verifies complete model/tokenizer inventories. Drift +at this boundary rejects before `run_once`; later dispatch remains uncertain. +The CPU artifact verifier requires every configured target in the exact final +model layer set and complete tokenizer snapshot selection. + +No accepting **ordinary** hardware authority is shipped. An unadmitted +candidate has observe/artifact capabilities disabled and preflight is not +ready. A separately verified `NativeRestrictedAdmissionV1` enables only its +exact initial run, attempt, source, runtime, prepared bytes and model/tokenizer +snapshots, subject to a finite expiry; it is not qualification. The host +consumer in `examples/training_qualification/native_host.py` verifies the +external signature using a separately supplied reviewed public-key SHA-256. +It has no signing key or acceptance switch. The same host checks the actual +clean pushed Git ref, full runtime recapture, private prepared publication, +model snapshot inventories and M4 hardware before issuing command material. +The proposal remains parameters, not an admission. See +`examples/training_qualification/native-host.md` for exact config fields, +preparation/readback commands, and the root signing boundary. No Mac training, +ML import, model download or host qualification has been performed for this +consumer yet. -No accepting hardware authority is shipped. Candidate descriptors have -observe/artifact capabilities disabled and preflight is not ready. A `NativeQualificationV1` contains exact selected profile/source/runtime/hardware and two distinct retained attempt-evidence references; its independent host authority must authenticate the supporting real runs, selected wheel/install @@ -186,6 +211,6 @@ configured member archive from the verified snapshot. Lineage/metrics retain the exact worker bindings. Finite file/aggregate budgets and public 1 MiB stream chunks apply. CPU verification does not rerun the final evaluation. -See `docs/plans/native-desktop-training-api-v1.md` for remaining host assembly, -review and real-Mac acceptance. The CPU facade tests substitute the ML child; +See `docs/plans/native-desktop-training-api-v1.md` for remaining review and +real-Mac acceptance. The CPU facade tests substitute the ML child; they are not a hardware result and must never unlock the candidate. diff --git a/.skills/fine-tuning/SKILL.md b/.skills/fine-tuning/SKILL.md index 7aae0806..751c1325 100644 --- a/.skills/fine-tuning/SKILL.md +++ b/.skills/fine-tuning/SKILL.md @@ -57,7 +57,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Check SFT loss masking (tokenizer only) | `python tuner.py doctor sft-mask --dataset-path --model ` | | Inspect native training host candidate | `python tuner.py doctor native-training --json` — read-only platform/Python/package metadata; see `reference/native-local-training.md` | | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | -| Native MLX prepared SFT candidate | Gated `ProviderFamilyV1` through existing TrainingAPI/RunsAPI; isolated worker, signed host catalog and native artifacts; no hardware qualification or launch CLI yet; see `reference/native-local-training.md` | +| Native MLX restricted host consumer | `python -B examples/training_qualification/native_host.py inspect ` is read-only; exact-run `start` requires an externally signed root admission and separately supplied public-key pin. No live Mac qualification yet; see `reference/native-local-training.md` | | Capture reviewed native runtime bytes | `python -B scripts/capture_native_mlx_runtime.py --config --dependency-lock --output ` — observation only; explicit roots, dependencies and original pins required; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.skills/fine-tuning/reference/native-local-training.md b/.skills/fine-tuning/reference/native-local-training.md index f7684383..2893ada9 100644 --- a/.skills/fine-tuning/reference/native-local-training.md +++ b/.skills/fine-tuning/reference/native-local-training.md @@ -60,10 +60,9 @@ original hash lock and use the existing `uv pip freeze`, `uv pip check`, `shasum -a 256` and `otool -L` tools for review evidence; do not install packages into the shared interpreter or treat a newly observed inventory as approved. -There is no native launch CLI yet. The internal worker may run only in an -independently admitted, host-owned bounded child. The desktop plan lists the -implemented runtime/family boundaries and remaining consumer/qualification work. -Do not call the worker as an ad hoc substitute for that composition. +The internal worker may run only in an independently admitted, host-owned +bounded child. Use the checked-in restricted host consumer described below; +do not call the worker directly as an ad hoc substitute. ## Execution-boundary continuation @@ -71,8 +70,8 @@ The internal runtime modules now include a read-only native material capture, retained-source/tokenizer join, explicit LoRA/constant-AdamW composition, independent safetensors delta verifier, and owned subprocess guardian. These are reviewable runtime boundaries. The gated family described below composes them -through the existing public coordinator; no global provider registration or -native launch CLI is supplied. +through the existing public coordinator; no global provider registration is +supplied. `scripts/capture_native_mlx_runtime.py` takes a JSON object with `capture` and `expectation`. `expectation` uses the existing `MLXRuntimeExpectation` fields. @@ -113,8 +112,11 @@ LoRA target/layer explicitly, checks the realized trainable names/shapes and use explicit AdamW parameters with constant schedule. Unsupported schedules reject. It checks the optimizer's actual step count. The native adapter verifier checks tensor headers, bounds, names/shapes/dtypes, finite values and a nonzero numeric -parameter delta. Final validation is a distinct pass **after reloading the saved -adapter**; upstream in-loop validation occurs before the last optimizer update. +parameter delta. Final validation loads a second clean pinned base, attaches the +same configured LoRA, loads the saved final adapter and compares every trainable +tensor against the saved bytes before evaluating that second model. A no-op or +partial reload rejects. Upstream in-loop validation occurs before the last +optimizer update. It saves native `adapter_config.json`, `input_binding.json` and `native_result.json` exclusively. These worker files become authenticated public artifact roles only through the family verifier described below. @@ -151,7 +153,11 @@ embedded in a launcher. `native_mlx_child.py` is an internal `-I -S -B` entrypoint. It restricts import roots, checks imported source/extension origins against capture members and compiles pinned Python source rather than ambient bytecode. It joins the exact -staged dataset, capture, lock and snapshots to the recompiled workload. It is +staged dataset, capture, lock and snapshots to the recompiled workload. Before +optimizer work, it builds the exact complete tokenizer role archive, extracts +those bytes into a private temporary directory and loads the tokenizer locally +with remote code disabled. The host later publishes that same deterministic +archive. It is not a public invocation or an independent source/qualification authority. `native_runtime_spec` represents the capture with the existing `RuntimeSpec` `native_environment` discriminator; it invents no OCI image. Legacy OCI @@ -166,9 +172,28 @@ must authenticate actual source evidence and current hardware before signing command material. `NativeCommandCatalogV1` retains create-only signed material through the existing record port. Signed terminal evidence must also match the retained process outcome on every read. It is not an engine database or replay grant. +Immediately before a submit process claim, the family recaptures the admitted +runtime/source closure and verifies complete model/tokenizer inventories. Drift +at this boundary rejects before `run_once`; later dispatch remains uncertain. +The CPU artifact verifier requires every configured target in the exact final +model layer set and complete tokenizer snapshot selection. + +No accepting **ordinary** hardware authority is shipped. An unadmitted +candidate has observe/artifact capabilities disabled and preflight is not +ready. A separately verified `NativeRestrictedAdmissionV1` enables only its +exact initial run, attempt, source, runtime, prepared bytes and model/tokenizer +snapshots, subject to a finite expiry; it is not qualification. The host +consumer in `examples/training_qualification/native_host.py` verifies the +external signature using a separately supplied reviewed public-key SHA-256. +It has no signing key or acceptance switch. The same host checks the actual +clean pushed Git ref, full runtime recapture, private prepared publication, +model snapshot inventories and M4 hardware before issuing command material. +The proposal remains parameters, not an admission. See +`examples/training_qualification/native-host.md` for exact config fields, +preparation/readback commands, and the root signing boundary. No Mac training, +ML import, model download or host qualification has been performed for this +consumer yet. -No accepting hardware authority is shipped. Candidate descriptors have -observe/artifact capabilities disabled and preflight is not ready. A `NativeQualificationV1` contains exact selected profile/source/runtime/hardware and two distinct retained attempt-evidence references; its independent host authority must authenticate the supporting real runs, selected wheel/install @@ -186,6 +211,6 @@ configured member archive from the verified snapshot. Lineage/metrics retain the exact worker bindings. Finite file/aggregate budgets and public 1 MiB stream chunks apply. CPU verification does not rerun the final evaluation. -See `docs/plans/native-desktop-training-api-v1.md` for remaining host assembly, -review and real-Mac acceptance. The CPU facade tests substitute the ML child; +See `docs/plans/native-desktop-training-api-v1.md` for remaining review and +real-Mac acceptance. The CPU facade tests substitute the ML child; they are not a hardware result and must never unlock the candidate. diff --git a/examples/training_qualification/native-host.md b/examples/training_qualification/native-host.md new file mode 100644 index 00000000..2083e8d0 --- /dev/null +++ b/examples/training_qualification/native-host.md @@ -0,0 +1,152 @@ +# Native MLX host consumer — restricted qualification candidate + +This example composes the existing public `TrainingAPI` and `RunsAPI`, reference +host record/stream stores, Foundation grants, and the native provider family. +It has no engine database, global provider registration, background training +job, replay, install, or model download. `inspect` and `profile-candidate` are +read-only. `start` is a one-shot effect and requires a separate root-signed +admission for its exact source, runtime, hardware, model snapshots, prepared +publication, run and attempt. An ordinary two-run qualification receipt is not +created by this workflow. + +## Inputs and private layout + +Use an isolated **clean pushed** engine checkout on the M4 Mac. The proposed +runtime is CPython 3.11.14, MLX/MLX-LM 0.32.2, original 50-dependency lock SHA +`f9a6298260cd17f2c3d567ae3674108a71efa68e99369906487a4d8c0feb7117`, +and model/tokenizer `mlx-community/Qwen3-0.6B-4bit` revision +`73e3e38d981303bc594367cd910ea6eb48349da8`. The proposal file under +`Trainers/mlx_sft_mac/runtime-candidate/` contains the exact 3-update, +5-minute, 3-train/2-validation, LoRA and artifact bounds. It is reviewable +parameters, not execution approval. + +Keep the existing private candidate at +`/Users/jrosenbaum/.local/share/synaptic-qualification/native-mlx-candidate-20261020` +intact. Place each new prepared publication, complete pinned model snapshot, +capture, reviewed profiles, admission and attempt state under a separate +owner-only 0700 directory there. The runtime, source checkout and model roots +must have no group/other write permission. The model and tokenizer snapshot +files list every regular member, size and SHA-256. They are reviewed inputs; +the consumer verifies complete current inventories and never downloads them. +Retain original private authority keys outside source control. The three +`host` secret references in the config resolve at start, not during `inspect`. +After a separately approved private snapshot acquisition, inventory its exact +bytes with the read-only `snapshot-candidate` action. The repository/revision +arguments are claims for independent review; this command does not fetch or +attest them. + +The checked-in [config template](native_host.example.json) is deliberately +incomplete. Replace **every** placeholder with the actual reviewed paths and +identities. Use a fresh `state_root`, `run_id`, `request_id`, and `attempt` for +each separately approved run. Select `attempt_kind=raw_full_sequence` for the +first publication and `prompt_completion_assistant` for the second. The second +public input must explicitly set `chat_template_kwargs` to +`{"enable_thinking":false}`; the shared preprocessor supplies exact labels. +The consumer checks declared split coverage and compilation, while the root +review checks representative complete rows and token/label semantics. + +Use the existing `python tuner.py prepare-dataset --config +--json` workflow to create the immutable synthetic publication. Supply its +actual `PreparedTrainingInputIdentity` and publication path; never construct +an identity from the filename. The existing dataset snapshot adapter verifies +the publication when the consumer reads it. For the message attempt, review +the actual shared preprocessor output; a `prompt_completion` label does not by +itself prove assistant-only labels. + +## Reusable preparation and review sequence + +These are commands for the **later approved Mac stage**. They have not been +run against the Mac for this increment. + +```sh +# From the clean pushed engine checkout, with the existing private 3.11.14 venv. +umask 077 +set -C # Every shell redirection below creates a new file or fails. +candidate_python=/Users/jrosenbaum/.local/share/synaptic-qualification/native-mlx-candidate-20261020/venv/bin/python +private_root=/Users/jrosenbaum/.local/share/synaptic-qualification/native-mlx-candidate-20261020/reviewed-attempt-1 +# Create and inspect this fresh owner-only directory before writing any files. +mkdir -m 700 "$private_root" + +# Prepare capture-config.json with exact stdlib, private package tree, narrow +# engine/tuner source tree, executable/framework and system-library roots. +# The capture command creates capture.json once and reports OBSERVED_NOT_ADMITTED. +"$candidate_python" -B scripts/capture_native_mlx_runtime.py \ + --config "$private_root/capture-config.json" \ + --dependency-lock Trainers/mlx_sft_mac/runtime-candidate/requirements.lock \ + --output "$private_root/capture.json" + +# Run separately for complete, already reviewed private model and tokenizer +# roots. Keep both manifests as root-review candidates, not authority. +"$candidate_python" -B examples/training_qualification/native_host.py \ + snapshot-candidate --snapshot-root "$private_root/model-snapshot" \ + --repository mlx-community/Qwen3-0.6B-4bit \ + --revision 73e3e38d981303bc594367cd910ea6eb48349da8 \ + > "$private_root/model-snapshot.json" +"$candidate_python" -B examples/training_qualification/native_host.py \ + snapshot-candidate --snapshot-root "$private_root/tokenizer-snapshot" \ + --repository mlx-community/Qwen3-0.6B-4bit \ + --revision 73e3e38d981303bc594367cd910ea6eb48349da8 \ + > "$private_root/tokenizer-snapshot.json" + +# Fill the template with exact paths, pushed ref and commit, scope, original +# private secret references, fresh attempt IDs, snapshots and publication. +# The render action only canonicalizes JSON; it grants no authority. +"$candidate_python" -B examples/training_qualification/native_host.py \ + canonicalize "$private_root/host-config-draft.json" > "$private_root/host-config.json" + +# The profile candidate uses the actual pushed Git ref, Mac hardware and the +# full runtime recapture. Root reviews this output before it is selected. +"$candidate_python" -B examples/training_qualification/native_host.py \ + profile-candidate "$private_root/host-config.json" > "$private_root/profile.json" +"$candidate_python" -B examples/training_qualification/native_host.py \ + inspect "$private_root/host-config.json" > "$private_root/inspection.json" + +# Root supplies the exact expiry and authority/key references. Review the +# inspection, complete source/capture/build provenance, wheel selections, +# tokenizer rows and admission content independently before signing. +"$candidate_python" -B examples/training_qualification/native_host.py \ + admission-candidate "$private_root/host-config.json" \ + --expires-at REVIEWED_EXPIRY_WITHIN_ONE_HOUR_UTC \ + --authority-ref ROOT_REVIEW --key-ref REVIEWED_KEY \ + > "$private_root/admission-content.json" + +# Root signing occurs outside the consumer, with a private key that is never +# placed on the execution Mac. The reviewer returns admission.sig and a pinned +# root public key. The host only verifies their exact content. +"$candidate_python" -B examples/training_qualification/native_host.py \ + seal-admission "$private_root/admission-content.json" \ + --signature "$private_root/admission.sig" > "$private_root/admission.json" + +# Only after root's bounded live authorization: one start, no retry on error. +"$candidate_python" -B examples/training_qualification/native_host.py \ + start "$private_root/host-config.json" \ + --trusted-root-key-sha256 REVIEWED_ROOT_PUBLIC_KEY_SHA256 + +# New process, same known run and preserved state. These are public API reads; +# logs, cancel and reconcile are intentionally unavailable. +"$candidate_python" -B examples/training_qualification/native_host.py \ + observe "$private_root/host-config.json" \ + --trusted-root-key-sha256 REVIEWED_ROOT_PUBLIC_KEY_SHA256 +"$candidate_python" -B examples/training_qualification/native_host.py \ + verify "$private_root/host-config.json" \ + --trusted-root-key-sha256 REVIEWED_ROOT_PUBLIC_KEY_SHA256 +"$candidate_python" -B examples/training_qualification/native_host.py \ + download "$private_root/host-config.json" \ + --trusted-root-key-sha256 REVIEWED_ROOT_PUBLIC_KEY_SHA256 \ + --destination "$private_root/downloaded-artifacts" +``` + +The `admission-content.json` file is the exact canonical message to sign with +`openssl dgst -sha256 -sign -out admission.sig`. +`seal-admission` merely records the signature hash; `start` verifies the +signature against the separately passed public-key fingerprint. A hash in +the config or inspection is not independent approval. Root must review the +source commit, selected wheels/install provenance, capture/build closure, +hardware, model snapshot and both distinct prepared datasets before signing. + +An admitted restricted run can only use its exact run/attempt and snapshots. +Its one-hour admission expires; collect cold public reads within that window. +The normal capability receipt still needs two real accepted runs, native +artifact verification, Darwin deadline/owner-loss evidence, and a real +Syntunia bridge readback. Public logs/cancel/reconcile, GRPO, and asynchronous +desktop operation remain unsupported. diff --git a/examples/training_qualification/native_host.example.json b/examples/training_qualification/native_host.example.json new file mode 100644 index 00000000..91e26f3a --- /dev/null +++ b/examples/training_qualification/native_host.example.json @@ -0,0 +1,46 @@ +{ + "schema": "synaptic-native-host-config/v1", + "proposal_file": "/REVIEWED_CLEAN_ENGINE/Trainers/mlx_sft_mac/runtime-candidate/sft-qualification-proposal.json", + "engine_root": "/REVIEWED_CLEAN_ENGINE", + "engine_remote": "https://github.com/ProfSynapse/Synaptic-Tuner.git", + "engine_ref": "refs/heads/REVIEWED_PUSHED_BRANCH", + "engine_commit": "REVIEWED_PUSHED_40_HEX_COMMIT", + "runtime_capture_file": "/PRIVATE_ROOT/capture.json", + "dependency_lock_file": "/REVIEWED_CLEAN_ENGINE/Trainers/mlx_sft_mac/runtime-candidate/requirements.lock", + "profile_file": "/PRIVATE_ROOT/profile.json", + "training_input_file": "/PRIVATE_ROOT/training-input.json", + "prepared_identity_file": "/PRIVATE_ROOT/prepared-identity.json", + "prepared_publication": "/PRIVATE_ROOT/prepared-publication", + "model_snapshot_file": "/PRIVATE_ROOT/model-snapshot.json", + "tokenizer_snapshot_file": "/PRIVATE_ROOT/tokenizer-snapshot.json", + "admission_file": "/PRIVATE_ROOT/admission.json", + "admission_signature_file": "/PRIVATE_ROOT/admission.sig", + "root_public_key_file": "/PRIVATE_ROOT/root-public-key.pem", + "root_public_key_sha256": "REVIEWED_ROOT_PUBLIC_KEY_SHA256", + "limits": { + "timeout_seconds": 300, + "termination_grace_seconds": 5, + "maximum_output_bytes": 1048576, + "maximum_file_bytes": 67108864 + }, + "scope": {"account_ref": "REVIEWED_ACCOUNT", "namespace_ref": "REVIEWED_MAC"}, + "run_id": "FRESH_RUN_ID", + "project_ref": "REVIEWED_PROJECT", + "request_id": "FRESH_REQUEST_ID", + "attempt": "FRESH_ATTEMPT_ID", + "attempt_kind": "raw_full_sequence", + "state_root": "/PRIVATE_ROOT/unused-attempt-state", + "host": { + "authority_secret": {"provider": "env", "name": "NATIVE_FOUNDATION_AUTHORITY"}, + "evidence_secret": {"provider": "env", "name": "NATIVE_READER_EVIDENCE"}, + "material_secret": {"provider": "env", "name": "NATIVE_COMMAND_MATERIAL"}, + "grant_ref": "REVIEWED_GRANT_REF", + "reader_authority_ref": "REVIEWED_READER_AUTHORITY", + "reader_key_ref": "REVIEWED_READER_KEY", + "material_authority_ref": "REVIEWED_MATERIAL_AUTHORITY", + "material_key_ref": "REVIEWED_MATERIAL_KEY" + }, + "tokenizer_members": ["REVIEWED_COMPLETE_TOKENIZER_MEMBER"], + "artifact_budget": {"per_file": 67108864, "aggregate": 134217728}, + "use_fast_tokenizer": true +} diff --git a/examples/training_qualification/native_host.py b/examples/training_qualification/native_host.py new file mode 100644 index 00000000..dce1504b --- /dev/null +++ b/examples/training_qualification/native_host.py @@ -0,0 +1,624 @@ +"""Host-owned, one-attempt native MLX consumer for the public v1 facades. + +No install, download, implicit admission, retry, or capability promotion lives +here. ``inspect`` is read-only. ``start`` needs an external root signature over +the exact admission candidate and consumes one fresh private state directory. +""" + +from __future__ import annotations + +from dataclasses import asdict +import hashlib +import json +import os +from pathlib import Path +import platform +import re +import subprocess +import sys + +ENGINE_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ENGINE_ROOT)) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from launch import Clock, NamedSecrets, _private, download + +from synaptic_tuner.api.v1.planning import ResolvedTrainingRequest +from synaptic_tuner.api.v1.reference import ReferenceRequestPortsV1, compose_reference_authority, compose_reference_host +from synaptic_tuner.api.v1.reference.provider_family import ReferenceHostPortsV1 +from synaptic_tuner.api.v1.reference.sqlite_store import SQLiteDurableStoreV1 +from synaptic_tuner.api.v1.execution import ExecutionGrant +from synaptic_tuner.api.v1.results import TrainingRunRef +from synaptic_tuner.api.v1.secrets import SecretRef +from synaptic_tuner.api.v1.training_facade import AuthorizationRequirement, TrainingRequest +from synaptic_tuner.api.v1.training_input import TrainingInputV1 +from synaptic_tuner.api.v1.training_sources import PreparedTrainingInputIdentity +from tuner.dataset_prep.training_input_source import LocalPreparedTrainingInputSource +from tuner.execution.foundation_v2.commands import parse_exact_command +from tuner.execution.foundation_v2.references import ExecutionScopeV1 +from tuner.runtime.native_files import NativeFile, NativeTree +from tuner.runtime.native_mlx_input import NativeModelSnapshot, read_retained_native_source +from tuner.runtime.native_mlx_material import canonical, native_runtime_spec, strict_json, verify_native_material +from tuner.runtime.native_mlx_publication import NativeArtifactBudget +from tuner.runtime.native_process import NativeInvocation, NativeProcessLimits +from tuner.runtime.packaged_worker_closure import stable_read +from tuner.training.coordinator_material import _domain, _runtime +from tuner.training.modal_host_authority import HMACAuthenticator +from tuner.training.native_mlx_authority import NativeEvidenceAuthority, NativeMaterialAuthority +from tuner.training.native_mlx_compilation import NativeMLXProfile, compile_native_sft +from tuner.training.native_mlx_family import NativeCommandCatalogV1, NativeCommandMaterialV1, native_mlx_family +from tuner.training.native_mlx_planning import NativePlanningV1, NativeRestrictedAdmissionV1 + + +def _read(path, maximum): + value = Path(path) + if not value.is_absolute(): + raise ValueError("native input path must be absolute") + return stable_read(value, maximum) + + +def _document(path, maximum=2 << 20): + raw = _read(path, maximum) + value = strict_json(raw, maximum=maximum) + if canonical(value) != raw: + raise ValueError("native host input must be canonical JSON") + return value + + +def _create_private_dir(path): + path = Path(path) + if not path.is_absolute() or str(path) != os.path.normpath(str(path)): + raise ValueError("native private directory path is invalid") + with NativeTree(path.parent, private=True) as parent: + os.mkdir(path.name, mode=0o700, dir_fd=parent.fd) + parent.check() + _private(path) + + +def _write_owned(root, name, raw, maximum): + if type(raw) is not bytes or not 0 < len(raw) <= maximum or "/" in name: + raise ValueError("native retained input is invalid") + with NativeTree(root, private=True) as tree: + tree.check() + fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, dir_fd=tree.fd) + with os.fdopen(fd, "wb") as target: + target.write(raw) + target.flush() + os.fsync(target.fileno()) + tree.check() + if tree.read(name, maximum=maximum) != raw: + raise ValueError("native retained input readback differs") + + +def _command(argv, *, cwd=None, maximum=4096): + env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "LC_ALL": "C", + "GIT_TERMINAL_PROMPT": "0", "GIT_CONFIG_NOSYSTEM": "1"} + result = subprocess.run(argv, cwd=cwd, env=env, capture_output=True, timeout=15, check=True) + if len(result.stdout) > maximum or len(result.stderr) > maximum: + raise ValueError("native host command output exceeds bound") + return result.stdout.decode("utf-8").strip() + + +def source_record(config): + """Observe the real clean checkout and exact pushed ref, never a supplied SHA alone.""" + root = Path(config["engine_root"]) + if (not root.is_absolute() or root.resolve(strict=True) != root + or root != ENGINE_ROOT): + raise ValueError("native engine checkout path is invalid") + if _command(["git", "status", "--porcelain", "--untracked-files=all"], cwd=root): + raise ValueError("native engine checkout is dirty") + commit = _command(["git", "rev-parse", "HEAD"], cwd=root) + remote = _command(["git", "remote", "get-url", "origin"], cwd=root) + ref = config["engine_ref"] + if (not re.fullmatch(r"refs/heads/[A-Za-z0-9._/-]+", ref) + or ".." in ref or "//" in ref or ref.endswith("/")): + raise ValueError("native engine ref is invalid") + if commit != config["engine_commit"] or remote != config["engine_remote"]: + raise ValueError("native local source selection differs") + remote_line = _command(["git", "ls-remote", "--exit-code", "origin", ref], cwd=root) + if remote_line != f"{commit}\t{ref}": + raise ValueError("native pushed source ref differs") + value = {"schema": "synaptic-native-pushed-source/v1", "url": remote, + "ref": ref, "commit": commit} + return value, hashlib.sha256(canonical(value)).hexdigest() + + +def hardware_record(): + if platform.system() != "Darwin" or platform.machine() != "arm64": + raise ValueError("native host requires Darwin arm64") + value = {"schema": "synaptic-native-hardware-observation/v1", + "machine": platform.machine(), "macos": platform.mac_ver()[0], + "kernel": platform.release(), + "model": _command(["sysctl", "-n", "hw.model"]), + "cpu": _command(["sysctl", "-n", "machdep.cpu.brand_string"]), + "memory_bytes": int(_command(["sysctl", "-n", "hw.memsize"]))} + return value, hashlib.sha256(canonical(value)).hexdigest() + + +def _snapshot(path): + value = _document(path) + if type(value) is not dict or set(value) != {"repository", "revision", "root", "files"}: + raise ValueError("native snapshot fields differ") + snapshot = NativeModelSnapshot(value["repository"], value["revision"], value["root"], + tuple(NativeFile(**item) for item in value["files"])) + with NativeTree(Path(snapshot.root), private=True) as tree: + snapshot.verify(tree) + return snapshot + + +def snapshot_candidate(*, root, repository, revision): + """Inventory existing private bytes; provenance still needs root review.""" + with NativeTree(Path(root), private=True) as tree: + snapshot = NativeModelSnapshot(repository, revision, str(root), tree.inventory()) + snapshot.verify(tree) + tree.check() + return asdict(snapshot) + + +def profile_candidate(config): + """Render unapproved exact profile parameters after source/runtime capture.""" + proposal = strict_json(_read(config["proposal_file"], 1 << 20)) + if (proposal.get("schema") != "synaptic-native-qualification-proposal/v1" + or proposal.get("status") != "ROOT_REVIEW_REQUIRED_NOT_EXECUTION_AUTHORITY"): + raise ValueError("native candidate proposal differs") + _, source_digest = source_record(config) + hardware, _ = hardware_record() + if (hardware["cpu"] != proposal["runtime"]["hardware"] + or hardware["memory_bytes"] != proposal["runtime"]["physical_memory_bytes"]): + raise ValueError("native candidate hardware differs") + capture = _read(config["runtime_capture_file"], 16 << 20) + lock = _read(config["dependency_lock_file"], 1 << 20) + verify_native_material(capture, expected_digest=hashlib.sha256(capture).hexdigest(), + dependency_lock=lock) + observed = strict_json(capture)["runtime"] + selected = proposal["runtime"] + if (observed["python_version"] != selected["python_version"] + or observed["dependency_lock_sha256"] != selected["dependency_lock_sha256"] + or observed["distributions"].get("mlx") != selected["mlx_version"] + or observed["distributions"].get("mlx-lm") != selected["mlx_lm_version"]): + raise ValueError("native candidate runtime differs") + selected_model, training, lora, optimizer = ( + proposal["model"], proposal["training"], proposal["lora"], proposal["optimizer"]) + value = NativeMLXProfile( + profile_ref=proposal["profile_ref"], model_ref=selected_model["repository"], + model_revision=selected_model["revision"], + tokenizer_revision=selected_model["tokenizer_revision"], + runtime_digest=hashlib.sha256(capture).hexdigest(), source_digest=source_digest, + lora_layers=lora["layers"], lora_scale_policy="alpha_over_rank", + beta1=optimizer["beta1"], beta2=optimizer["beta2"], + epsilon=optimizer["epsilon"], weight_decay=optimizer["weight_decay"], + bias_correction=optimizer["bias_correction"], schedule=optimizer["schedule"], + use_fast_tokenizer=config["use_fast_tokenizer"], + steps_per_report=training["steps_per_report"], + steps_per_eval=training["steps_per_eval"], + maximum_updates=training["iterations"], + maximum_batch_tokens=training["max_batch_tokens"], + ) + return asdict(value) + + +class RootSignatureAuthority: + """Verify an external signature using only a pinned public key on this host.""" + + def __init__(self, public_key_path, expected_key_digest, signature_path): + self.key_path = Path(public_key_path) + self.key = _read(self.key_path, 16384) + if hashlib.sha256(self.key).hexdigest() != expected_key_digest: + raise ValueError("reviewed root public key differs") + self.signature_path = Path(signature_path) + self.signature = _read(self.signature_path, 16384) + + def authenticate(self, admission): + if type(admission) is not NativeRestrictedAdmissionV1: + return False + try: + if admission.tag != hashlib.sha256(self.signature).hexdigest(): + return False + content = asdict(admission) + content.pop("tag") + result = subprocess.run( + ["openssl", "dgst", "-sha256", "-verify", str(self.key_path), + "-signature", str(self.signature_path)], + input=canonical(content), capture_output=True, timeout=10, check=False, + ) + return (result.returncode == 0 and result.stdout.strip() == b"Verified OK" + and _read(self.key_path, 16384) == self.key + and _read(self.signature_path, 16384) == self.signature) + except (OSError, subprocess.SubprocessError, ValueError): + return False + + +class _Requests: + def __init__(self, request, resolved, planning): + self.request, self.resolved, self.planning = request, resolved, planning + + def load(self, raw): + if raw != self.request.canonical_json: + raise ValueError("native public request differs") + return self.request + + def resolve(self, request): + if request != self.request: + raise ValueError("native request differs") + return self.resolved + + def for_plan(self, plan): + return self.planning.for_plan(plan) + + +class _Grants: + def __init__(self, selection, catalog, material_authority): + self.s, self.catalog, self.authority = selection, catalog, material_authority + + def authorize(self, requirements): + allowed = (AuthorizationRequirement("training.start", False),) + if tuple(requirements) != allowed: + raise ValueError("native host authorization differs") + return ExecutionGrant(self.s["grant_ref"]) + + def bind(self, grant, *, operation, requirements): + if grant != self.authorize(requirements): + raise ValueError("native host grant differs") + command = parse_exact_command(operation.canonical_bytes) + _, source_digest = source_record(self.s["config"]) + _, hardware_digest = hardware_record() + admission = self.s["admission"] + if (source_digest != admission.source_digest + or hardware_digest != admission.hardware_digest + or self.s["root_authority"].authenticate(admission) is not True): + raise ValueError("native source or hardware admission changed") + if (command.preparation.run_id != self.s["run_id"] + or command.preparation.project_ref != self.s["project_ref"]): + raise ValueError("native host command scope differs") + invocation = NativeInvocation( + self.s["attempt"], command.digest, self.s["executable"], + self.s["executable_digest"], self.s["guardian"], self.s["guardian_digest"], + self.s["worker"], self.s["worker_digest"], self.s["request_path"], + self.s["request_digest"], self.s["private_root"], self.s["limits"], + ) + unsigned = NativeCommandMaterialV1( + operation.canonical_bytes, invocation, self.s["model"], self.s["tokenizer"], + self.s["tokenizer_members"], self.s["output"], self.s["budget"], + self.s["material_authority_ref"], self.s["material_key_ref"], "0" * 64, + ) + self.catalog.publish_once(self.authority.issue(unsigned)) + return grant + + +def inspect(config, *, require_admission=False, trusted_root_key_sha256=None): + """Rebuild source, runtime, prepared publication and snapshot bindings.""" + fields = { + "schema", "proposal_file", "engine_root", "engine_remote", "engine_ref", "engine_commit", + "runtime_capture_file", "dependency_lock_file", "profile_file", "training_input_file", + "prepared_identity_file", "prepared_publication", "model_snapshot_file", + "tokenizer_snapshot_file", "admission_file", "admission_signature_file", + "root_public_key_file", "root_public_key_sha256", "limits", "scope", "run_id", + "project_ref", "request_id", "attempt", "attempt_kind", "state_root", "host", + "tokenizer_members", "artifact_budget", "use_fast_tokenizer", + } + if (type(config) is not dict or set(config) != fields + or config.get("schema") != "synaptic-native-host-config/v1"): + raise ValueError("native host configuration invalid") + if (type(config["host"]) is not dict or set(config["host"]) != { + "authority_secret", "evidence_secret", "material_secret", "grant_ref", + "reader_authority_ref", "reader_key_ref", "material_authority_ref", "material_key_ref", + } or type(config["tokenizer_members"]) is not list + or tuple(config["tokenizer_members"]) != tuple(sorted(set(config["tokenizer_members"])))): + raise ValueError("native host authority or tokenizer selection invalid") + source, source_digest = source_record(config) + hardware, hardware_digest = hardware_record() + proposal = strict_json(_read(config["proposal_file"], 1 << 20)) + if (proposal.get("schema") != "synaptic-native-qualification-proposal/v1" + or proposal.get("status") != "ROOT_REVIEW_REQUIRED_NOT_EXECUTION_AUTHORITY" + or hardware["cpu"] != proposal["runtime"]["hardware"] + or hardware["memory_bytes"] != proposal["runtime"]["physical_memory_bytes"]): + raise ValueError("native hardware differs from reviewed candidate") + capture = _read(config["runtime_capture_file"], 16 << 20) + lock = _read(config["dependency_lock_file"], 1 << 20) + capture_digest = hashlib.sha256(capture).hexdigest() + verify_native_material(capture, expected_digest=capture_digest, dependency_lock=lock) + observed_runtime = strict_json(capture)["runtime"] + selected_runtime = proposal["runtime"] + if (observed_runtime["python_version"] != selected_runtime["python_version"] + or observed_runtime["dependency_lock_sha256"] != selected_runtime["dependency_lock_sha256"] + or observed_runtime["distributions"].get("mlx") != selected_runtime["mlx_version"] + or observed_runtime["distributions"].get("mlx-lm") != selected_runtime["mlx_lm_version"]): + raise ValueError("native runtime differs from reviewed candidate") + runtime = native_runtime_spec(capture) + if config["engine_root"] + "/tuner" not in strict_json(capture)["config"]["sources"]: + raise ValueError("native captured source is not selected checkout") + profile = NativeMLXProfile.from_dict(_document(config["profile_file"])) + selected_model = proposal["model"] + if (profile.source_digest != source_digest or profile.runtime_digest != capture_digest + or profile.profile_ref != proposal["profile_ref"] + or profile.model_ref != selected_model["repository"] + or profile.model_revision != selected_model["revision"] + or profile.tokenizer_revision != selected_model["tokenizer_revision"] + or profile.use_fast_tokenizer is not config["use_fast_tokenizer"]): + raise ValueError("native profile source/runtime differs") + training = TrainingInputV1.from_dict(_document(config["training_input_file"])) + identity = PreparedTrainingInputIdentity.from_dict(_document(config["prepared_identity_file"])) + prepared = LocalPreparedTrainingInputSource(Path(config["prepared_publication"]), identity) + dataset = read_retained_native_source(prepared, identity) + rows = [strict_json(line, maximum=1 << 20) for line in dataset.splitlines()] + counts = {name: sum(row.get("split") == name for row in rows) + for name in ("train", "validation")} + if (counts != proposal["input_review"]["rows_per_attempt"] + or sum(counts.values()) != len(rows)): + raise ValueError("native synthetic publication split coverage differs") + kind = config["attempt_kind"] + hp = training.hyperparameters + if kind == "raw_full_sequence": + if (identity.format != "syntunia-sft-row/v1" or hp.dataset_format != "raw_text" + or hp.prompt_render != "full_conversation" or hp.chat_template_kwargs is not None): + raise ValueError("native raw attempt input differs") + elif kind == "prompt_completion_assistant": + if (identity.format not in {"syntunia-sft-row/v2", "synaptic-conversation-sft-row/v1"} + or hp.dataset_format != "messages" or hp.prompt_render != "prompt_completion" + or hp.chat_template_kwargs != {"enable_thinking": False}): + raise ValueError("native prompt/completion attempt input differs") + else: + raise ValueError("native attempt kind is invalid") + model, tokenizer = _snapshot(config["model_snapshot_file"]), _snapshot(config["tokenizer_snapshot_file"]) + if (model.repository != selected_model["repository"] + or model.revision != selected_model["revision"] + or tokenizer.repository != selected_model["tokenizer_repository"] + or tokenizer.revision != selected_model["tokenizer_revision"] + or sum(item.size for item in model.files) > selected_model["download_budget_bytes"] + or tuple(config["tokenizer_members"]) != tuple(item.path for item in tokenizer.files)): + raise ValueError("native complete model or tokenizer snapshot differs") + projection = compile_native_sft(training, identity, profile) + proposed_training, proposed_lora, proposed_optimizer = ( + proposal["training"], proposal["lora"], proposal["optimizer"]) + if (projection.controls.iterations != proposed_training["iterations"] + or projection.controls.batch_size != proposed_training["batch_size"] + or projection.controls.max_seq_length != proposed_training["max_seq_length"] + or projection.controls.seed != proposed_training["seed"] + or projection.controls.steps_per_save != proposed_training["steps_per_save"] + or projection.lora.layers != proposed_lora["layers"] + or projection.lora.rank != proposed_lora["rank"] + or projection.lora.scale != proposed_lora["scale"] + or projection.lora.dropout != proposed_lora["dropout"] + or tuple(projection.lora.targets) != tuple(proposed_lora["targets"]) + or projection.optimizer.learning_rate != proposed_optimizer["learning_rate"] + or projection.optimizer.beta1 != proposed_optimizer["beta1"] + or projection.optimizer.beta2 != proposed_optimizer["beta2"] + or projection.optimizer.epsilon != proposed_optimizer["epsilon"] + or projection.optimizer.weight_decay != proposed_optimizer["weight_decay"] + or projection.optimizer.bias_correction != proposed_optimizer["bias_correction"] + or projection.optimizer.schedule != proposed_optimizer["schedule"]): + raise ValueError("native attempt controls differ from reviewed candidate") + resolved = ResolvedTrainingRequest( + "synaptic-resolved-training-request/v1", config["request_id"], config["project_ref"], + source_digest, hashlib.sha256(training.canonical_json().encode()).hexdigest(), + projection.workload.fingerprint, _domain("synaptic-coordinator-runtime/v1", _runtime(runtime)), + _domain("synaptic-coordinator-artifact-policy/v1", training.to_dict()["artifacts"]), + ) + limits = NativeProcessLimits(**config["limits"]) + proposed_bounds = proposal["bounds"] + if (limits.timeout_seconds != proposed_bounds["timeout_seconds_per_attempt"] + or limits.termination_grace_seconds != proposed_bounds["termination_grace_seconds"] + or limits.maximum_output_bytes != proposed_bounds["maximum_output_bytes_per_stream"] + or limits.maximum_file_bytes != proposed_bounds["maximum_artifact_file_bytes"] + or config["artifact_budget"] != { + "per_file": proposed_bounds["maximum_artifact_file_bytes"], + "aggregate": proposed_bounds["maximum_artifact_set_bytes"], + }): + raise ValueError("native attempt bounds differ from reviewed candidate") + scope = ExecutionScopeV1(**config["scope"]) + run = TrainingRunRef(config["run_id"], config["project_ref"]) + base = dict(resolved=resolved, run=run, workload=projection.workload.canonical_bytes, + runtime_capture=capture, limits=limits, scope=scope, clock=Clock()) + candidate = NativePlanningV1(**base) + admission_fields = dict( + profile_digest=candidate.profile_digest, source_digest=source_digest, + runtime_digest=resolved.runtime_digest, hardware_digest=hardware_digest, + prepared_digest=identity.content_digest, model_digest=model.digest, + tokenizer_digest=tokenizer.digest, run_id=run.run_id, attempt=config["attempt"], + maximum_updates=projection.controls.iterations, + ) + if not require_admission: + return {"source": source, "hardware": hardware, "admission_candidate": admission_fields, + "capture_digest": capture_digest, "prepared_rows_bytes": len(dataset)}, None + admission_value = _document(config["admission_file"]) + if set(admission_value) != set(NativeRestrictedAdmissionV1.__dataclass_fields__): + raise ValueError("restricted admission fields differ") + admission = NativeRestrictedAdmissionV1(**admission_value) + if any(getattr(admission, name) != value for name, value in admission_fields.items()): + raise ValueError("restricted admission selection differs") + if trusted_root_key_sha256 is None or trusted_root_key_sha256 != config["root_public_key_sha256"]: + raise ValueError("independent root key pin is required") + root = RootSignatureAuthority(config["root_public_key_file"], + trusted_root_key_sha256, config["admission_signature_file"]) + if not root.authenticate(admission): + raise ValueError("external restricted admission signature invalid") + planning = NativePlanningV1(**base, restricted_admission=admission, + restricted_authority=root) + if not planning.restricted(): + raise ValueError("restricted native admission expired or refused") + return {"source": source, "hardware": hardware, "admission_candidate": admission_fields, + "capture_digest": capture_digest, "prepared_rows_bytes": len(dataset)}, ( + planning, training, projection, dataset, lock, model, tokenizer, + ) + + +def operate(config, *, action="start", destination=None, trusted_root_key_sha256=None): + """Start once, or recompose the same selected run for a cold public read.""" + if action not in {"start", "show", "observe", "verify", "download"}: + raise ValueError("native public action is unavailable") + _, selected = inspect(config, require_admission=True, + trusted_root_key_sha256=trusted_root_key_sha256) + planning, training, projection, dataset, lock, model, tokenizer = selected + state = Path(config["state_root"]) + if action == "start": + _create_private_dir(state) + _private(state) + private = state / "native" + if action == "start": + _create_private_dir(private) + _private(private) + output = private / "output" + if action == "start": + _create_private_dir(output) + _private(output) + if action == "start": + _write_owned(private, "capture.json", planning.runtime_capture, 16 << 20) + _write_owned(private, "dependency.lock", lock, 1 << 20) + _write_owned(private, "dataset.jsonl", dataset, 64 << 20) + elif (_read(private / "capture.json", 16 << 20) != planning.runtime_capture + or _read(private / "dependency.lock", 1 << 20) != lock + or _read(private / "dataset.jsonl", 64 << 20) != dataset): + raise ValueError("retained native input differs") + request_raw = canonical({ + "schema": "synaptic-native-child-request/v1", "workload": projection.workload.document, + "runtime_capture": str(private / "capture.json"), "dependency_lock": str(private / "dependency.lock"), + "dataset": str(private / "dataset.jsonl"), "output": str(output), + "model_snapshot": asdict(model), "tokenizer_snapshot": asdict(tokenizer), + "tokenizer_members": config["tokenizer_members"], + "artifact_budget": config["artifact_budget"], + }) + selection_document = {"plan": planning.plan.to_dict(), + "admission": asdict(planning._restricted_admission)} + if action == "start": + _write_owned(private, "request.json", request_raw, 2 << 20) + _write_owned(state, "config.json", canonical(config), 4 << 20) + _write_owned(state, "selection.json", canonical(selection_document), 2 << 20) + _create_private_dir(state / "scratch") + elif (_read(private / "request.json", 2 << 20) != request_raw + or _document(state / "config.json", 4 << 20) != config + or _document(state / "selection.json") != selection_document): + raise ValueError("retained native selection differs") + _private(state / "scratch") + secrets, clock = NamedSecrets(), Clock() + with SQLiteDurableStoreV1(state / "workflow.sqlite3", create=action == "start") as store: + key = secrets.resolve(SecretRef.from_dict(config["host"]["evidence_secret"])).encode() + material_key = secrets.resolve(SecretRef.from_dict(config["host"]["material_secret"])).encode() + host = config["host"] + material_authority = NativeMaterialAuthority(host["material_authority_ref"], + host["material_key_ref"], HMACAuthenticator({host["material_key_ref"]: material_key}, + allowed_purposes=frozenset({"native-command-material/v1"}))) + evidence = NativeEvidenceAuthority(host["reader_authority_ref"], host["reader_key_ref"], + HMACAuthenticator({host["reader_key_ref"]: key}, allowed_purposes=frozenset({ + "provider-run-observation/v1", "provider-log-page/v1", + "native-artifact-verification/v1", "native-process-outcome/v1", + }))) + catalog = NativeCommandCatalogV1(store, material_authority) + capture = strict_json(planning.runtime_capture) + members = {str(Path(tree["root"]) / item["path"]): item["sha256"] + for tree in capture["trees"] for item in tree["members"]} + worker = config["engine_root"] + "/tuner/runtime/native_mlx_child.py" + guardian = config["engine_root"] + "/tuner/runtime/native_process_guardian.py" + selection = dict(run_id=config["run_id"], project_ref=config["project_ref"], + grant_ref=host["grant_ref"], attempt=config["attempt"], config=config, + admission=planning._restricted_admission, + root_authority=planning._restricted_authority, + executable=capture["executable"], executable_digest=capture["runtime"]["executable_sha256"], + worker=worker, worker_digest=members[worker], guardian=guardian, guardian_digest=members[guardian], + request_path=str(private / "request.json"), request_digest=hashlib.sha256(request_raw).hexdigest(), + private_root=str(private), output=str(output), limits=planning.limits, + model=model, tokenizer=tokenizer, tokenizer_members=tuple(config["tokenizer_members"]), + budget=NativeArtifactBudget(**config["artifact_budget"]), + material_authority_ref=host["material_authority_ref"], material_key_ref=host["material_key_ref"]) + ports = ReferenceHostPortsV1(store, store, clock, _Grants(selection, catalog, material_authority), secrets) + authority = compose_reference_authority( + clock=clock, secrets=secrets, + authority_secret=SecretRef.from_dict(host["authority_secret"]), + ) + family = native_mlx_family( + planning=planning, catalog=catalog, material_authority=material_authority, + records=store, evidence_authority=evidence, + foundation_authenticator=authority.foundation_authenticator, + assessment_authenticator=authority.assessment_authority, clock=clock, + ) + request = TrainingRequest(config["request_id"], config["project_ref"], training.canonical_json()) + requests = _Requests(request, planning.resolved, planning) + api = compose_reference_host( + family=family, ports=ports, + requests=ReferenceRequestPortsV1(requests, requests, requests), authority=authority, + ).api() + if action == "start": + plan = api.training.plan(api.training.resolve(api.training.load(request.canonical_json)), planning.provider) + preflight = api.training.preflight(plan) + if not preflight.ready or not preflight.binds(plan): + raise ValueError("restricted native public preflight refused") + started = api.training.start(plan, preflight) + return {"run": started.run.to_dict(), "accepted": started.accepted, + "stored": api.runs.show(started.run).to_dict()} + run = planning.run + if action == "show": + return api.runs.show(run).to_dict() + if action == "observe": + return api.runs.outcome(run).to_dict() + if action == "verify": + report = api.runs.verify(run) + return {"run": report.run.to_dict(), "verified": report.verified, + "checked_at": report.checked_at} + if destination is None: + raise ValueError("native artifact destination required") + return {"paths": [str(path) for path in download(api.runs, run, destination, + maximum_bytes=64 << 20)]} + + +def main(argv=None): + import argparse + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("canonicalize", "profile-candidate", "inspect", + "admission-candidate", "seal-admission", "start", + "show", "observe", "verify", "download", + "snapshot-candidate")) + parser.add_argument("config", type=Path, nargs="?") + parser.add_argument("--destination", type=Path) + parser.add_argument("--trusted-root-key-sha256") + parser.add_argument("--expires-at") + parser.add_argument("--authority-ref") + parser.add_argument("--key-ref") + parser.add_argument("--signature", type=Path) + parser.add_argument("--snapshot-root", type=Path) + parser.add_argument("--repository") + parser.add_argument("--revision") + args = parser.parse_args(argv) + try: + if args.action == "snapshot-candidate": + if args.config is not None or args.snapshot_root is None or not args.repository or not args.revision: + raise ValueError("native snapshot candidate arguments differ") + result = snapshot_candidate(root=args.snapshot_root, repository=args.repository, + revision=args.revision) + else: + if args.config is None: + raise ValueError("native host configuration required") + config = (strict_json(_read(args.config, 4 << 20), maximum=4 << 20) + if args.action == "canonicalize" else _document(args.config, 4 << 20)) + if args.action == "snapshot-candidate": + pass + elif args.action == "canonicalize": + result = config + elif args.action == "profile-candidate": + result = profile_candidate(config) + elif args.action == "inspect": + result = inspect(config)[0] + elif args.action == "admission-candidate": + if not all((args.expires_at, args.authority_ref, args.key_ref)): + raise ValueError("reviewer admission fields required") + result = {**inspect(config)[0]["admission_candidate"], + "expires_at": args.expires_at, "authority_ref": args.authority_ref, + "key_ref": args.key_ref} + NativeRestrictedAdmissionV1(**result, tag="0" * 64) + elif args.action == "seal-admission": + if args.signature is None: + raise ValueError("external signature file required") + if set(config) != set(NativeRestrictedAdmissionV1.__dataclass_fields__) - {"tag"}: + raise ValueError("unsigned admission fields differ") + result = {**config, "tag": hashlib.sha256(_read(args.signature, 16384)).hexdigest()} + NativeRestrictedAdmissionV1(**result) + else: + result = operate(config, action=args.action, destination=args.destination, + trusted_root_key_sha256=args.trusted_root_key_sha256) + except Exception: + print(json.dumps({"diagnostic_code": "native_host_refused"})) + return 2 + print(json.dumps(result, sort_keys=True, separators=(",", ":"), allow_nan=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/cli/test_native_host.py b/tests/cli/test_native_host.py new file mode 100644 index 00000000..47d3ffd0 --- /dev/null +++ b/tests/cli/test_native_host.py @@ -0,0 +1,98 @@ +"""Host trust boundary tests; no ML package import or Mac execution.""" + +from dataclasses import asdict, replace +import hashlib +from pathlib import Path +import subprocess +import tempfile + +import pytest + +from examples.training_qualification import native_host +from tuner.runtime.native_mlx_material import canonical +from tuner.training.native_mlx_planning import NativeRestrictedAdmissionV1 + + +def test_external_root_signature_is_required_and_binds_exact_admission(tmp_path): + private = tmp_path / "root-private.pem" + public = tmp_path / "root-public.pem" + signature = tmp_path / "admission.sig" + subprocess.run(["openssl", "genpkey", "-algorithm", "EC", "-pkeyopt", + "ec_paramgen_curve:P-256", "-out", str(private)], check=True, + capture_output=True) + subprocess.run(["openssl", "pkey", "-in", str(private), "-pubout", "-out", str(public)], + check=True, capture_output=True) + candidate = NativeRestrictedAdmissionV1( + *("a" * 64 for _ in range(7)), "run", "attempt-1", 3, + "2026-10-20T12:30:00Z", "root-review", "root-key", "0" * 64, + ) + signed = asdict(candidate) + signed.pop("tag") + subprocess.run(["openssl", "dgst", "-sha256", "-sign", str(private), + "-out", str(signature)], input=canonical(signed), check=True, + capture_output=True) + proof = replace(candidate, tag=hashlib.sha256(signature.read_bytes()).hexdigest()) + trusted = hashlib.sha256(public.read_bytes()).hexdigest() + authority = native_host.RootSignatureAuthority(public, trusted, signature) + assert authority.authenticate(proof) + assert not authority.authenticate(replace(proof, run_id="another-run")) + with pytest.raises(ValueError, match="reviewed root public key"): + native_host.RootSignatureAuthority(public, "f" * 64, signature) + signature.write_bytes(b"replacement") + assert not authority.authenticate(proof) + + +def test_source_record_requires_clean_exact_pushed_ref(tmp_path, monkeypatch): + root = tmp_path.resolve() + monkeypatch.setattr(native_host, "ENGINE_ROOT", root) + commit = "a" * 40 + config = {"engine_root": str(root), "engine_commit": commit, + "engine_remote": "https://example.invalid/engine.git", + "engine_ref": "refs/heads/native-review"} + observations = { + ("status", "--porcelain", "--untracked-files=all"): "", + ("rev-parse", "HEAD"): commit, + ("remote", "get-url", "origin"): config["engine_remote"], + ("ls-remote", "--exit-code", "origin", config["engine_ref"]): + f"{commit}\t{config['engine_ref']}", + } + + def command(argv, **_): + return observations[tuple(argv[1:])] + + monkeypatch.setattr(native_host, "_command", command) + record, digest = native_host.source_record(config) + assert record["commit"] == commit and digest == hashlib.sha256(canonical(record)).hexdigest() + observations[("status", "--porcelain", "--untracked-files=all")] = "?? stray.py" + with pytest.raises(ValueError, match="dirty"): + native_host.source_record(config) + observations[("status", "--porcelain", "--untracked-files=all")] = "" + observations[("ls-remote", "--exit-code", "origin", config["engine_ref"])] = "b" * 40 + with pytest.raises(ValueError, match="pushed source"): + native_host.source_record(config) + + +def test_snapshot_candidate_inventories_complete_owned_bytes(): + import os + original_umask = os.umask(0o077) + try: + with tempfile.TemporaryDirectory(prefix="native-host-snapshot-", dir=Path.home()) as name: + root = Path(name) + (root / "config.json").write_bytes(b'{"num_hidden_layers":2}') + (root / "model.safetensors").write_bytes(b"candidate-bytes") + candidate = native_host.snapshot_candidate(root=root, repository="configured/model", + revision="a" * 40) + assert tuple(item["path"] for item in candidate["files"]) == ( + "config.json", "model.safetensors", + ) + manifest = root.parent / (root.name + "-manifest.json") + try: + manifest.write_bytes(canonical(candidate)) + native_host._snapshot(manifest) + (root / "model.safetensors").write_bytes(b"drift") + with pytest.raises(ValueError, match="snapshot"): + native_host._snapshot(manifest) + finally: + manifest.unlink(missing_ok=True) + finally: + os.umask(original_umask) diff --git a/tests/runtime/test_native_mlx_child.py b/tests/runtime/test_native_mlx_child.py index c86db67b..607dd149 100644 --- a/tests/runtime/test_native_mlx_child.py +++ b/tests/runtime/test_native_mlx_child.py @@ -5,7 +5,7 @@ import subprocess import sys import tempfile -from types import ModuleType +from types import ModuleType, SimpleNamespace import pytest @@ -133,16 +133,24 @@ def test_worker_recompiles_and_joins_workload_before_invoking_runtime(private_ro output.mkdir(mode=0o700) for name, content in (("capture.json", b"capture"), ("lock", b"lock"), ("dataset.jsonl", b"dataset")): (private_root / name).write_bytes(content) - snapshot = { - "repository": "configured/model", "revision": "c" * 40, - "root": str(private_root / "model"), - "files": [{"path": "config.json", "size": 2, "sha256": "f" * 64}], - } + from dataclasses import asdict + from tuner.runtime.native_files import NativeTree + from tuner.runtime.native_mlx_input import NativeModelSnapshot + model_root = private_root / "model" + model_root.mkdir(mode=0o700) + (model_root / "config.json").write_bytes(b"{}") + with NativeTree(model_root, private=True) as model_tree: + snapshot = asdict(NativeModelSnapshot( + "configured/model", "c" * 40, str(model_root), model_tree.inventory(), + )) + snapshot["files"] = list(snapshot["files"]) request = { "schema": "synaptic-native-child-request/v1", "workload": projection.workload.document, "runtime_capture": str(private_root / "capture.json"), "dependency_lock": str(private_root / "lock"), "dataset": str(private_root / "dataset.jsonl"), "model_snapshot": snapshot, "tokenizer_snapshot": snapshot, "output": str(output), + "tokenizer_members": ["config.json"], + "artifact_budget": {"per_file": 1024 * 1024, "aggregate": 4 * 1024 * 1024}, } observed = {} @@ -157,6 +165,9 @@ def execute(**kwargs): monkeypatch.setattr(native_mlx_input, "prepare_native_input", prepare) monkeypatch.setattr(native_mlx_composition, "execute_native_sft", execute) + from tuner.runtime import native_mlx_publication, native_mlx_child + monkeypatch.setattr(native_mlx_publication, "tokenizer_publication_bytes", lambda *a, **k: b"fixture") + monkeypatch.setattr(native_mlx_child, "_load_published_tokenizer", lambda *a, **k: None) assert run_request(request, request_root=private_root) == b"verified-fixture-result" assert observed["workload_digest"] == projection.workload.fingerprint assert observed["runtime_digest"] == projection.profile.runtime_digest @@ -171,3 +182,25 @@ def test_child_requires_isolation_and_has_closed_diagnostics(private_root): capture_output=True, timeout=5, ) assert result.returncode == 120 and result.stdout == result.stderr == b"" + + +def test_child_loads_only_complete_published_tokenizer_bytes(private_root, monkeypatch): + from tuner.runtime.native_mlx_child import _load_published_tokenizer + from tuner.runtime.native_mlx_publication import _archive, NativeArtifactBudget + loaded = [] + transformers = ModuleType("transformers") + def from_pretrained(root, **kwargs): + loaded.append(((Path(root) / "tokenizer.json").read_bytes(), kwargs)) + return object() + transformers.AutoTokenizer = SimpleNamespace(from_pretrained=from_pretrained) + monkeypatch.setitem(sys.modules, "transformers", transformers) + payload = _archive({"tokenizer.json": b"complete", "native_tokenizer.json": b"{}"}, + NativeArtifactBudget(1024, 16384)) + _load_published_tokenizer(payload, request_root=private_root, + expected_members=("tokenizer.json",), use_fast=True) + assert loaded == [(b"complete", {"local_files_only": True, + "trust_remote_code": False, "use_fast": True})] + with pytest.raises(ValueError, match="archive inventory"): + _load_published_tokenizer(payload, request_root=private_root, + expected_members=("tokenizer.json", "missing.json"), use_fast=True) + assert not list(private_root.glob("tokenizer-publication-*")) diff --git a/tests/runtime/test_native_mlx_execution.py b/tests/runtime/test_native_mlx_execution.py index 14cdd860..50dcc307 100644 --- a/tests/runtime/test_native_mlx_execution.py +++ b/tests/runtime/test_native_mlx_execution.py @@ -224,7 +224,8 @@ def convert(model, *args, **kwargs): convert=convert, flatten=lambda v: v.items()) -def test_composition_evaluates_reloaded_saved_final_adapter_after_last_update(monkeypatch): +@pytest.mark.parametrize("reload_mode", ["complete", "noop", "partial"]) +def test_composition_evaluates_reloaded_saved_final_adapter_after_last_update(monkeypatch, reload_mode): from tuner.runtime import native_mlx_composition as composition from tuner.runtime.native_files import NativeTree @@ -239,17 +240,43 @@ def test_composition_evaluates_reloaded_saved_final_adapter_after_last_update(mo with NativeTree(snapshot_root) as tree: snapshot = NativeModelSnapshot("configured/model", "a" * 40, str(snapshot_root), tree.inventory()) events = [] - shapes = (("layers.0.q_proj.lora_a", (1, 1)),) - tensors = {"layers.0.q_proj.lora_a": object()} - model = SimpleNamespace( - load_weights=lambda values, strict: events.append(("reload", values, strict)), - parameters=lambda: {}, - ) + names = ("layers.0.q_proj.lora_a", "layers.0.q_proj.lora_b") + shapes = tuple((name, (1, 1)) for name in names) + def adapter_bytes(value): + headers = {name: {"dtype": "F32", "shape": [1, 1], + "data_offsets": [4 * index, 4 * (index + 1)]} + for index, name in enumerate(names)} + raw_header = json.dumps(headers, sort_keys=True, separators=(",", ":")).encode() + return struct.pack(" budget.aggregate: - raise ValueError("native tokenizer aggregate exceeds budget") - tokenizer_files[name] = raw - tokenizer_files["native_tokenizer.json"] = canonical({ - "schema": "synaptic-native-tokenizer/v1", "repository": tokenizer.repository, - "revision": tokenizer.revision, "snapshot_digest": tokenizer.digest, - "tokenization": projection.tokenization.to_dict(), - }) + tokenizer_archive = tokenizer_publication_bytes( + tokenizer, tokenizer_members, projection.tokenization, budget, token_tree, + ) roles = { "final_model": _archive(model_members, budget), - "tokenizer": _archive(tokenizer_files, budget), + "tokenizer": tokenizer_archive, "workload_record": workload, "training_metrics": files["native_result.json"], "training_lineage": canonical({ @@ -217,6 +241,7 @@ def verify_native_publication( or any(len(raw) > budget.per_file for raw in roles.values())): raise ValueError("native public role byte budget exceeded") token_tree.check() + model_tree.check() tree.check() return tuple(NativePublishedArtifact( VerifiedArtifact(role, hashlib.sha256(raw).hexdigest(), len(raw)), raw, diff --git a/tuner/training/native_mlx_family.py b/tuner/training/native_mlx_family.py index abf48d50..61759899 100644 --- a/tuner/training/native_mlx_family.py +++ b/tuner/training/native_mlx_family.py @@ -25,10 +25,11 @@ from tuner.execution.foundation_v2.references import ProviderStageRefV1, ScopedProviderRunRefV1 from tuner.runtime.native_files import NativeFile, NativeTree from tuner.runtime.native_mlx_input import NativeModelSnapshot -from tuner.runtime.native_mlx_material import canonical, strict_json +from tuner.runtime.native_mlx_material import canonical, strict_json, verify_native_material from tuner.runtime.native_mlx_publication import NativeArtifactBudget, verify_native_publication from tuner.runtime.native_process import NativeInvocation, NativeOwnedProcess, NativeProcessLimits from tuner.training.native_mlx_planning import NativePlanningV1 +from tuner.training.native_mlx_compilation import parse_native_workload @dataclass(frozen=True) @@ -56,7 +57,7 @@ def __post_init__(self): or type(self.model) is not NativeModelSnapshot or type(self.tokenizer) is not NativeModelSnapshot or type(self.artifact_budget) is not NativeArtifactBudget or type(self.tokenizer_members) is not tuple or not self.tokenizer_members - or self.tokenizer_members != tuple(sorted(set(self.tokenizer_members))) + or self.tokenizer_members != tuple(item.path for item in self.tokenizer.files) or any(type(name) is not str for name in self.tokenizer_members) or type(self.output) is not str or Path(self.output).parent != Path(self.invocation.private_root)): @@ -196,6 +197,8 @@ def _material(self, command_digest): or canonical(request["workload"]) != self.planning.workload or canonical(request["model_snapshot"]) != canonical(asdict(material.model)) or canonical(request["tokenizer_snapshot"]) != canonical(asdict(material.tokenizer)) + or tuple(request["tokenizer_members"]) != material.tokenizer_members + or canonical(request["artifact_budget"]) != canonical(asdict(material.artifact_budget)) or request["output"] != material.output): raise ValueError("native_request_material_differs") for key in ("runtime_capture", "dependency_lock", "dataset", "output"): @@ -230,8 +233,12 @@ def _material(self, command_digest): return material, command def execute_once(self, payload, request): - self.planning.require_qualified() material, command = self._material(request.command_digest) + self.planning.require_admitted( + attempt=material.invocation.attempt, + model_digest=material.model.digest, + tokenizer_digest=material.tokenizer.digest, + ) expected_payload = self.planning.payload(command.preparation, command.operation.effect.kind) if (payload != expected_payload or request.descriptor_digest != self.descriptor.digest or (request.provider_id, request.profile_ref, request.account_ref, request.namespace_ref) @@ -254,6 +261,24 @@ def execute_once(self, payload, request): "native-stage-" + command.digest, ), ) + # Keep this outside the dispatch/uncertainty catch: drift discovered + # before run_once has not consumed a process claim. + with NativeTree(Path(material.invocation.private_root), private=True) as root: + child_request = strict_json(root.read(Path(material.invocation.request).name, + maximum=2 * 1024 * 1024)) + lock = root.read(Path(child_request["dependency_lock"]).name, maximum=1024 * 1024) + verify_native_material( + self.planning.runtime_capture, + expected_digest=parse_native_workload(self.planning.workload).profile.runtime_digest, + dependency_lock=lock, + ) + root.check() + with NativeTree(Path(material.model.root), private=True) as model_tree: + material.model.verify(model_tree) + model_tree.check() + with NativeTree(Path(material.tokenizer.root), private=True) as tokenizer_tree: + material.tokenizer.verify(tokenizer_tree) + tokenizer_tree.check() try: outcome = self.transport.run_once(material.invocation) terminal = self.authority.process_outcome(material.invocation.digest, outcome) diff --git a/tuner/training/native_mlx_planning.py b/tuner/training/native_mlx_planning.py index 1bf366b4..df057a62 100644 --- a/tuner/training/native_mlx_planning.py +++ b/tuner/training/native_mlx_planning.py @@ -64,13 +64,50 @@ def __post_init__(self): safe_ref(self.key_ref, "key_ref") +@dataclass(frozen=True) +class NativeRestrictedAdmissionV1: + """An independently authenticated grant for one initial hardware attempt. + + This is not a qualification receipt. Its verifier is supplied by the host; + the engine neither issues admissions nor trusts a digest as approval. + """ + + profile_digest: str + source_digest: str + runtime_digest: str + hardware_digest: str + prepared_digest: str + model_digest: str + tokenizer_digest: str + run_id: str + attempt: str + maximum_updates: int + expires_at: str + authority_ref: str + key_ref: str + tag: str + + def __post_init__(self): + for name in ("profile_digest", "source_digest", "runtime_digest", "hardware_digest", + "prepared_digest", "model_digest", "tokenizer_digest", "tag"): + digest_text(getattr(self, name), name) + for name in ("run_id", "attempt", "authority_ref", "key_ref"): + safe_ref(getattr(self, name), name) + if type(self.maximum_updates) is not int or not 1 <= self.maximum_updates <= 1000000: + raise ValueError("restricted native update bound is invalid") + expiry = datetime.fromisoformat(self.expires_at.replace("Z", "+00:00")) + if expiry.tzinfo is None or expiry.utcoffset() is None: + raise ValueError("restricted native admission needs a timezone") + + class NativePlanningV1: """One immutable native material selection; reusable across host restarts.""" __slots__ = ( "resolved", "run", "workload", "runtime_capture", "limits", "scope", "provider", "profile_digest", "_clock", "_qualification", - "_qualification_authority", "descriptor", "executor_descriptor", + "_qualification_authority", "_restricted_admission", "_restricted_authority", + "descriptor", "executor_descriptor", "adapter_descriptor", "binding", "context_value", "plan", "_sealed", ) @@ -83,6 +120,7 @@ def __init__( self, *, resolved: ResolvedTrainingRequest, run: TrainingRunRef, workload: bytes, runtime_capture: bytes, limits: NativeProcessLimits, scope: ExecutionScopeV1, clock, qualification=None, qualification_authority=None, + restricted_admission=None, restricted_authority=None, ): if (type(resolved) is not ResolvedTrainingRequest or type(run) is not TrainingRunRef or type(limits) is not NativeProcessLimits or type(scope) is not ExecutionScopeV1): @@ -108,10 +146,11 @@ def __init__( }) self._clock = clock self._qualification, self._qualification_authority = qualification, qualification_authority - qualified = self.qualified() + self._restricted_admission, self._restricted_authority = restricted_admission, restricted_authority + available = self.admitted() self.descriptor = ProviderDescriptor( "synaptic-provider-descriptor/v1", "native-mlx", "Apple native MLX", "1.0.0", - ProviderCapabilities(qualified, False, False, False, qualified, False), + ProviderCapabilities(available, False, False, False, available, False), ) self.executor_descriptor = ExecutorDescriptorV1("native-mlx", "owned-native-process", "1.0.0") self.adapter_descriptor = AdapterDescriptorV1("native-mlx", "no-replay-native-reconciliation", "1.0.0") @@ -149,6 +188,38 @@ def require_qualified(self): if not self.qualified(): raise ValueError("native_hardware_qualification_required") + def restricted(self): + proof = self._restricted_admission + try: + workload = parse_native_workload(self.workload) + now = datetime.fromisoformat(self._clock.now().replace("Z", "+00:00")) + expires = datetime.fromisoformat(proof.expires_at.replace("Z", "+00:00")) + return ( + type(proof) is NativeRestrictedAdmissionV1 + and proof.profile_digest == self.profile_digest + and proof.source_digest == self.resolved.source_digest + and proof.runtime_digest == self.resolved.runtime_digest + and proof.prepared_digest == workload.source.content_digest + and proof.run_id == self.run.run_id + and proof.maximum_updates == workload.controls.iterations + and now < expires <= now + timedelta(hours=1) + and self._restricted_authority.authenticate(proof) is True + ) + except Exception: + return False + + def admitted(self): + return self.qualified() or self.restricted() + + def require_admitted(self, *, attempt=None, model_digest=None, tokenizer_digest=None): + if self.qualified(): + return + proof = self._restricted_admission + if (not self.restricted() or proof.attempt != attempt + or proof.model_digest != model_digest + or proof.tokenizer_digest != tokenizer_digest): + raise ValueError("native_restricted_admission_required") + def describe(self, provider): if provider != self.provider: raise ValueError("native provider differs") @@ -174,7 +245,7 @@ def preflight(self, plan): raise ValueError("native plan differs") now = self._clock.now() expiry = (datetime.fromisoformat(now.replace("Z", "+00:00")) + timedelta(minutes=15)) - ready = self.qualified() + ready = self.admitted() return TrainingPreflight( plan.plan_fingerprint, ready, now, expiry.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"), From c10836f00afcc2a1f3407b9e22acae4206ab7937 Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:12:55 -0400 Subject: [PATCH 02/12] Bind native restricted admission and preserve cold readback --- .../reference/native-local-training.md | 7 +- .../reference/native-local-training.md | 7 +- .../reference/native-local-training.md | 7 +- .../training_qualification/bounded_command.py | 90 ++++++ .../training_qualification/native-host.md | 14 +- .../training_qualification/native_host.py | 79 +++-- tests/cli/test_native_host.py | 299 +++++++++++++++++- tests/runtime/test_native_mlx_family.py | 43 ++- tuner/training/native_mlx_planning.py | 37 ++- 9 files changed, 541 insertions(+), 42 deletions(-) create mode 100644 examples/training_qualification/bounded_command.py diff --git a/.agents/skills/fine-tuning/reference/native-local-training.md b/.agents/skills/fine-tuning/reference/native-local-training.md index 2893ada9..ee200a38 100644 --- a/.agents/skills/fine-tuning/reference/native-local-training.md +++ b/.agents/skills/fine-tuning/reference/native-local-training.md @@ -181,8 +181,11 @@ model layer set and complete tokenizer snapshot selection. No accepting **ordinary** hardware authority is shipped. An unadmitted candidate has observe/artifact capabilities disabled and preflight is not ready. A separately verified `NativeRestrictedAdmissionV1` enables only its -exact initial run, attempt, source, runtime, prepared bytes and model/tokenizer -snapshots, subject to a finite expiry; it is not qualification. The host +exact compiled workload and host selection, including project, request, run, +attempt, private state root, process limits, artifact budget, source, runtime, +prepared bytes and model/tokenizer snapshots, subject to a finite start expiry; +it is not qualification. Retained public reads authenticate that signed selection +after expiry and remote branch advancement without new dispatch authority. The host consumer in `examples/training_qualification/native_host.py` verifies the external signature using a separately supplied reviewed public-key SHA-256. It has no signing key or acceptance switch. The same host checks the actual diff --git a/.claude/skills/fine-tuning/reference/native-local-training.md b/.claude/skills/fine-tuning/reference/native-local-training.md index 2893ada9..ee200a38 100644 --- a/.claude/skills/fine-tuning/reference/native-local-training.md +++ b/.claude/skills/fine-tuning/reference/native-local-training.md @@ -181,8 +181,11 @@ model layer set and complete tokenizer snapshot selection. No accepting **ordinary** hardware authority is shipped. An unadmitted candidate has observe/artifact capabilities disabled and preflight is not ready. A separately verified `NativeRestrictedAdmissionV1` enables only its -exact initial run, attempt, source, runtime, prepared bytes and model/tokenizer -snapshots, subject to a finite expiry; it is not qualification. The host +exact compiled workload and host selection, including project, request, run, +attempt, private state root, process limits, artifact budget, source, runtime, +prepared bytes and model/tokenizer snapshots, subject to a finite start expiry; +it is not qualification. Retained public reads authenticate that signed selection +after expiry and remote branch advancement without new dispatch authority. The host consumer in `examples/training_qualification/native_host.py` verifies the external signature using a separately supplied reviewed public-key SHA-256. It has no signing key or acceptance switch. The same host checks the actual diff --git a/.skills/fine-tuning/reference/native-local-training.md b/.skills/fine-tuning/reference/native-local-training.md index 2893ada9..ee200a38 100644 --- a/.skills/fine-tuning/reference/native-local-training.md +++ b/.skills/fine-tuning/reference/native-local-training.md @@ -181,8 +181,11 @@ model layer set and complete tokenizer snapshot selection. No accepting **ordinary** hardware authority is shipped. An unadmitted candidate has observe/artifact capabilities disabled and preflight is not ready. A separately verified `NativeRestrictedAdmissionV1` enables only its -exact initial run, attempt, source, runtime, prepared bytes and model/tokenizer -snapshots, subject to a finite expiry; it is not qualification. The host +exact compiled workload and host selection, including project, request, run, +attempt, private state root, process limits, artifact budget, source, runtime, +prepared bytes and model/tokenizer snapshots, subject to a finite start expiry; +it is not qualification. Retained public reads authenticate that signed selection +after expiry and remote branch advancement without new dispatch authority. The host consumer in `examples/training_qualification/native_host.py` verifies the external signature using a separately supplied reviewed public-key SHA-256. It has no signing key or acceptance switch. The same host checks the actual diff --git a/examples/training_qualification/bounded_command.py b/examples/training_qualification/bounded_command.py new file mode 100644 index 00000000..9bd9e10a --- /dev/null +++ b/examples/training_qualification/bounded_command.py @@ -0,0 +1,90 @@ +"""Bounded, owned POSIX command output for host-side observations. + +The caller supplies exact argv and a small environment. Output is drained as +it arrives with one combined stdout/stderr limit; no exception exposes output, +argv, environment, or subprocess exception text. +""" + +from __future__ import annotations + +import os +import selectors +import signal +import subprocess +import time + + +class BoundedCommandError(RuntimeError): + pass + + +def run_bounded_command(argv, *, cwd=None, environment=None, input_bytes=b"", + maximum=4096, timeout=15): + if (os.name != "posix" or not hasattr(os, "killpg") + or not isinstance(argv, (tuple, list)) or not argv + or any(type(value) is not str or not value for value in argv) + or type(input_bytes) is not bytes or len(input_bytes) > 1024 * 1024 + or type(maximum) is not int or not 1 <= maximum <= 1024 * 1024 + or type(timeout) not in (int, float) or not 0 < timeout <= 300): + raise BoundedCommandError("bounded_command_invalid") + process = None + selector = selectors.DefaultSelector() + try: + process = subprocess.Popen( + argv, cwd=cwd, env=environment, stdin=subprocess.PIPE if input_bytes else subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, start_new_session=True, close_fds=True, + ) + deadline = time.monotonic() + timeout + stdout, total, written = [], 0, 0 + for stream, role in ((process.stdout, "stdout"), (process.stderr, "stderr")): + os.set_blocking(stream.fileno(), False) + selector.register(stream, selectors.EVENT_READ, role) + if input_bytes: + os.set_blocking(process.stdin.fileno(), False) + selector.register(process.stdin, selectors.EVENT_WRITE, "stdin") + while selector.get_map(): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise BoundedCommandError("bounded_command_deadline") + for key, _ in selector.select(remaining): + stream, role = key.fileobj, key.data + if role == "stdin": + try: + written += os.write(stream.fileno(), input_bytes[written:written + 65536]) + except BrokenPipeError: + written = len(input_bytes) + if written == len(input_bytes): + selector.unregister(stream) + stream.close() + continue + raw = os.read(stream.fileno(), min(65536, maximum + 1 - total)) + if raw: + total += len(raw) + if total > maximum: + raise BoundedCommandError("bounded_command_output_limit") + if role == "stdout": + stdout.append(raw) + else: + selector.unregister(stream) + stream.close() + remaining = deadline - time.monotonic() + if remaining <= 0: + raise BoundedCommandError("bounded_command_deadline") + return process.wait(timeout=remaining), b"".join(stdout) + except Exception: + if process is not None: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + pass + raise BoundedCommandError("bounded_command_failed") from None + finally: + selector.close() + if process is not None: + for stream in (process.stdin, process.stdout, process.stderr): + if stream is not None and not stream.closed: + stream.close() diff --git a/examples/training_qualification/native-host.md b/examples/training_qualification/native-host.md index 2083e8d0..672b5326 100644 --- a/examples/training_qualification/native-host.md +++ b/examples/training_qualification/native-host.md @@ -5,8 +5,9 @@ host record/stream stores, Foundation grants, and the native provider family. It has no engine database, global provider registration, background training job, replay, install, or model download. `inspect` and `profile-candidate` are read-only. `start` is a one-shot effect and requires a separate root-signed -admission for its exact source, runtime, hardware, model snapshots, prepared -publication, run and attempt. An ordinary two-run qualification receipt is not +admission for its exact compiled workload and host selection: source, runtime, +hardware, model snapshots, prepared publication, project, request, run, attempt, +private state root, process limits and artifact budget. An ordinary two-run qualification receipt is not created by this workflow. ## Inputs and private layout @@ -144,8 +145,13 @@ the config or inspection is not independent approval. Root must review the source commit, selected wheels/install provenance, capture/build closure, hardware, model snapshot and both distinct prepared datasets before signing. -An admitted restricted run can only use its exact run/attempt and snapshots. -Its one-hour admission expires; collect cold public reads within that window. +An admitted restricted run can only use its exact signed selection. Its +one-hour admission is a start deadline. Cold show, observe, verify and download +recompose the same signed selection and retained state after expiry, without +renewing dispatch authority; they do not require the remote branch to remain +at its former head. A new start or replay still requires fresh reviewed admission. +Host metadata commands use an owned subprocess with a combined output byte +bound and deadline; exceeding either ends its process group. The normal capability receipt still needs two real accepted runs, native artifact verification, Darwin deadline/owner-loss evidence, and a real Syntunia bridge readback. Public logs/cancel/reconcile, GRPO, and asynchronous diff --git a/examples/training_qualification/native_host.py b/examples/training_qualification/native_host.py index dce1504b..12f031c1 100644 --- a/examples/training_qualification/native_host.py +++ b/examples/training_qualification/native_host.py @@ -14,12 +14,12 @@ from pathlib import Path import platform import re -import subprocess import sys ENGINE_ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ENGINE_ROOT)) sys.path.insert(0, str(Path(__file__).resolve().parent)) +from bounded_command import BoundedCommandError, run_bounded_command from launch import Clock, NamedSecrets, _private, download from synaptic_tuner.api.v1.planning import ResolvedTrainingRequest @@ -93,13 +93,14 @@ def _write_owned(root, name, raw, maximum): def _command(argv, *, cwd=None, maximum=4096): env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "LC_ALL": "C", "GIT_TERMINAL_PROMPT": "0", "GIT_CONFIG_NOSYSTEM": "1"} - result = subprocess.run(argv, cwd=cwd, env=env, capture_output=True, timeout=15, check=True) - if len(result.stdout) > maximum or len(result.stderr) > maximum: - raise ValueError("native host command output exceeds bound") - return result.stdout.decode("utf-8").strip() + status, stdout = run_bounded_command(argv, cwd=cwd, environment=env, + maximum=maximum, timeout=15) + if status != 0: + raise ValueError("native host command refused") + return stdout.decode("utf-8").strip() -def source_record(config): +def source_record(config, *, check_remote=True): """Observe the real clean checkout and exact pushed ref, never a supplied SHA alone.""" root = Path(config["engine_root"]) if (not root.is_absolute() or root.resolve(strict=True) != root @@ -115,9 +116,10 @@ def source_record(config): raise ValueError("native engine ref is invalid") if commit != config["engine_commit"] or remote != config["engine_remote"]: raise ValueError("native local source selection differs") - remote_line = _command(["git", "ls-remote", "--exit-code", "origin", ref], cwd=root) - if remote_line != f"{commit}\t{ref}": - raise ValueError("native pushed source ref differs") + if check_remote: + remote_line = _command(["git", "ls-remote", "--exit-code", "origin", ref], cwd=root) + if remote_line != f"{commit}\t{ref}": + raise ValueError("native pushed source ref differs") value = {"schema": "synaptic-native-pushed-source/v1", "url": remote, "ref": ref, "commit": commit} return value, hashlib.sha256(canonical(value)).hexdigest() @@ -216,15 +218,16 @@ def authenticate(self, admission): return False content = asdict(admission) content.pop("tag") - result = subprocess.run( + status, stdout = run_bounded_command( ["openssl", "dgst", "-sha256", "-verify", str(self.key_path), "-signature", str(self.signature_path)], - input=canonical(content), capture_output=True, timeout=10, check=False, + input_bytes=canonical(content), maximum=4096, timeout=10, + environment={"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "LC_ALL": "C"}, ) - return (result.returncode == 0 and result.stdout.strip() == b"Verified OK" + return (status == 0 and stdout.strip() == b"Verified OK" and _read(self.key_path, 16384) == self.key and _read(self.signature_path, 16384) == self.signature) - except (OSError, subprocess.SubprocessError, ValueError): + except (OSError, BoundedCommandError, ValueError): return False @@ -285,7 +288,18 @@ def bind(self, grant, *, operation, requirements): return grant -def inspect(config, *, require_admission=False, trusted_root_key_sha256=None): +class _ReadOnlyGrants: + """Cold public reads cannot mint or bind a new training grant.""" + + def authorize(self, requirements): + raise ValueError("native retained read has no grant authority") + + def bind(self, grant, *, operation, requirements): + raise ValueError("native retained read has no dispatch authority") + + +def inspect(config, *, require_admission=False, trusted_root_key_sha256=None, + retained_read=False): """Rebuild source, runtime, prepared publication and snapshot bindings.""" fields = { "schema", "proposal_file", "engine_root", "engine_remote", "engine_ref", "engine_commit", @@ -305,7 +319,13 @@ def inspect(config, *, require_admission=False, trusted_root_key_sha256=None): } or type(config["tokenizer_members"]) is not list or tuple(config["tokenizer_members"]) != tuple(sorted(set(config["tokenizer_members"])))): raise ValueError("native host authority or tokenizer selection invalid") - source, source_digest = source_record(config) + if type(retained_read) is not bool or (retained_read and not require_admission): + raise ValueError("native retained read requires signed selection") + state_root = Path(config["state_root"]) + if (not state_root.is_absolute() or str(state_root) != os.path.normpath(str(state_root)) + or str(state_root) != config["state_root"]): + raise ValueError("native state root is noncanonical") + source, source_digest = source_record(config, check_remote=not retained_read) hardware, hardware_digest = hardware_record() proposal = strict_json(_read(config["proposal_file"], 1 << 20)) if (proposal.get("schema") != "synaptic-native-qualification-proposal/v1" @@ -410,11 +430,22 @@ def inspect(config, *, require_admission=False, trusted_root_key_sha256=None): base = dict(resolved=resolved, run=run, workload=projection.workload.canonical_bytes, runtime_capture=capture, limits=limits, scope=scope, clock=Clock()) candidate = NativePlanningV1(**base) + selection_digest = hashlib.sha256(canonical({ + "schema": "synaptic-native-host-execution-selection/v1", + "config": config, "source": source, "hardware": hardware, + "proposal": proposal, "profile": asdict(profile), + "training_input": training.to_dict(), "prepared_identity": identity.to_dict(), + "prepared_digest": hashlib.sha256(dataset).hexdigest(), + "capture_digest": capture_digest, "dependency_lock_digest": hashlib.sha256(lock).hexdigest(), + "model": asdict(model), "tokenizer": asdict(tokenizer), + "workload_digest": projection.workload.fingerprint, + })).hexdigest() admission_fields = dict( profile_digest=candidate.profile_digest, source_digest=source_digest, runtime_digest=resolved.runtime_digest, hardware_digest=hardware_digest, prepared_digest=identity.content_digest, model_digest=model.digest, tokenizer_digest=tokenizer.digest, run_id=run.run_id, attempt=config["attempt"], + workload_digest=projection.workload.fingerprint, selection_digest=selection_digest, maximum_updates=projection.controls.iterations, ) if not require_admission: @@ -433,8 +464,9 @@ def inspect(config, *, require_admission=False, trusted_root_key_sha256=None): if not root.authenticate(admission): raise ValueError("external restricted admission signature invalid") planning = NativePlanningV1(**base, restricted_admission=admission, - restricted_authority=root) - if not planning.restricted(): + restricted_authority=root, selection_digest=selection_digest, + retained_read=retained_read) + if not (planning.restricted_selection() if retained_read else planning.restricted()): raise ValueError("restricted native admission expired or refused") return {"source": source, "hardware": hardware, "admission_candidate": admission_fields, "capture_digest": capture_digest, "prepared_rows_bytes": len(dataset)}, ( @@ -446,10 +478,15 @@ def operate(config, *, action="start", destination=None, trusted_root_key_sha256 """Start once, or recompose the same selected run for a cold public read.""" if action not in {"start", "show", "observe", "verify", "download"}: raise ValueError("native public action is unavailable") + state = Path(config["state_root"]) + if action != "start": + _private(state) + if _document(state / "config.json", 4 << 20) != config: + raise ValueError("retained native host configuration differs") _, selected = inspect(config, require_admission=True, - trusted_root_key_sha256=trusted_root_key_sha256) + trusted_root_key_sha256=trusted_root_key_sha256, + retained_read=action != "start") planning, training, projection, dataset, lock, model, tokenizer = selected - state = Path(config["state_root"]) if action == "start": _create_private_dir(state) _private(state) @@ -519,7 +556,9 @@ def operate(config, *, action="start", destination=None, trusted_root_key_sha256 model=model, tokenizer=tokenizer, tokenizer_members=tuple(config["tokenizer_members"]), budget=NativeArtifactBudget(**config["artifact_budget"]), material_authority_ref=host["material_authority_ref"], material_key_ref=host["material_key_ref"]) - ports = ReferenceHostPortsV1(store, store, clock, _Grants(selection, catalog, material_authority), secrets) + grants = (_Grants(selection, catalog, material_authority) + if action == "start" else _ReadOnlyGrants()) + ports = ReferenceHostPortsV1(store, store, clock, grants, secrets) authority = compose_reference_authority( clock=clock, secrets=secrets, authority_secret=SecretRef.from_dict(host["authority_secret"]), diff --git a/tests/cli/test_native_host.py b/tests/cli/test_native_host.py index 47d3ffd0..a5ff0255 100644 --- a/tests/cli/test_native_host.py +++ b/tests/cli/test_native_host.py @@ -1,17 +1,32 @@ """Host trust boundary tests; no ML package import or Mac execution.""" from dataclasses import asdict, replace +from copy import deepcopy import hashlib +import json +import multiprocessing +import os from pathlib import Path import subprocess +import sys import tempfile +import time import pytest from examples.training_qualification import native_host +from examples.training_qualification.bounded_command import BoundedCommandError, run_bounded_command from tuner.runtime.native_mlx_material import canonical from tuner.training.native_mlx_planning import NativeRestrictedAdmissionV1 +from tests.runtime import test_native_mlx_publication as publication_fixtures +publication = publication_fixtures.publication +from tests.runtime.test_native_mlx_child import inputs +from tests.runtime.test_native_mlx_family import prepare as prepare_family +from tuner.runtime.native_mlx_material import strict_json +from tuner.runtime.native_process import NativeOwnedProcess +from tuner.training.native_mlx_compilation import compile_native_sft + def test_external_root_signature_is_required_and_binds_exact_admission(tmp_path): private = tmp_path / "root-private.pem" @@ -23,7 +38,7 @@ def test_external_root_signature_is_required_and_binds_exact_admission(tmp_path) subprocess.run(["openssl", "pkey", "-in", str(private), "-pubout", "-out", str(public)], check=True, capture_output=True) candidate = NativeRestrictedAdmissionV1( - *("a" * 64 for _ in range(7)), "run", "attempt-1", 3, + *("a" * 64 for _ in range(9)), "run", "attempt-1", 3, "2026-10-20T12:30:00Z", "root-review", "root-key", "0" * 64, ) signed = asdict(candidate) @@ -73,7 +88,6 @@ def command(argv, **_): def test_snapshot_candidate_inventories_complete_owned_bytes(): - import os original_umask = os.umask(0o077) try: with tempfile.TemporaryDirectory(prefix="native-host-snapshot-", dir=Path.home()) as name: @@ -96,3 +110,284 @@ def test_snapshot_candidate_inventories_complete_owned_bytes(): manifest.unlink(missing_ok=True) finally: os.umask(original_umask) + + +@pytest.mark.skipif(os.name != "posix", reason="native host requires POSIX process groups") +@pytest.mark.parametrize("stdout_size,stderr_size", [ + (8192, 0), (0, 8192), (2048, 2049), +]) +def test_host_command_output_is_bounded_while_draining(stdout_size, stderr_size): + code = ("import sys; sys.stdout.buffer.write(b'a' * %d); " + "sys.stderr.buffer.write(b'b' * %d)" % (stdout_size, stderr_size)) + with pytest.raises(BoundedCommandError, match="bounded_command_failed"): + run_bounded_command([sys.executable, "-I", "-c", code], + environment={"PATH": "/usr/bin:/bin"}, maximum=4096, timeout=3) + status, output = run_bounded_command( + [sys.executable, "-I", "-c", "import sys; sys.stdout.write('OK')"], + maximum=4096, timeout=3, + ) + assert status == 0 and output == b"OK" + + +@pytest.mark.skipif(os.name != "posix", reason="native host requires POSIX process groups") +def test_host_command_deadline_kills_owned_session(tmp_path): + pid_file = tmp_path / "owned-pid" + code = ("import os,time; from pathlib import Path; " + f"Path({str(pid_file)!r}).write_text(str(os.getpid())); " + "time.sleep(30)") + started = time.monotonic() + with pytest.raises(BoundedCommandError, match="bounded_command_failed"): + run_bounded_command([sys.executable, "-I", "-c", code], maximum=4096, timeout=.2) + assert time.monotonic() - started < 3 + pid = int(pid_file.read_text()) + with pytest.raises(ProcessLookupError): + os.kill(pid, 0) + + +@pytest.mark.skipif(os.name != "posix", reason="native host requires POSIX process groups") +def test_host_command_flood_kills_owned_session(tmp_path): + pid_file = tmp_path / "flood-pid" + code = ("import os,sys; from pathlib import Path; " + f"Path({str(pid_file)!r}).write_text(str(os.getpid()))\n" + "while True:\n sys.stdout.buffer.write(b'x' * 65536)\n sys.stdout.buffer.flush()") + with pytest.raises(BoundedCommandError, match="bounded_command_failed"): + run_bounded_command([sys.executable, "-I", "-c", code], maximum=4096, timeout=3) + pid = int(pid_file.read_text()) + with pytest.raises(ProcessLookupError): + os.kill(pid, 0) + + +@pytest.fixture +def host_case(publication, monkeypatch): + """Real host composition/store with only Mac metadata and ML child substituted.""" + from tuner.training import native_mlx_family as family_module + state = prepare_family(publication) + root = state.root + training, identity, profile = inputs() + rows = [ + {"schema_version": "syntunia-sft-row/v1", "format": "raw_text", + "row_id": f"row-{index}", "text": "example", "split": "train" if index < 3 else "validation"} + for index in range(5) + ] + dataset = b"".join(canonical(row) + b"\n" for row in rows) + identity = replace(identity, content_digest=hashlib.sha256(dataset).hexdigest(), + size_bytes=len(dataset)) + source = {"schema": "synaptic-native-pushed-source/v1", "url": "https://example.invalid/engine.git", + "ref": "refs/heads/native-review", "commit": "a" * 40} + source_digest = hashlib.sha256(canonical(source)).hexdigest() + hardware = {"schema": "synaptic-native-hardware-observation/v1", "machine": "arm64", + "macos": "15.0", "kernel": "fixture", "model": "fixture-M4", "cpu": "Apple M4", + "memory_bytes": 25769803776} + hardware_digest = hashlib.sha256(canonical(hardware)).hexdigest() + profile = replace(profile, source_digest=source_digest, + runtime_digest=hashlib.sha256(state.kwargs["runtime_capture"]).hexdigest()) + projection = compile_native_sft(training, identity, profile) + binding_path = publication["output"] / "input_binding.json" + binding = strict_json(binding_path.read_bytes()) + binding.update(source=identity.to_dict(), workload_digest=projection.workload.fingerprint, + runtime_digest=profile.runtime_digest) + binding_path.write_bytes(canonical(binding)) + result_path = publication["output"] / "native_result.json" + result = strict_json(result_path.read_bytes()) + result.update(input_binding_digest=hashlib.sha256(binding_path.read_bytes()).hexdigest(), + workload_digest=projection.workload.fingerprint, + runtime_digest=profile.runtime_digest) + result_path.write_bytes(canonical(result)) + (publication["output"] / "workload.json").write_bytes(projection.workload.canonical_bytes) + proposal_path = Path(native_host.ENGINE_ROOT) / "Trainers/mlx_sft_mac/runtime-candidate/sft-qualification-proposal.json" + proposal = json.loads(proposal_path.read_text()) + proposal["profile_ref"] = profile.profile_ref + proposal["model"].update(repository=profile.model_ref, revision=profile.model_revision, + tokenizer_repository=profile.model_ref, + tokenizer_revision=profile.tokenizer_revision) + proposal["runtime"].update(dependency_lock_sha256=hashlib.sha256(b"fixture-hash-lock").hexdigest()) + proposal["bounds"].update(timeout_seconds_per_attempt=5, termination_grace_seconds=1, + maximum_output_bytes_per_stream=4096, + maximum_artifact_file_bytes=1024 * 1024, + maximum_artifact_set_bytes=4 * 1024 * 1024) + files = { + "proposal.json": proposal, + "profile.json": asdict(profile), + "training.json": training.to_dict(), + "identity.json": identity.to_dict(), + "model-manifest.json": asdict(publication["model"]), + "tokenizer-manifest.json": asdict(publication["tokenizer"]), + } + for filename, value in files.items(): + (root / filename).write_bytes(canonical(value)) + public_key, private_key = root / "root-public.pem", root / "root-private.pem" + subprocess.run(["openssl", "genpkey", "-algorithm", "EC", "-pkeyopt", + "ec_paramgen_curve:P-256", "-out", str(private_key)], check=True, + capture_output=True) + subprocess.run(["openssl", "pkey", "-in", str(private_key), "-pubout", "-out", str(public_key)], + check=True, capture_output=True) + config = { + "schema": "synaptic-native-host-config/v1", "proposal_file": str(root / "proposal.json"), + "engine_root": str(root / "engine"), "engine_remote": source["url"], + "engine_ref": source["ref"], "engine_commit": source["commit"], + "runtime_capture_file": str(root / "capture.json"), + "dependency_lock_file": str(root / "dependency.lock"), + "profile_file": str(root / "profile.json"), "training_input_file": str(root / "training.json"), + "prepared_identity_file": str(root / "identity.json"), + "prepared_publication": str(root / "prepared-publication"), + "model_snapshot_file": str(root / "model-manifest.json"), + "tokenizer_snapshot_file": str(root / "tokenizer-manifest.json"), + "admission_file": str(root / "admission.json"), + "admission_signature_file": str(root / "admission.sig"), + "root_public_key_file": str(public_key), + "root_public_key_sha256": hashlib.sha256(public_key.read_bytes()).hexdigest(), + "limits": {"timeout_seconds": 5, "termination_grace_seconds": 1, + "maximum_output_bytes": 4096, "maximum_file_bytes": 1024 * 1024}, + "scope": {"account_ref": "owner", "namespace_ref": "machine"}, + "run_id": "run", "project_ref": "project", "request_id": "request", + "attempt": "attempt-1", "attempt_kind": "raw_full_sequence", + "state_root": str(root / "attempt-state"), + "host": { + "authority_secret": {"provider": "env", "name": "NATIVE_TEST_AUTHORITY"}, + "evidence_secret": {"provider": "env", "name": "NATIVE_TEST_EVIDENCE"}, + "material_secret": {"provider": "env", "name": "NATIVE_TEST_MATERIAL"}, + "grant_ref": "native-test-grant", "reader_authority_ref": "reader-test", + "reader_key_ref": "reader-key", "material_authority_ref": "material-test", + "material_key_ref": "material-key", + }, + "tokenizer_members": list(publication["tokenizer_members"]), + "artifact_budget": {"per_file": 1024 * 1024, "aggregate": 4 * 1024 * 1024}, + "use_fast_tokenizer": profile.use_fast_tokenizer, + } + for key in ("NATIVE_TEST_AUTHORITY", "NATIVE_TEST_EVIDENCE", "NATIVE_TEST_MATERIAL"): + monkeypatch.setenv(key, "test-secret-" + key + "-" + "x" * 32) + clock = {"now": "2026-10-20T12:00:00Z"} + + class TestClock: + def now(self): + return clock["now"] + + def now_epoch(self): + from datetime import datetime + return int(datetime.fromisoformat(clock["now"].replace("Z", "+00:00")).timestamp()) + + monkeypatch.setattr(native_host, "Clock", TestClock) + remote = {"advanced": False} + def source_record(selected, *, check_remote=True): + if check_remote and remote["advanced"]: + raise ValueError("native pushed source ref differs") + return source, source_digest + monkeypatch.setattr(native_host, "source_record", source_record) + monkeypatch.setattr(native_host, "hardware_record", lambda: (hardware, hardware_digest)) + monkeypatch.setattr(native_host, "verify_native_material", lambda *a, **k: None) + monkeypatch.setattr(family_module, "verify_native_material", lambda *a, **k: None) + monkeypatch.setattr(native_host, "read_retained_native_source", lambda prepared, selected: dataset) + admission_fields = native_host.inspect(config)[0]["admission_candidate"] + unsigned = {**admission_fields, "expires_at": "2026-10-20T12:30:00Z", + "authority_ref": "root-review", "key_ref": "root-key"} + signature = root / "admission.sig" + subprocess.run(["openssl", "dgst", "-sha256", "-sign", str(private_key), + "-out", str(signature)], input=canonical(unsigned), check=True, + capture_output=True) + admission = {**unsigned, "tag": hashlib.sha256(signature.read_bytes()).hexdigest()} + (root / "admission.json").write_bytes(canonical(admission)) + calls = [] + def child(self, invocation): + calls.append(invocation.digest) + self._retain(invocation, "claim", {"automatic_replay": False}) + output = Path(invocation.private_root) / "output" + for item in publication["output"].iterdir(): + (output / item.name).write_bytes(item.read_bytes()) + outcome = {"schema": "synaptic-native-process-outcome/v1", "reason": "worker_exited", + "exit_code": 0, "termination_confirmed": True, + "stdout_bytes": 0, "stderr_bytes": 0} + self._retain(invocation, "outcome", outcome) + return outcome + monkeypatch.setattr(NativeOwnedProcess, "run_once", child) + return config, clock, remote, calls + + +@pytest.mark.parametrize("mutation", [ + "learning_rate", "rank", "targets", "batch", "tokenizer", "state_root", + "project", "request", "artifact_budget", +]) +def test_root_admission_rejects_changed_execution_selection(host_case, mutation): + config, _, _, calls = host_case + changed = deepcopy(config) + def edit(path, update): + document = strict_json(Path(path).read_bytes()) + update(document) + Path(path).write_bytes(canonical(document)) + if mutation == "learning_rate": + edit(config["training_input_file"], lambda d: d["hyperparameters"].update(learning_rate=.0002)) + edit(config["proposal_file"], lambda d: d["optimizer"].update(learning_rate=.0002)) + elif mutation == "rank": + edit(config["training_input_file"], lambda d: d["hyperparameters"].update( + lora_rank=8, lora_alpha=64)) + edit(config["proposal_file"], lambda d: d["lora"].update(rank=8)) + elif mutation == "targets": + edit(config["training_input_file"], lambda d: d["hyperparameters"].update( + lora_target_modules=["self_attn.q_proj"])) + edit(config["proposal_file"], lambda d: d["lora"].update(targets=["self_attn.q_proj"])) + elif mutation == "batch": + edit(config["training_input_file"], lambda d: d["hyperparameters"].update( + batch_size=2, max_seq_length=64)) + edit(config["proposal_file"], lambda d: d["training"].update(batch_size=2, max_seq_length=64)) + elif mutation == "tokenizer": + changed["use_fast_tokenizer"] = False + edit(config["profile_file"], lambda d: d.update(use_fast_tokenizer=False)) + elif mutation == "state_root": + changed["state_root"] = str(Path(config["state_root"]).with_name("fresh-state")) + elif mutation == "project": + changed["project_ref"] = "different-project" + elif mutation == "request": + changed["request_id"] = "different-request" + elif mutation == "artifact_budget": + changed["artifact_budget"]["aggregate"] = 8 * 1024 * 1024 + edit(config["proposal_file"], lambda d: d["bounds"].update( + maximum_artifact_set_bytes=8 * 1024 * 1024)) + with pytest.raises(ValueError, match="restricted admission selection differs"): + native_host.inspect(changed, require_admission=True, + trusted_root_key_sha256=config["root_public_key_sha256"]) + assert not Path(changed["state_root"]).exists() and calls == [] + + +@pytest.mark.skipif(os.name != "posix", reason="cold native host read uses POSIX fork") +def test_cold_public_readback_survives_expiry_and_remote_ref_advance(host_case): + config, clock, remote, calls = host_case + trusted = config["root_public_key_sha256"] + start = native_host.operate(config, action="start", trusted_root_key_sha256=trusted) + assert start["accepted"] and len(calls) == 1 + clock["now"] = "2026-10-20T14:00:00Z" + remote["advanced"] = True + def cold_reader(connection): + try: + shown = native_host.operate(config, action="show", trusted_root_key_sha256=trusted) + observed = native_host.operate(config, action="observe", trusted_root_key_sha256=trusted) + verified = native_host.operate(config, action="verify", trusted_root_key_sha256=trusted) + downloaded = native_host.operate( + config, action="download", trusted_root_key_sha256=trusted, + destination=Path(config["state_root"]) / "downloaded", + ) + connection.send((shown, observed, verified, downloaded, len(calls))) + finally: + connection.close() + parent, child = multiprocessing.get_context("fork").Pipe(duplex=False) + reader = multiprocessing.get_context("fork").Process(target=cold_reader, args=(child,)) + reader.start() + child.close() + try: + assert parent.poll(10), "cold native reader did not finish" + shown, observed, verified, downloaded, child_calls = parent.recv() + reader.join(2) + assert reader.exitcode == 0 and child_calls == 1 + finally: + if reader.is_alive(): + reader.terminate() + reader.join(2) + parent.close() + assert shown["run"]["run_id"] == "run" + assert observed["state"] == "succeeded" + assert verified["verified"] is True + assert len(downloaded["paths"]) == 5 and len(calls) == 1 + with pytest.raises(ValueError, match="pushed source"): + native_host.operate(config, action="start", trusted_root_key_sha256=trusted) + remote["advanced"] = False + with pytest.raises(ValueError, match="expired"): + native_host.operate(config, action="start", trusted_root_key_sha256=trusted) + assert len(calls) == 1 diff --git a/tests/runtime/test_native_mlx_family.py b/tests/runtime/test_native_mlx_family.py index 8fbedeae..3bd498f0 100644 --- a/tests/runtime/test_native_mlx_family.py +++ b/tests/runtime/test_native_mlx_family.py @@ -19,6 +19,7 @@ from synaptic_tuner.api.v1.results import TrainingRunRef from synaptic_tuner.api.v1.runs_facade import RunArtifactRequest, RunLogsRequest, RunOperationError from synaptic_tuner.api.v1.secrets import SecretRef +from synaptic_tuner.api.v1.training_input import TrainingInputV1 from synaptic_tuner.api.v1.training_facade import TrainingRequest from tuner.execution.foundation_v2.references import ExecutionScopeV1 from tuner.runtime.native_mlx_material import canonical, native_runtime_spec, strict_json @@ -133,12 +134,13 @@ def test_restricted_admission_is_exact_run_and_does_not_qualify_profile(publicat candidate.profile_digest, state.resolved.source_digest, state.resolved.runtime_digest, "4" * 64, parse_native_workload(state.kwargs["workload"]).source.content_digest, publication["model"].digest, publication["tokenizer"].digest, + state.resolved.workload_digest, "7" * 64, "run", "attempt-1", 3, "2026-10-20T12:30:00Z", "root-review", "root-public-key", "f" * 64, ) admitted = NativePlanningV1( **state.kwargs, restricted_admission=proof, - restricted_authority=TestQualificationAuthority(), + restricted_authority=TestQualificationAuthority(), selection_digest="7" * 64, ) assert admitted.restricted() and admitted.admitted() and not admitted.qualified() assert admitted.preflight(admitted.plan).ready @@ -151,7 +153,44 @@ def test_restricted_admission_is_exact_run_and_does_not_qualify_profile(publicat admitted.require_qualified() changed = replace(proof, run_id="another-run") assert not NativePlanningV1(**state.kwargs, restricted_admission=changed, - restricted_authority=TestQualificationAuthority()).admitted() + restricted_authority=TestQualificationAuthority(), + selection_digest="7" * 64).admitted() + training_document = state.training.to_dict() + training_document["hyperparameters"]["learning_rate"] = .0002 + changed_training = TrainingInputV1.from_dict(training_document) + prior = parse_native_workload(state.kwargs["workload"]) + changed_workload = compile_native_sft(changed_training, prior.source, prior.profile).workload + changed_kwargs = dict(state.kwargs) + changed_kwargs["workload"] = changed_workload.canonical_bytes + changed_kwargs["resolved"] = replace( + state.resolved, + resolved_config_digest=hashlib.sha256(changed_training.canonical_json().encode()).hexdigest(), + workload_digest=changed_workload.fingerprint, + ) + substituted = NativePlanningV1( + **changed_kwargs, restricted_admission=proof, + restricted_authority=TestQualificationAuthority(), selection_digest="7" * 64, + ) + assert substituted.profile_digest == admitted.profile_digest + assert not substituted.restricted_selection() and not substituted.preflight(substituted.plan).ready + + class LateClock: + def now(self): + return "2026-10-20T14:00:00Z" + + late_kwargs = {**state.kwargs, "clock": LateClock()} + retained = NativePlanningV1( + **late_kwargs, restricted_admission=proof, + restricted_authority=TestQualificationAuthority(), selection_digest="7" * 64, + retained_read=True, + ) + assert retained.restricted_selection() and not retained.restricted() + assert retained.descriptor.descriptor_digest == admitted.descriptor.descriptor_digest + assert retained.plan == admitted.plan + assert not retained.preflight(retained.plan).ready + with pytest.raises(ValueError, match="no_dispatch_authority"): + retained.require_admitted(attempt="attempt-1", model_digest=publication["model"].digest, + tokenizer_digest=publication["tokenizer"].digest) def compose(state, publication, monkeypatch, *, fail=False): diff --git a/tuner/training/native_mlx_planning.py b/tuner/training/native_mlx_planning.py index df057a62..8504af1e 100644 --- a/tuner/training/native_mlx_planning.py +++ b/tuner/training/native_mlx_planning.py @@ -79,6 +79,8 @@ class NativeRestrictedAdmissionV1: prepared_digest: str model_digest: str tokenizer_digest: str + workload_digest: str + selection_digest: str run_id: str attempt: str maximum_updates: int @@ -89,7 +91,8 @@ class NativeRestrictedAdmissionV1: def __post_init__(self): for name in ("profile_digest", "source_digest", "runtime_digest", "hardware_digest", - "prepared_digest", "model_digest", "tokenizer_digest", "tag"): + "prepared_digest", "model_digest", "tokenizer_digest", + "workload_digest", "selection_digest", "tag"): digest_text(getattr(self, name), name) for name in ("run_id", "attempt", "authority_ref", "key_ref"): safe_ref(getattr(self, name), name) @@ -107,6 +110,7 @@ class NativePlanningV1: "resolved", "run", "workload", "runtime_capture", "limits", "scope", "provider", "profile_digest", "_clock", "_qualification", "_qualification_authority", "_restricted_admission", "_restricted_authority", + "_selection_digest", "_retained_read", "descriptor", "executor_descriptor", "adapter_descriptor", "binding", "context_value", "plan", "_sealed", ) @@ -121,10 +125,14 @@ def __init__( workload: bytes, runtime_capture: bytes, limits: NativeProcessLimits, scope: ExecutionScopeV1, clock, qualification=None, qualification_authority=None, restricted_admission=None, restricted_authority=None, + selection_digest=None, retained_read=False, ): if (type(resolved) is not ResolvedTrainingRequest or type(run) is not TrainingRunRef - or type(limits) is not NativeProcessLimits or type(scope) is not ExecutionScopeV1): + or type(limits) is not NativeProcessLimits or type(scope) is not ExecutionScopeV1 + or type(retained_read) is not bool): raise TypeError("exact native plan inputs required") + if selection_digest is not None: + digest_text(selection_digest, "selection_digest") projection = parse_native_workload(workload) runtime = native_runtime_spec(runtime_capture) if (run.project_ref != resolved.project_ref @@ -147,7 +155,8 @@ def __init__( self._clock = clock self._qualification, self._qualification_authority = qualification, qualification_authority self._restricted_admission, self._restricted_authority = restricted_admission, restricted_authority - available = self.admitted() + self._selection_digest, self._retained_read = selection_digest, retained_read + available = self.qualified() or self.restricted() or (retained_read and self.restricted_selection()) self.descriptor = ProviderDescriptor( "synaptic-provider-descriptor/v1", "native-mlx", "Apple native MLX", "1.0.0", ProviderCapabilities(available, False, False, False, available, False), @@ -188,30 +197,42 @@ def require_qualified(self): if not self.qualified(): raise ValueError("native_hardware_qualification_required") - def restricted(self): + def restricted_selection(self): proof = self._restricted_admission try: workload = parse_native_workload(self.workload) - now = datetime.fromisoformat(self._clock.now().replace("Z", "+00:00")) - expires = datetime.fromisoformat(proof.expires_at.replace("Z", "+00:00")) return ( type(proof) is NativeRestrictedAdmissionV1 and proof.profile_digest == self.profile_digest and proof.source_digest == self.resolved.source_digest and proof.runtime_digest == self.resolved.runtime_digest and proof.prepared_digest == workload.source.content_digest + and proof.workload_digest == self.resolved.workload_digest + and self._selection_digest is not None + and proof.selection_digest == self._selection_digest and proof.run_id == self.run.run_id and proof.maximum_updates == workload.controls.iterations - and now < expires <= now + timedelta(hours=1) and self._restricted_authority.authenticate(proof) is True ) except Exception: return False + def restricted(self): + if not self.restricted_selection(): + return False + try: + now = datetime.fromisoformat(self._clock.now().replace("Z", "+00:00")) + expires = datetime.fromisoformat(self._restricted_admission.expires_at.replace("Z", "+00:00")) + return now < expires <= now + timedelta(hours=1) + except Exception: + return False + def admitted(self): return self.qualified() or self.restricted() def require_admitted(self, *, attempt=None, model_digest=None, tokenizer_digest=None): + if self._retained_read: + raise ValueError("native_retained_read_has_no_dispatch_authority") if self.qualified(): return proof = self._restricted_admission @@ -245,7 +266,7 @@ def preflight(self, plan): raise ValueError("native plan differs") now = self._clock.now() expiry = (datetime.fromisoformat(now.replace("Z", "+00:00")) + timedelta(minutes=15)) - ready = self.admitted() + ready = not self._retained_read and self.admitted() return TrainingPreflight( plan.plan_fingerprint, ready, now, expiry.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"), From aba3c57699f6617878440b3126d15e475e295651 Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:19:56 -0400 Subject: [PATCH 03/12] Reap owned native host commands on interruption --- .../training_qualification/bounded_command.py | 4 +- tests/cli/test_native_host.py | 37 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/examples/training_qualification/bounded_command.py b/examples/training_qualification/bounded_command.py index 9bd9e10a..bda32ed5 100644 --- a/examples/training_qualification/bounded_command.py +++ b/examples/training_qualification/bounded_command.py @@ -71,7 +71,7 @@ def run_bounded_command(argv, *, cwd=None, environment=None, input_bytes=b"", if remaining <= 0: raise BoundedCommandError("bounded_command_deadline") return process.wait(timeout=remaining), b"".join(stdout) - except Exception: + except BaseException as error: if process is not None: try: os.killpg(process.pid, signal.SIGKILL) @@ -81,6 +81,8 @@ def run_bounded_command(argv, *, cwd=None, environment=None, input_bytes=b"", process.wait(timeout=2) except subprocess.TimeoutExpired: pass + if not isinstance(error, Exception): + raise raise BoundedCommandError("bounded_command_failed") from None finally: selector.close() diff --git a/tests/cli/test_native_host.py b/tests/cli/test_native_host.py index a5ff0255..482a4083 100644 --- a/tests/cli/test_native_host.py +++ b/tests/cli/test_native_host.py @@ -157,6 +157,43 @@ def test_host_command_flood_kills_owned_session(tmp_path): os.kill(pid, 0) +@pytest.mark.skipif(os.name != "posix", reason="native host requires POSIX process groups") +@pytest.mark.parametrize("interruption", [KeyboardInterrupt, SystemExit]) +def test_host_command_interruption_reaps_owned_child(monkeypatch, interruption): + from examples.training_qualification import bounded_command + + children = [] + real_popen = bounded_command.subprocess.Popen + selector_class = bounded_command.selectors.DefaultSelector + + def popen(*args, **kwargs): + child = real_popen(*args, **kwargs) + children.append(child) + return child + + class InterruptingSelector(selector_class): + def select(self, timeout=None): + raise interruption() + + monkeypatch.setattr(bounded_command.subprocess, "Popen", popen) + monkeypatch.setattr(bounded_command.selectors, "DefaultSelector", InterruptingSelector) + try: + with pytest.raises(interruption): + run_bounded_command( + [sys.executable, "-I", "-c", "import time; time.sleep(30)"], + maximum=4096, timeout=3, + ) + assert len(children) == 1 and children[0].returncode is not None + with pytest.raises(ProcessLookupError): + os.kill(children[0].pid, 0) + finally: + # A failed regression must not leave the test's child behind. + for child in children: + if child.poll() is None: + child.kill() + child.wait(timeout=2) + + @pytest.fixture def host_case(publication, monkeypatch): """Real host composition/store with only Mac metadata and ML child substituted.""" From 8baceba2f08c4750ecc4c96a3f3aae20f0c58078 Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:44:17 -0400 Subject: [PATCH 04/12] Prepare bounded native Mac model acquisition candidate and r5 evidence --- .agents/skills/fine-tuning/SKILL.md | 1 + .../reference/native-local-training.md | 16 + .claude/skills/fine-tuning/SKILL.md | 1 + .../reference/native-local-training.md | 16 + .skills/fine-tuning/SKILL.md | 1 + .../reference/native-local-training.md | 16 + .../training_qualification/native-host.md | 67 +++- .../native-model-acquisition.example.json | 11 + .../native_model_preparation.py | 361 ++++++++++++++++++ tests/cli/test_native_model_preparation.py | 165 ++++++++ 10 files changed, 653 insertions(+), 2 deletions(-) create mode 100644 examples/training_qualification/native-model-acquisition.example.json create mode 100644 examples/training_qualification/native_model_preparation.py create mode 100644 tests/cli/test_native_model_preparation.py diff --git a/.agents/skills/fine-tuning/SKILL.md b/.agents/skills/fine-tuning/SKILL.md index 751c1325..5e7c1136 100644 --- a/.agents/skills/fine-tuning/SKILL.md +++ b/.agents/skills/fine-tuning/SKILL.md @@ -59,6 +59,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | | Native MLX restricted host consumer | `python -B examples/training_qualification/native_host.py inspect ` is read-only; exact-run `start` requires an externally signed root admission and separately supplied public-key pin. No live Mac qualification yet; see `reference/native-local-training.md` | | Capture reviewed native runtime bytes | `python -B scripts/capture_native_mlx_runtime.py --config --dependency-lock --output ` — observation only; explicit roots, dependencies and original pins required; see `reference/native-local-training.md` | +| Prepare a reviewed private native model snapshot | `python -B examples/training_qualification/native_model_preparation.py inspect ` is local-only; `metadata-candidate` performs one bounded SDK metadata request and `acquire` downloads only after separate exact inventory review. Neither grants training admission; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | | Warm Space deploy | `python3 Trainers/cloud/scripts/manage_space.py deploy --space-id / --template vllm_warm --base-image ghcr.io//: --hardware a10g-small --sleep-time 3600 --var BASE_MODEL=` | diff --git a/.agents/skills/fine-tuning/reference/native-local-training.md b/.agents/skills/fine-tuning/reference/native-local-training.md index ee200a38..25278c15 100644 --- a/.agents/skills/fine-tuning/reference/native-local-training.md +++ b/.agents/skills/fine-tuning/reference/native-local-training.md @@ -214,6 +214,22 @@ configured member archive from the verified snapshot. Lineage/metrics retain the exact worker bindings. Finite file/aggregate budgets and public 1 MiB stream chunks apply. CPU verification does not rerun the final evaluation. +R5 Mac preparation retained a clean pushed `aba3c576` source clone and two +genuine synthetic prepared publications under a fresh 0700 private stage. +The selected Homebrew stdlib cannot currently pass static capture: its Cellar +ancestor is group-writable. The candidate venv lacks `jsonschema` required by +the current host composition. Preserve both as blockers rather than changing +shared Homebrew permissions, weakening native tree traversal, or claiming a +capture. Existing wheel-cache metadata does not attest absent wheel archives. +The checked-in `examples/training_qualification/native_model_preparation.py` +is the later reviewed model-acquisition action: `inspect` is local, the bounded +`metadata-candidate` requests only upstream metadata, and `acquire` requires a +separately reviewed complete member/tokenizer selection. It verifies upstream +hashes and publishes private create-only snapshots. Its receipt is +`OBSERVED_NOT_ADMITTED`. No Mac model bytes or tokenizer labels have yet been +acquired/reviewed. See `examples/training_qualification/native-host.md` for +exact staging paths, publication digests and commands. + See `docs/plans/native-desktop-training-api-v1.md` for remaining review and real-Mac acceptance. The CPU facade tests substitute the ML child; they are not a hardware result and must never unlock the candidate. diff --git a/.claude/skills/fine-tuning/SKILL.md b/.claude/skills/fine-tuning/SKILL.md index 751c1325..5e7c1136 100644 --- a/.claude/skills/fine-tuning/SKILL.md +++ b/.claude/skills/fine-tuning/SKILL.md @@ -59,6 +59,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | | Native MLX restricted host consumer | `python -B examples/training_qualification/native_host.py inspect ` is read-only; exact-run `start` requires an externally signed root admission and separately supplied public-key pin. No live Mac qualification yet; see `reference/native-local-training.md` | | Capture reviewed native runtime bytes | `python -B scripts/capture_native_mlx_runtime.py --config --dependency-lock --output ` — observation only; explicit roots, dependencies and original pins required; see `reference/native-local-training.md` | +| Prepare a reviewed private native model snapshot | `python -B examples/training_qualification/native_model_preparation.py inspect ` is local-only; `metadata-candidate` performs one bounded SDK metadata request and `acquire` downloads only after separate exact inventory review. Neither grants training admission; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | | Warm Space deploy | `python3 Trainers/cloud/scripts/manage_space.py deploy --space-id / --template vllm_warm --base-image ghcr.io//: --hardware a10g-small --sleep-time 3600 --var BASE_MODEL=` | diff --git a/.claude/skills/fine-tuning/reference/native-local-training.md b/.claude/skills/fine-tuning/reference/native-local-training.md index ee200a38..25278c15 100644 --- a/.claude/skills/fine-tuning/reference/native-local-training.md +++ b/.claude/skills/fine-tuning/reference/native-local-training.md @@ -214,6 +214,22 @@ configured member archive from the verified snapshot. Lineage/metrics retain the exact worker bindings. Finite file/aggregate budgets and public 1 MiB stream chunks apply. CPU verification does not rerun the final evaluation. +R5 Mac preparation retained a clean pushed `aba3c576` source clone and two +genuine synthetic prepared publications under a fresh 0700 private stage. +The selected Homebrew stdlib cannot currently pass static capture: its Cellar +ancestor is group-writable. The candidate venv lacks `jsonschema` required by +the current host composition. Preserve both as blockers rather than changing +shared Homebrew permissions, weakening native tree traversal, or claiming a +capture. Existing wheel-cache metadata does not attest absent wheel archives. +The checked-in `examples/training_qualification/native_model_preparation.py` +is the later reviewed model-acquisition action: `inspect` is local, the bounded +`metadata-candidate` requests only upstream metadata, and `acquire` requires a +separately reviewed complete member/tokenizer selection. It verifies upstream +hashes and publishes private create-only snapshots. Its receipt is +`OBSERVED_NOT_ADMITTED`. No Mac model bytes or tokenizer labels have yet been +acquired/reviewed. See `examples/training_qualification/native-host.md` for +exact staging paths, publication digests and commands. + See `docs/plans/native-desktop-training-api-v1.md` for remaining review and real-Mac acceptance. The CPU facade tests substitute the ML child; they are not a hardware result and must never unlock the candidate. diff --git a/.skills/fine-tuning/SKILL.md b/.skills/fine-tuning/SKILL.md index 751c1325..5e7c1136 100644 --- a/.skills/fine-tuning/SKILL.md +++ b/.skills/fine-tuning/SKILL.md @@ -59,6 +59,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | | Native MLX restricted host consumer | `python -B examples/training_qualification/native_host.py inspect ` is read-only; exact-run `start` requires an externally signed root admission and separately supplied public-key pin. No live Mac qualification yet; see `reference/native-local-training.md` | | Capture reviewed native runtime bytes | `python -B scripts/capture_native_mlx_runtime.py --config --dependency-lock --output ` — observation only; explicit roots, dependencies and original pins required; see `reference/native-local-training.md` | +| Prepare a reviewed private native model snapshot | `python -B examples/training_qualification/native_model_preparation.py inspect ` is local-only; `metadata-candidate` performs one bounded SDK metadata request and `acquire` downloads only after separate exact inventory review. Neither grants training admission; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | | Warm Space deploy | `python3 Trainers/cloud/scripts/manage_space.py deploy --space-id / --template vllm_warm --base-image ghcr.io//: --hardware a10g-small --sleep-time 3600 --var BASE_MODEL=` | diff --git a/.skills/fine-tuning/reference/native-local-training.md b/.skills/fine-tuning/reference/native-local-training.md index ee200a38..25278c15 100644 --- a/.skills/fine-tuning/reference/native-local-training.md +++ b/.skills/fine-tuning/reference/native-local-training.md @@ -214,6 +214,22 @@ configured member archive from the verified snapshot. Lineage/metrics retain the exact worker bindings. Finite file/aggregate budgets and public 1 MiB stream chunks apply. CPU verification does not rerun the final evaluation. +R5 Mac preparation retained a clean pushed `aba3c576` source clone and two +genuine synthetic prepared publications under a fresh 0700 private stage. +The selected Homebrew stdlib cannot currently pass static capture: its Cellar +ancestor is group-writable. The candidate venv lacks `jsonschema` required by +the current host composition. Preserve both as blockers rather than changing +shared Homebrew permissions, weakening native tree traversal, or claiming a +capture. Existing wheel-cache metadata does not attest absent wheel archives. +The checked-in `examples/training_qualification/native_model_preparation.py` +is the later reviewed model-acquisition action: `inspect` is local, the bounded +`metadata-candidate` requests only upstream metadata, and `acquire` requires a +separately reviewed complete member/tokenizer selection. It verifies upstream +hashes and publishes private create-only snapshots. Its receipt is +`OBSERVED_NOT_ADMITTED`. No Mac model bytes or tokenizer labels have yet been +acquired/reviewed. See `examples/training_qualification/native-host.md` for +exact staging paths, publication digests and commands. + See `docs/plans/native-desktop-training-api-v1.md` for remaining review and real-Mac acceptance. The CPU facade tests substitute the ML child; they are not a hardware result and must never unlock the candidate. diff --git a/examples/training_qualification/native-host.md b/examples/training_qualification/native-host.md index 672b5326..2011e131 100644 --- a/examples/training_qualification/native-host.md +++ b/examples/training_qualification/native-host.md @@ -54,10 +54,73 @@ the publication when the consumer reads it. For the message attempt, review the actual shared preprocessor output; a `prompt_completion` label does not by itself prove assistant-only labels. +## R5 Mac preparation and remaining boundary + +The authorized preparation used a new 0700 directory at +`/Users/jrosenbaum/.local/share/synaptic-qualification/native-mlx-candidate-20261020/prep-r5-aba3c576-20261020`. +Its `source/` is a clean isolated clone of pushed commit +`aba3c57699f6617878440b3126d15e475e295651`; the user's dirty checkout +was untouched. The existing private venv is CPython 3.11.14 with the original +50-dependency lock (SHA-256 `f9a6298260cd17f2c3d567ae3674108a71efa68e99369906487a4d8c0feb7117`). +`uv pip check` passes. Retained install logs and unpacked wheel-cache metadata +are observations, not selected archive attestation: the original wheel archives +are absent. The host reports M4, 24 GiB and macOS 27.0.1 (26A434). + +The checked-in static capture refused the actual Homebrew stdlib root because +`/opt/homebrew/Cellar` has mode 0775, while `NativeTree` requires every runtime +ancestor to be free of group/other write access. No capture was published. Do +not change shared Homebrew permissions or weaken the capture check to pass it. +The candidate venv also lacks `jsonschema`, imported by the current host +composition through `tuner.project`; the existing venv was not modified. +Resolve these exact closure/composition blockers and recapture on the final +pushed source before any signed execution admission. + +Two synthetic publications were made through the existing `ingest` and +`prepare-dataset` CLIs, then copied into private Mac staging. Raw/full-sequence +publication digest: `0e4951bb2e54ed0985e6ca0d4c9940f0f5d75ccb78821c14544ef44897580cbe`. +Prompt/completion publication digest: +`d22b95afbf5e415fa6a363381fda7714cfd0d7817d3b98a5e44e3149250ee9a0`. +Each has three training and two validation rows, with declared source IDs and +split identities. Their private project is at `prep-r5-aba3c576-20261020/prepared-project/`. +Mac content hashes matched the local publications; Mac API readback is blocked +by the missing `jsonschema`. The complete rows were reviewed for their source +shape, but exact tokenizer IDs, assistant-only label masks, and +`enable_thinking=false` still require the pinned tokenizer and shared +preprocessor before training. These data hashes do not grant training authority. + +The model workspace is empty. Its adjacent `model-acquisition-draft.json` is +incomplete by design: `members` and `tokenizer_members` are empty. No model +metadata request, weights download or ML import has run. The reusable +`native_model_preparation.py` action supports a later separately reviewed +acquisition using exact complete upstream metadata, finite bounds and private +create-only publication. Its observations never sign admission. The next +reviewed sequence is: + +```sh +candidate_python=/Users/jrosenbaum/.local/share/synaptic-qualification/native-mlx-candidate-20261020/venv/bin/python +stage=/Users/jrosenbaum/.local/share/synaptic-qualification/native-mlx-candidate-20261020/prep-r5-aba3c576-20261020 +# After explicit approval for an SDK metadata request, from the final clean source: +"$candidate_python" -B examples/training_qualification/native_model_preparation.py \ + metadata-candidate "$stage/model-acquisition-draft.json" +# Independently review every candidate member and complete tokenizer selection; +# copy the reviewed selection into a fresh private config and workspace. Inspect +# is local-only. Acquire is a later bounded model download, not approved here. +"$candidate_python" -B examples/training_qualification/native_model_preparation.py \ + inspect /ABSOLUTE/REVIEWED/model-acquisition.json +"$candidate_python" -B examples/training_qualification/native_model_preparation.py \ + acquire /ABSOLUTE/REVIEWED/model-acquisition.json +``` + +The exact acquisition schema and pins are in +[`native-model-acquisition.example.json`](native-model-acquisition.example.json). +The template paths are placeholders, never execution authority. Review the +complete metadata candidate, selected bytes, both snapshot manifests and +tokenizer loadability before wiring them into the host config. + ## Reusable preparation and review sequence -These are commands for the **later approved Mac stage**. They have not been -run against the Mac for this increment. +The following capture/admission/start commands are for a **later reviewed +stage**. The current source and runtime closure cannot pass capture yet. ```sh # From the clean pushed engine checkout, with the existing private 3.11.14 venv. diff --git a/examples/training_qualification/native-model-acquisition.example.json b/examples/training_qualification/native-model-acquisition.example.json new file mode 100644 index 00000000..51021434 --- /dev/null +++ b/examples/training_qualification/native-model-acquisition.example.json @@ -0,0 +1,11 @@ +{ + "schema": "synaptic-native-model-acquisition/v1", + "repository": "mlx-community/Qwen3-0.6B-4bit", + "revision": "73e3e38d981303bc594367cd910ea6eb48349da8", + "workspace_root": "/FRESH_OWNER_ONLY_0700_MODEL_WORKSPACE", + "maximum_bytes": 536870912, + "maximum_files": 128, + "timeout_seconds": 300, + "members": [], + "tokenizer_members": [] +} diff --git a/examples/training_qualification/native_model_preparation.py b/examples/training_qualification/native_model_preparation.py new file mode 100644 index 00000000..0c7ac8e1 --- /dev/null +++ b/examples/training_qualification/native_model_preparation.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python3 +"""Private, bounded model snapshot preparation candidate; never grants training authority. + +``inspect`` validates an independently reviewed inventory without network access. +``acquire`` is a later, separately authorized effect. It retains a consumed +workspace on any uncertain failure and never retries automatically. +""" + +from __future__ import annotations + +import argparse +from contextlib import ExitStack, redirect_stderr, redirect_stdout +import hashlib +import json +import os +from pathlib import Path +import re +import stat +import sys +import unicodedata + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from bounded_command import run_bounded_command +from tuner.runtime.native_files import NativeTree, relative_member + + +_SCHEMA = "synaptic-native-model-acquisition/v1" +_ENDPOINT = "https://huggingface.co" +_MAX_BYTES = 512 * 1024 * 1024 +_MAX_FILES = 128 +_MEMBER_FIELDS = {"path", "size", "kind", "upstream_digest"} + + +def _canonical(value): + return json.dumps(value, sort_keys=True, separators=(",", ":"), + ensure_ascii=False, allow_nan=False).encode("utf-8") + + +def _pairs(items): + result = {} + for name, value in items: + if name in result: + raise ValueError("duplicate model preparation field") + result[name] = value + return result + + +def _read_config(path): + path = Path(path) + if not path.is_absolute() or str(path) != os.path.normpath(str(path)): + raise ValueError("model preparation config path differs") + with NativeTree(path.parent, private=True) as parent: + raw = parent.read(path.name, maximum=1 << 20) + value = json.loads(raw, object_pairs_hook=_pairs, + parse_constant=lambda _: (_ for _ in ()).throw(ValueError())) + return value + + +def _member(value): + if type(value) is not dict or set(value) != _MEMBER_FIELDS: + raise ValueError("model member fields differ") + name = value["path"] + relative_member(name) + if (any(part in {".git", ".cache"} for part in name.split("/")) + or unicodedata.normalize("NFC", name) != name + or any(character in name for character in "*?[]")): + raise ValueError("model member path is unsupported") + size, kind, digest = value["size"], value["kind"], value["upstream_digest"] + if (type(size) is not int or not 0 <= size <= _MAX_BYTES + or kind not in ("git", "lfs") or type(kind) is not str + or type(digest) is not str + or re.fullmatch(r"[0-9a-f]{40}" if kind == "git" else r"[0-9a-f]{64}", digest) is None): + raise ValueError("model member metadata is invalid") + return value + + +def _validate_common(value): + fields = {"schema", "repository", "revision", "workspace_root", + "maximum_bytes", "maximum_files", "timeout_seconds", "members", + "tokenizer_members"} + if type(value) is not dict or set(value) != fields or value["schema"] != _SCHEMA: + raise ValueError("model preparation configuration differs") + repository, revision = value["repository"], value["revision"] + if (type(repository) is not str + or re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repository) is None + or type(revision) is not str or re.fullmatch(r"[0-9a-f]{40}", revision) is None): + raise ValueError("model repository pin is invalid") + workspace = Path(value["workspace_root"]) + if (type(value["workspace_root"]) is not str or not workspace.is_absolute() + or str(workspace) != value["workspace_root"] + or str(workspace) != os.path.normpath(str(workspace))): + raise ValueError("model private workspace is invalid") + if (type(value["maximum_bytes"]) is not int or not 1 <= value["maximum_bytes"] <= _MAX_BYTES + or type(value["maximum_files"]) is not int or not 1 <= value["maximum_files"] <= _MAX_FILES + or type(value["timeout_seconds"]) is not int or not 1 <= value["timeout_seconds"] <= 300): + raise ValueError("model acquisition bounds differ") + with NativeTree(workspace, private=True): + pass + return workspace + + +def _validate(value): + workspace = _validate_common(value) + raw_members = value["members"] + if (type(raw_members) is not list or not 1 <= len(raw_members) <= value["maximum_files"]): + raise ValueError("model member count differs") + members = tuple(_member(item) for item in raw_members) + names = tuple(item["path"] for item in members) + if (names != tuple(sorted(set(names))) + or len({name.casefold() for name in names}) != len(names) + or sum(item["size"] for item in members) > value["maximum_bytes"] + or "config.json" not in names or not any(name.endswith(".safetensors") for name in names)): + raise ValueError("model complete selection or bound differs") + selected = value["tokenizer_members"] + if (type(selected) is not list or not selected + or selected != sorted(set(selected)) or not set(selected) <= set(names)): + raise ValueError("model tokenizer selection differs") + with NativeTree(workspace, private=True) as root: + for name in ("scratch", "model-snapshot", "tokenizer-snapshot", + "model-snapshot.json", "tokenizer-snapshot.json", "acquisition-result.json"): + try: + os.stat(name, dir_fd=root.fd, follow_symlinks=False) + except FileNotFoundError: + continue + raise ValueError("model workspace has a prior acquisition claim") + return members + + +def _upstream(member, raw): + if len(raw) != member["size"]: + raise ValueError("model member byte count differs") + if member["kind"] == "lfs": + digest = hashlib.sha256(raw).hexdigest() + else: + digest = hashlib.sha1(f"blob {len(raw)}\0".encode() + raw).hexdigest() + if digest != member["upstream_digest"]: + raise ValueError("model member upstream digest differs") + return hashlib.sha256(raw).hexdigest() + + +def _observe_metadata(config, *, api_type): + info = api_type(endpoint=_ENDPOINT, token=False).model_info( + config["repository"], revision=config["revision"], files_metadata=True, + token=False, timeout=30, + ) + if info.sha != config["revision"] or type(info.siblings) not in (list, tuple): + raise ValueError("model metadata revision differs") + observed = [] + for sibling in info.siblings: + kind = "lfs" if sibling.lfs is not None else "git" + digest = sibling.lfs.sha256 if kind == "lfs" else sibling.blob_id + observed.append(_member({"path": sibling.rfilename, "size": sibling.size, + "kind": kind, "upstream_digest": digest})) + if len(observed) > config["maximum_files"]: + raise ValueError("model metadata count exceeds bound") + names = tuple(item["path"] for item in observed) + if (len(names) != len(set(names)) or len({name.casefold() for name in names}) != len(names) + or sum(item["size"] for item in observed) > config["maximum_bytes"]): + raise ValueError("model metadata exceeds exact selection bounds") + return tuple(sorted(observed, key=lambda item: item["path"])) + + +def _metadata(members, config, *, api_type): + observed = _observe_metadata(config, api_type=api_type) + if observed != members: + raise ValueError("model metadata differs from reviewed complete selection") + + +def _metadata_candidate(config, *, api_type=None): + workspace = _validate_common(config) + if config["members"] != [] or config["tokenizer_members"] != []: + raise ValueError("metadata candidate must not contain reviewed members") + with NativeTree(workspace, private=True) as root: + try: + os.stat("metadata-candidate.json", dir_fd=root.fd, follow_symlinks=False) + except FileNotFoundError: + pass + else: + raise ValueError("metadata candidate already exists") + if api_type is None: + from huggingface_hub import HfApi + api_type = HfApi + with open(os.devnull, "w") as sink, redirect_stdout(sink), redirect_stderr(sink): + observed = _observe_metadata(config, api_type=api_type) + names = tuple(item["path"] for item in observed) + if "config.json" not in names or not any(name.endswith(".safetensors") for name in names): + raise ValueError("model metadata is incomplete") + candidate = {"schema": "synaptic-native-model-metadata-candidate/v1", + "status": "REVIEW_REQUIRED_NO_AUTHORITY", "repository": config["repository"], + "revision": config["revision"], "members": list(observed), + "declared_bytes": sum(item["size"] for item in observed), + "member_count": len(observed)} + with NativeTree(workspace, private=True) as root: + root.write_exclusive("metadata-candidate.json", _canonical(candidate)) + return {"status": candidate["status"], "repository": config["repository"], + "revision": config["revision"], "member_count": candidate["member_count"], + "declared_bytes": candidate["declared_bytes"], + "metadata_candidate_sha256": hashlib.sha256(_canonical(candidate)).hexdigest()} + + +def _publish_member(root, name, raw): + parts = relative_member(name) + with ExitStack() as stack: + parent = root.fd + for part in parts[:-1]: + try: + os.mkdir(part, mode=0o700, dir_fd=parent) + except FileExistsError: + pass + fd = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent) + stack.callback(os.close, fd) + info = os.fstat(fd) + if info.st_uid != os.getuid() or not stat.S_ISDIR(info.st_mode) or info.st_mode & 0o077: + raise ValueError("model destination parent is not private") + parent = fd + fd = os.open(parts[-1], os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, dir_fd=parent) + with os.fdopen(fd, "wb") as output: + output.write(raw) + output.flush() + os.fsync(output.fileno()) + root.check() + + +def _acquire(config, *, api_type=None, download=None): + previous = os.umask(0o077) + try: + return _acquire_private(config, api_type=api_type, download=download) + finally: + os.umask(previous) + + +def _acquire_private(config, *, api_type=None, download=None): + members = _validate(config) + workspace = Path(config["workspace_root"]) + with NativeTree(workspace, private=True) as root: + os.mkdir("scratch", mode=0o700, dir_fd=root.fd) + root.check() + if api_type is None or download is None: + from huggingface_hub import HfApi, snapshot_download + api_type = api_type or HfApi + download = download or snapshot_download + with open(os.devnull, "w") as sink, redirect_stdout(sink), redirect_stderr(sink): + _metadata(members, config, api_type=api_type) + scratch = workspace / "scratch" + with NativeTree(scratch, private=True) as held: + os.mkdir("repository", mode=0o700, dir_fd=held.fd) + held.check() + repository = scratch / "repository" + with open(os.devnull, "w") as sink, redirect_stdout(sink), redirect_stderr(sink): + result = download( + repo_id=config["repository"], revision=config["revision"], token=False, + endpoint=_ENDPOINT, local_dir=repository, cache_dir=scratch / "sdk-cache", + allow_patterns=[item["path"] for item in members], max_workers=1, + etag_timeout=10, + ) + if Path(result) != repository: + raise ValueError("model SDK destination differs") + bytes_by_name = {} + content_hashes = {} + with NativeTree(repository, private=True) as source: + for member in members: + raw = source.read(member["path"], maximum=member["size"]) + content_hashes[member["path"]] = _upstream(member, raw) + bytes_by_name[member["path"]] = raw + source.check() + with NativeTree(workspace, private=True) as root: + os.mkdir("model-snapshot", mode=0o700, dir_fd=root.fd) + os.mkdir("tokenizer-snapshot", mode=0o700, dir_fd=root.fd) + root.check() + manifests = [] + for role, names in (("model", tuple(item["path"] for item in members)), + ("tokenizer", tuple(config["tokenizer_members"]))): + target = workspace / f"{role}-snapshot" + with NativeTree(target, private=True) as published: + for name in names: + _publish_member(published, name, bytes_by_name[name]) + inventory = published.inventory(maximum_total=config["maximum_bytes"]) + if (tuple(item.path for item in inventory) != names + or any(item.sha256 != content_hashes[item.path] for item in inventory)): + raise ValueError("model publication readback differs") + manifest = {"repository": config["repository"], "revision": config["revision"], + "root": str(target), + "files": [{"path": item.path, "size": item.size, "sha256": item.sha256} + for item in inventory]} + manifests.append((role, manifest)) + receipt = {"schema": "synaptic-native-model-acquisition-observation/v1", + "status": "OBSERVED_NOT_ADMITTED", "repository": config["repository"], + "revision": config["revision"], "selection_sha256": hashlib.sha256(_canonical(config)).hexdigest(), + "model_manifest_sha256": hashlib.sha256(_canonical(manifests[0][1])).hexdigest(), + "tokenizer_manifest_sha256": hashlib.sha256(_canonical(manifests[1][1])).hexdigest()} + with NativeTree(workspace, private=True) as root: + for role, manifest in manifests: + root.write_exclusive(f"{role}-snapshot.json", _canonical(manifest)) + root.write_exclusive("acquisition-result.json", _canonical(receipt)) + return receipt + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("inspect", "metadata-candidate", "acquire", + "_metadata_worker", "_worker")) + parser.add_argument("config", type=Path) + args = parser.parse_args(argv) + try: + config = _read_config(args.config) + if args.action in {"metadata-candidate", "_metadata_worker"}: + _validate_common(config) + else: + _validate(config) + if args.action == "inspect": + result = {"status": "REVIEW_REQUIRED_NO_NETWORK", + "selection_sha256": hashlib.sha256(_canonical(config)).hexdigest(), + "member_count": len(config["members"]), + "declared_bytes": sum(item["size"] for item in config["members"])} + elif args.action == "_metadata_worker": + result = _metadata_candidate(config) + elif args.action == "_worker": + result = _acquire(config) + else: + environment = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "LC_ALL": "C", + "HF_HOME": config["workspace_root"], + "HF_HUB_DISABLE_TELEMETRY": "1", "TMPDIR": config["workspace_root"]} + status, output = run_bounded_command( + [sys.executable, "-I", "-B", str(Path(__file__).resolve()), + "_metadata_worker" if args.action == "metadata-candidate" else "_worker", + str(args.config)], environment=environment, + maximum=4096, timeout=config["timeout_seconds"], + ) + if status != 0: + raise ValueError("model acquisition worker uncertain") + worker_result = json.loads(output, object_pairs_hook=_pairs) + if _canonical(worker_result) != output: + raise ValueError("model acquisition worker result differs") + if args.action == "metadata-candidate": + with NativeTree(Path(config["workspace_root"]), private=True) as root: + candidate = json.loads(root.read("metadata-candidate.json", maximum=1 << 20)) + result = {"status": candidate["status"], "repository": config["repository"], + "revision": config["revision"], "member_count": candidate["member_count"], + "declared_bytes": candidate["declared_bytes"], + "metadata_candidate_sha256": hashlib.sha256(_canonical(candidate)).hexdigest()} + if result != worker_result or result["status"] != "REVIEW_REQUIRED_NO_AUTHORITY": + raise ValueError("model metadata candidate differs") + else: + with NativeTree(Path(config["workspace_root"]), private=True) as root: + result = json.loads(root.read("acquisition-result.json", maximum=1 << 20)) + if (result != worker_result or result.get("status") != "OBSERVED_NOT_ADMITTED" + or result.get("selection_sha256") != hashlib.sha256(_canonical(config)).hexdigest()): + raise ValueError("model acquisition result differs") + print(_canonical(result).decode()) + return 0 + except Exception: + print('{"diagnostic_code":"native_model_preparation_refused"}') + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/cli/test_native_model_preparation.py b/tests/cli/test_native_model_preparation.py new file mode 100644 index 00000000..2c1a29d4 --- /dev/null +++ b/tests/cli/test_native_model_preparation.py @@ -0,0 +1,165 @@ +"""Private native model preparation contracts; no SDK network or ML imports.""" + +from hashlib import sha1 +import json +import os +from pathlib import Path +from types import SimpleNamespace +import tempfile + +import pytest + +from examples.training_qualification import native_model_preparation as preparation + + +def _git(raw): + return sha1(f"blob {len(raw)}\0".encode() + raw).hexdigest() + + +@pytest.fixture +def candidate(): + previous = os.umask(0o077) + try: + with tempfile.TemporaryDirectory(prefix="native-model-prep-", dir=Path.home()) as name: + root = Path(name) + files = {"config.json": b"{}", "model.safetensors": b"weights", + "tokenizer.json": b"{}"} + config = { + "schema": "synaptic-native-model-acquisition/v1", + "repository": "reviewed/model", "revision": "a" * 40, + "workspace_root": str(root), "maximum_bytes": 512, + "maximum_files": 3, "timeout_seconds": 5, + "members": [{"path": path, "size": len(raw), "kind": "git", + "upstream_digest": _git(raw)} for path, raw in sorted(files.items())], + "tokenizer_members": ["tokenizer.json"], + } + (root / "config.json").write_bytes(preparation._canonical(config)) + yield config, files, root + finally: + os.umask(previous) + + +def _sdk(config, files, *, add_extra=False, redirect=None): + calls = [] + + class Api: + def __init__(self, *, endpoint, token): + assert endpoint == "https://huggingface.co" and token is False + + def model_info(self, repository, *, revision, files_metadata, token, timeout): + calls.append("metadata") + assert repository == config["repository"] and revision == config["revision"] + assert files_metadata and token is False and timeout == 30 + siblings = [SimpleNamespace(rfilename=name, size=len(raw), lfs=None, + blob_id=_git(raw)) for name, raw in sorted(files.items())] + if add_extra: + siblings.append(SimpleNamespace(rfilename="extra.txt", size=1, lfs=None, + blob_id=_git(b"x"))) + return SimpleNamespace(sha=revision, siblings=siblings) + + def download(**options): + calls.append("download") + assert options["repo_id"] == config["repository"] + assert options["revision"] == config["revision"] and options["token"] is False + assert options["max_workers"] == 1 and options["etag_timeout"] == 10 + assert options["allow_patterns"] == sorted(files) + root = Path(options["local_dir"]) + for name, raw in files.items(): + target = root / name + target.parent.mkdir(parents=True, exist_ok=True) + if redirect == name: + target.symlink_to(root / "config.json") + else: + target.write_bytes(raw) + return str(root) + + return Api, download, calls + + +def test_model_plan_is_local_and_fake_sdk_publishes_only_exact_bytes(candidate, capsys): + config, files, root = candidate + assert preparation.main(["inspect", str(root / "config.json")]) == 0 + assert json.loads(capsys.readouterr().out)["status"] == "REVIEW_REQUIRED_NO_NETWORK" + api, download, calls = _sdk(config, files) + receipt = preparation._acquire(config, api_type=api, download=download) + assert calls == ["metadata", "download"] + assert receipt["status"] == "OBSERVED_NOT_ADMITTED" + model = json.loads((root / "model-snapshot.json").read_bytes()) + tokenizer = json.loads((root / "tokenizer-snapshot.json").read_bytes()) + assert [item["path"] for item in model["files"]] == sorted(files) + assert [item["path"] for item in tokenizer["files"]] == ["tokenizer.json"] + assert (root / "model-snapshot/model.safetensors").read_bytes() == b"weights" + assert (root / "tokenizer-snapshot/tokenizer.json").read_bytes() == b"{}" + with pytest.raises(ValueError, match="prior acquisition claim"): + preparation._acquire(config, api_type=api, download=download) + assert calls == ["metadata", "download"] + + +def test_nested_model_publication_stays_private_under_ambient_group_umask(candidate): + config, files, root = candidate + files["weights/model.safetensors"] = files.pop("model.safetensors") + config["members"] = [ + {"path": name, "size": len(raw), "kind": "git", "upstream_digest": _git(raw)} + for name, raw in sorted(files.items()) + ] + api, download, calls = _sdk(config, files) + previous = os.umask(0o002) + try: + preparation._acquire(config, api_type=api, download=download) + finally: + os.umask(previous) + assert calls == ["metadata", "download"] + assert (root / "model-snapshot/weights/model.safetensors").read_bytes() == b"weights" + assert (root / "model-snapshot/weights").stat().st_mode & 0o077 == 0 + + +def test_model_metadata_substitution_rejects_before_download(candidate): + config, files, root = candidate + config["maximum_files"] = 4 + api, download, calls = _sdk(config, files, add_extra=True) + with pytest.raises(ValueError, match="reviewed complete selection"): + preparation._acquire(config, api_type=api, download=download) + assert calls == ["metadata"] + assert (root / "scratch").is_dir() and not (root / "model-snapshot").exists() + + +def test_metadata_candidate_is_review_only_and_create_once(candidate): + config, files, root = candidate + config["members"] = [] + config["tokenizer_members"] = [] + api, download, calls = _sdk(config, files) + result = preparation._metadata_candidate(config, api_type=api) + assert result["status"] == "REVIEW_REQUIRED_NO_AUTHORITY" + assert result["member_count"] == 3 and calls == ["metadata"] + stored = json.loads((root / "metadata-candidate.json").read_bytes()) + assert stored["members"] == [ + {"path": name, "size": len(raw), "kind": "git", "upstream_digest": _git(raw)} + for name, raw in sorted(files.items()) + ] + with pytest.raises(ValueError, match="already exists"): + preparation._metadata_candidate(config, api_type=api) + assert calls == ["metadata"] and not (root / "model-snapshot").exists() + + +def test_model_sdk_symlink_rejects_before_publication(candidate): + config, files, root = candidate + api, download, calls = _sdk(config, files, redirect="tokenizer.json") + with pytest.raises(OSError): + preparation._acquire(config, api_type=api, download=download) + assert calls == ["metadata", "download"] + assert not (root / "model-snapshot").exists() + + +@pytest.mark.parametrize("change", [ + lambda c: c.update(maximum_bytes=512 * 1024 * 1024 + 1), + lambda c: c.update(tokenizer_members=["missing.json"]), + lambda c: c["members"][0].update(path="../config.json"), + lambda c: c["members"][1].update(size=1024), +]) +def test_model_unreviewed_or_overbudget_selection_rejects_before_claim(candidate, change): + config, files, root = candidate + change(config) + api, download, calls = _sdk(config, files) + with pytest.raises(ValueError): + preparation._acquire(config, api_type=api, download=download) + assert calls == [] and not (root / "scratch").exists() From 37d895d2f13f39c3ed840068fa36c1a0766f3c3d Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:17:35 -0400 Subject: [PATCH 05/12] Prepare bounded private native runtime and model acquisition --- .agents/skills/fine-tuning/SKILL.md | 1 + .../reference/native-local-training.md | 40 +- .claude/skills/fine-tuning/SKILL.md | 1 + .../reference/native-local-training.md | 40 +- .skills/fine-tuning/SKILL.md | 1 + .../reference/native-local-training.md | 40 +- AGENTS.md | 8 + .../mlx_sft_mac/runtime-candidate/README.md | 56 + .../native-private-runtime.example.json | 458 ++++ .../requirements.host-additions.lock | 4 + .../requirements.runtime.lock | 1894 +++++++++++++++++ .../sft-qualification-proposal.json | 4 +- .../training_qualification/native-host.md | 57 +- .../native_model_preparation.py | 192 +- .../native_runtime_preparation.py | 491 +++++ tests/cli/test_native_model_preparation.py | 114 +- tests/cli/test_native_runtime_preparation.py | 189 ++ 17 files changed, 3540 insertions(+), 50 deletions(-) create mode 100644 Trainers/mlx_sft_mac/runtime-candidate/native-private-runtime.example.json create mode 100644 Trainers/mlx_sft_mac/runtime-candidate/requirements.host-additions.lock create mode 100644 Trainers/mlx_sft_mac/runtime-candidate/requirements.runtime.lock create mode 100644 examples/training_qualification/native_runtime_preparation.py create mode 100644 tests/cli/test_native_runtime_preparation.py diff --git a/.agents/skills/fine-tuning/SKILL.md b/.agents/skills/fine-tuning/SKILL.md index 5e7c1136..9c54c10b 100644 --- a/.agents/skills/fine-tuning/SKILL.md +++ b/.agents/skills/fine-tuning/SKILL.md @@ -59,6 +59,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | | Native MLX restricted host consumer | `python -B examples/training_qualification/native_host.py inspect ` is read-only; exact-run `start` requires an externally signed root admission and separately supplied public-key pin. No live Mac qualification yet; see `reference/native-local-training.md` | | Capture reviewed native runtime bytes | `python -B scripts/capture_native_mlx_runtime.py --config --dependency-lock --output ` — observation only; explicit roots, dependencies and original pins required; see `reference/native-local-training.md` | +| Plan private native Mac runtime | `python -B examples/training_qualification/native_runtime_preparation.py plan ` — local 54-wheel archive plan only; later `acquire`/`install` consume separate private claims and need independent review; see `reference/native-local-training.md` | | Prepare a reviewed private native model snapshot | `python -B examples/training_qualification/native_model_preparation.py inspect ` is local-only; `metadata-candidate` performs one bounded SDK metadata request and `acquire` downloads only after separate exact inventory review. Neither grants training admission; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.agents/skills/fine-tuning/reference/native-local-training.md b/.agents/skills/fine-tuning/reference/native-local-training.md index 25278c15..e1de1f0f 100644 --- a/.agents/skills/fine-tuning/reference/native-local-training.md +++ b/.agents/skills/fine-tuning/reference/native-local-training.md @@ -221,11 +221,43 @@ ancestor is group-writable. The candidate venv lacks `jsonschema` required by the current host composition. Preserve both as blockers rather than changing shared Homebrew permissions, weakening native tree traversal, or claiming a capture. Existing wheel-cache metadata does not attest absent wheel archives. +The current child also imports `jsonschema` through +`tuner.training.contracts` → `tuner.project.execution_source` → the eager +`tuner.project` initializer. The host and child need the same exact inventory +under the current `verify_native_material` contract. The r7 candidate keeps +the original 50 lock unchanged and adds four reviewed Mac wheel hashes in +`requirements.host-additions.lock`. Its flat `requirements.runtime.lock` is +exactly the original bytes plus those four lines (SHA-256 +`7c167018ee6bd61a3609971e3f834082c798e7bd2a3fa0c5c2564887da4fde4f`). +The candidate proposal selects this 54-package lock; it grants no runtime or +hardware authority. + +Use `examples/training_qualification/native_runtime_preparation.py plan` with +the exact checked-in `native-private-runtime.example.json` to inspect the +166,269,790-byte archive selection without effects. Only after independent +review, copy that config to a new 0600 file in an existing 0700 private parent +and use its `acquire`, `install`, and `verify` phases from a clean pushed source. +The CLI retains the exact Astral 3.11.14+20260211 interpreter archive and 54 +wheel archives, checks hashes before an offline uv local-mirror install, binds +the uv executable hash/version, confines caches and temporary paths to the +private root, and retains consumed claims. HTTPS transfers stop before writing +an excess byte; finite expansion checks are pre-install inventory bounds, not +a hard aggregate filesystem quota for uv temporary files. A 2 GiB retained +private-tree guard runs after each phase. Include narrow +`source/shared` in the subsequent final-source static capture alongside +`source/tuner` and `source/synaptic_tuner`. Do not chmod Homebrew or reuse the +old venv/capture attempt. + The checked-in `examples/training_qualification/native_model_preparation.py` -is the later reviewed model-acquisition action: `inspect` is local, the bounded -`metadata-candidate` requests only upstream metadata, and `acquire` requires a -separately reviewed complete member/tokenizer selection. It verifies upstream -hashes and publishes private create-only snapshots. Its receipt is +is the later reviewed model-acquisition action: `inspect` is local, +`metadata-candidate` reads at most 2 MiB of the exact Hub metadata response +before JSON/SDK `ModelInfo` parsing, and `acquire` requires a separately +reviewed complete member/tokenizer selection. Model files stream from the +exact pinned Hub resolve URLs into precreated private 0700 nested parents; +per-file and aggregate caps reject before excess writes. The SDK's unbounded +snapshot/cache download path is not used. Unexpected redirect hosts fail +closed. It verifies upstream hashes and publishes private create-only +snapshots. Its receipt is `OBSERVED_NOT_ADMITTED`. No Mac model bytes or tokenizer labels have yet been acquired/reviewed. See `examples/training_qualification/native-host.md` for exact staging paths, publication digests and commands. diff --git a/.claude/skills/fine-tuning/SKILL.md b/.claude/skills/fine-tuning/SKILL.md index 5e7c1136..9c54c10b 100644 --- a/.claude/skills/fine-tuning/SKILL.md +++ b/.claude/skills/fine-tuning/SKILL.md @@ -59,6 +59,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | | Native MLX restricted host consumer | `python -B examples/training_qualification/native_host.py inspect ` is read-only; exact-run `start` requires an externally signed root admission and separately supplied public-key pin. No live Mac qualification yet; see `reference/native-local-training.md` | | Capture reviewed native runtime bytes | `python -B scripts/capture_native_mlx_runtime.py --config --dependency-lock --output ` — observation only; explicit roots, dependencies and original pins required; see `reference/native-local-training.md` | +| Plan private native Mac runtime | `python -B examples/training_qualification/native_runtime_preparation.py plan ` — local 54-wheel archive plan only; later `acquire`/`install` consume separate private claims and need independent review; see `reference/native-local-training.md` | | Prepare a reviewed private native model snapshot | `python -B examples/training_qualification/native_model_preparation.py inspect ` is local-only; `metadata-candidate` performs one bounded SDK metadata request and `acquire` downloads only after separate exact inventory review. Neither grants training admission; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.claude/skills/fine-tuning/reference/native-local-training.md b/.claude/skills/fine-tuning/reference/native-local-training.md index 25278c15..e1de1f0f 100644 --- a/.claude/skills/fine-tuning/reference/native-local-training.md +++ b/.claude/skills/fine-tuning/reference/native-local-training.md @@ -221,11 +221,43 @@ ancestor is group-writable. The candidate venv lacks `jsonschema` required by the current host composition. Preserve both as blockers rather than changing shared Homebrew permissions, weakening native tree traversal, or claiming a capture. Existing wheel-cache metadata does not attest absent wheel archives. +The current child also imports `jsonschema` through +`tuner.training.contracts` → `tuner.project.execution_source` → the eager +`tuner.project` initializer. The host and child need the same exact inventory +under the current `verify_native_material` contract. The r7 candidate keeps +the original 50 lock unchanged and adds four reviewed Mac wheel hashes in +`requirements.host-additions.lock`. Its flat `requirements.runtime.lock` is +exactly the original bytes plus those four lines (SHA-256 +`7c167018ee6bd61a3609971e3f834082c798e7bd2a3fa0c5c2564887da4fde4f`). +The candidate proposal selects this 54-package lock; it grants no runtime or +hardware authority. + +Use `examples/training_qualification/native_runtime_preparation.py plan` with +the exact checked-in `native-private-runtime.example.json` to inspect the +166,269,790-byte archive selection without effects. Only after independent +review, copy that config to a new 0600 file in an existing 0700 private parent +and use its `acquire`, `install`, and `verify` phases from a clean pushed source. +The CLI retains the exact Astral 3.11.14+20260211 interpreter archive and 54 +wheel archives, checks hashes before an offline uv local-mirror install, binds +the uv executable hash/version, confines caches and temporary paths to the +private root, and retains consumed claims. HTTPS transfers stop before writing +an excess byte; finite expansion checks are pre-install inventory bounds, not +a hard aggregate filesystem quota for uv temporary files. A 2 GiB retained +private-tree guard runs after each phase. Include narrow +`source/shared` in the subsequent final-source static capture alongside +`source/tuner` and `source/synaptic_tuner`. Do not chmod Homebrew or reuse the +old venv/capture attempt. + The checked-in `examples/training_qualification/native_model_preparation.py` -is the later reviewed model-acquisition action: `inspect` is local, the bounded -`metadata-candidate` requests only upstream metadata, and `acquire` requires a -separately reviewed complete member/tokenizer selection. It verifies upstream -hashes and publishes private create-only snapshots. Its receipt is +is the later reviewed model-acquisition action: `inspect` is local, +`metadata-candidate` reads at most 2 MiB of the exact Hub metadata response +before JSON/SDK `ModelInfo` parsing, and `acquire` requires a separately +reviewed complete member/tokenizer selection. Model files stream from the +exact pinned Hub resolve URLs into precreated private 0700 nested parents; +per-file and aggregate caps reject before excess writes. The SDK's unbounded +snapshot/cache download path is not used. Unexpected redirect hosts fail +closed. It verifies upstream hashes and publishes private create-only +snapshots. Its receipt is `OBSERVED_NOT_ADMITTED`. No Mac model bytes or tokenizer labels have yet been acquired/reviewed. See `examples/training_qualification/native-host.md` for exact staging paths, publication digests and commands. diff --git a/.skills/fine-tuning/SKILL.md b/.skills/fine-tuning/SKILL.md index 5e7c1136..9c54c10b 100644 --- a/.skills/fine-tuning/SKILL.md +++ b/.skills/fine-tuning/SKILL.md @@ -59,6 +59,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | | Native MLX restricted host consumer | `python -B examples/training_qualification/native_host.py inspect ` is read-only; exact-run `start` requires an externally signed root admission and separately supplied public-key pin. No live Mac qualification yet; see `reference/native-local-training.md` | | Capture reviewed native runtime bytes | `python -B scripts/capture_native_mlx_runtime.py --config --dependency-lock --output ` — observation only; explicit roots, dependencies and original pins required; see `reference/native-local-training.md` | +| Plan private native Mac runtime | `python -B examples/training_qualification/native_runtime_preparation.py plan ` — local 54-wheel archive plan only; later `acquire`/`install` consume separate private claims and need independent review; see `reference/native-local-training.md` | | Prepare a reviewed private native model snapshot | `python -B examples/training_qualification/native_model_preparation.py inspect ` is local-only; `metadata-candidate` performs one bounded SDK metadata request and `acquire` downloads only after separate exact inventory review. Neither grants training admission; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.skills/fine-tuning/reference/native-local-training.md b/.skills/fine-tuning/reference/native-local-training.md index 25278c15..e1de1f0f 100644 --- a/.skills/fine-tuning/reference/native-local-training.md +++ b/.skills/fine-tuning/reference/native-local-training.md @@ -221,11 +221,43 @@ ancestor is group-writable. The candidate venv lacks `jsonschema` required by the current host composition. Preserve both as blockers rather than changing shared Homebrew permissions, weakening native tree traversal, or claiming a capture. Existing wheel-cache metadata does not attest absent wheel archives. +The current child also imports `jsonschema` through +`tuner.training.contracts` → `tuner.project.execution_source` → the eager +`tuner.project` initializer. The host and child need the same exact inventory +under the current `verify_native_material` contract. The r7 candidate keeps +the original 50 lock unchanged and adds four reviewed Mac wheel hashes in +`requirements.host-additions.lock`. Its flat `requirements.runtime.lock` is +exactly the original bytes plus those four lines (SHA-256 +`7c167018ee6bd61a3609971e3f834082c798e7bd2a3fa0c5c2564887da4fde4f`). +The candidate proposal selects this 54-package lock; it grants no runtime or +hardware authority. + +Use `examples/training_qualification/native_runtime_preparation.py plan` with +the exact checked-in `native-private-runtime.example.json` to inspect the +166,269,790-byte archive selection without effects. Only after independent +review, copy that config to a new 0600 file in an existing 0700 private parent +and use its `acquire`, `install`, and `verify` phases from a clean pushed source. +The CLI retains the exact Astral 3.11.14+20260211 interpreter archive and 54 +wheel archives, checks hashes before an offline uv local-mirror install, binds +the uv executable hash/version, confines caches and temporary paths to the +private root, and retains consumed claims. HTTPS transfers stop before writing +an excess byte; finite expansion checks are pre-install inventory bounds, not +a hard aggregate filesystem quota for uv temporary files. A 2 GiB retained +private-tree guard runs after each phase. Include narrow +`source/shared` in the subsequent final-source static capture alongside +`source/tuner` and `source/synaptic_tuner`. Do not chmod Homebrew or reuse the +old venv/capture attempt. + The checked-in `examples/training_qualification/native_model_preparation.py` -is the later reviewed model-acquisition action: `inspect` is local, the bounded -`metadata-candidate` requests only upstream metadata, and `acquire` requires a -separately reviewed complete member/tokenizer selection. It verifies upstream -hashes and publishes private create-only snapshots. Its receipt is +is the later reviewed model-acquisition action: `inspect` is local, +`metadata-candidate` reads at most 2 MiB of the exact Hub metadata response +before JSON/SDK `ModelInfo` parsing, and `acquire` requires a separately +reviewed complete member/tokenizer selection. Model files stream from the +exact pinned Hub resolve URLs into precreated private 0700 nested parents; +per-file and aggregate caps reject before excess writes. The SDK's unbounded +snapshot/cache download path is not used. Unexpected redirect hosts fail +closed. It verifies upstream hashes and publishes private create-only +snapshots. Its receipt is `OBSERVED_NOT_ADMITTED`. No Mac model bytes or tokenizer labels have yet been acquired/reviewed. See `examples/training_qualification/native-host.md` for exact staging paths, publication digests and commands. diff --git a/AGENTS.md b/AGENTS.md index ef099261..39441996 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -51,6 +51,14 @@ This repository has a few cloud-training constraints that are easy to relearn th durable claims, child lifecycle and native artifact verification host-owned. Do not enable native training in Syntunia before those joins and real Mac qualification are complete. See `docs/plans/native-desktop-training-api-v1.md`. +- For Mac qualification preparation, preserve the original 50-package MLX lock + and use the reviewed four-wheel additive/flat 54-package lock for the current + host and child import graph. Prepare a new owner-private interpreter/runtime + through the checked-in bounded `native_runtime_preparation.py`; never chmod + Homebrew or treat cache metadata as selected wheel archive provenance. Capture + narrow `shared`, `tuner`, and `synaptic_tuner` sources on the final pushed + commit. Model metadata and weights remain separate reviewed effects through + `native_model_preparation.py`; its candidate and receipts grant no admission. - Capture native runtime files and linked libraries, including the Homebrew Python framework, startup files and bytecode. A stub executable hash or package metadata match cannot attest this closure. Treat the OS shared cache as an diff --git a/Trainers/mlx_sft_mac/runtime-candidate/README.md b/Trainers/mlx_sft_mac/runtime-candidate/README.md index cea4d936..d4fd87cd 100644 --- a/Trainers/mlx_sft_mac/runtime-candidate/README.md +++ b/Trainers/mlx_sft_mac/runtime-candidate/README.md @@ -86,6 +86,62 @@ Frozen inventory SHA-256: This is dependency-installation evidence only; no ML framework was imported and no Metal probe, model load or training step was executed. +## Reviewed private runtime preparation candidate — r7 + +The original 50-package lock above remains unchanged (SHA-256 +`f9a6298260cd17f2c3d567ae3674108a71efa68e99369906487a4d8c0feb7117`). +The current host **and** isolated child import `jsonschema` through the eager +`tuner.project` package initializer. A separate host environment is incompatible +with the current exact-runtime verifier, which recaptures the calling executable, +stdlib and complete distribution inventory. The additive +[`requirements.host-additions.lock`](requirements.host-additions.lock) pins the +four Mac wheels for `jsonschema`, `jsonschema-specifications`, `referencing` and +`rpds-py`. [`requirements.runtime.lock`](requirements.runtime.lock) is exactly +the original lock bytes followed by those four lines, SHA-256 +`7c167018ee6bd61a3609971e3f834082c798e7bd2a3fa0c5c2564887da4fde4f`. +It is a new proposed 54-package capture input; it does not rewrite the original +installation evidence or admit training. + +The checked-in +[`native-private-runtime.example.json`](native-private-runtime.example.json) +declares the exact Astral CPython 3.11.14+20260211 arm64 archive (20,051,753 +bytes, SHA-256 `d5f10f6fdc6f4afb5d251578edb89d8c0ece8f9f56e38b0019e2a94e8f0f4852`), +uv 0.12.23 executable hash, every selected wheel filename/URL/size/hash and an +owner-private **unused** output root. The 54 wheel archives total 146,218,037 +bytes; all retained download archives total 166,269,790 bytes. These are +official upstream metadata selections awaiting actual archive readback. + +From the final clean pushed source, `native_runtime_preparation.py plan` is +read-only. Copy an independently reviewed exact config to a separate fresh +0600 file in an existing 0700 private parent before the effectful phases. The +`acquire` phase creates its selected root once, streams only the declared HTTPS +archives with per-archive and aggregate prewrite byte limits, retains the +archive bytes and exact original/additive/combined lock copies, and creates an +observation receipt. It does not resolve or fetch +any additional package. `install` consumes a separate claim, rechecks every +archive and bounded expansion inventory, copies the exact hashed uv binary to +the private root, uses a local `file://` Python mirror with `--no-bin`, and +syncs only the retained wheelhouse with `--offline --no-index --require-hashes +--only-binary :all: --link-mode copy`. `verify` checks the 54 installed versions +without ML imports. Each owned process has a 300-second deadline and 4 KiB +combined output cap. Its environment points temporary/cache paths into the +private root and disables uv's reusable cache. Archive transfer bounds are hard; archive expansion inventory is +bounded before installation and retained private-tree observations reject over +2 GiB after each phase. That postphase guard does not claim a hard aggregate +quota on uv's transient filesystem use. A failed acquisition or install keeps +its consumed private state; choose a new reviewed root for another attempt. + +The future static capture must use this new private interpreter and the +`requirements.runtime.lock` bytes. Its narrow source list must include +`source/shared` as well as `source/tuner` and `source/synaptic_tuner`; the r5 +capture omitted `shared`. Select executable/framework/stdlib/package and +native library paths from observed private bytes after installation, then +capture against the final reviewed pushed commit. No selected-wheel, install, +runtime, model, tokenizer or hardware attestation follows from this template. +Model preparation uses the pinned Hub SDK's `ModelInfo` parser and URL builder +only after the metadata body is capped or before the independently capped +download transport; it does not invoke the SDK's unbounded snapshot downloader. + ## Prepared worker boundary The pinned diff --git a/Trainers/mlx_sft_mac/runtime-candidate/native-private-runtime.example.json b/Trainers/mlx_sft_mac/runtime-candidate/native-private-runtime.example.json new file mode 100644 index 00000000..43996057 --- /dev/null +++ b/Trainers/mlx_sft_mac/runtime-candidate/native-private-runtime.example.json @@ -0,0 +1,458 @@ +{ + "schema": "synaptic-native-private-runtime-preparation/v1", + "status": "REVIEW_REQUIRED_NO_EXECUTION_AUTHORITY", + "private_root": "/Users/jrosenbaum/.local/share/synaptic-qualification/native-mlx-private-runtime-r6-20261020", + "uv_executable": "/opt/homebrew/bin/uv", + "uv_sha256": "c87143aea2480b09f107a9f97fa93899d116f0307b4d210649a735e141bdd889", + "uv_version": "0.12.23", + "python_archive": { + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20260211/cpython-3.11.14%2B20260211-aarch64-apple-darwin-install_only_stripped.tar.gz", + "filename": "cpython-3.11.14+20260211-aarch64-apple-darwin-install_only_stripped.tar.gz", + "size": 20051753, + "sha256": "d5f10f6fdc6f4afb5d251578edb89d8c0ece8f9f56e38b0019e2a94e8f0f4852" + }, + "python_install_key": "cpython-3.11.14-macos-aarch64-none", + "original_lock_sha256": "f9a6298260cd17f2c3d567ae3674108a71efa68e99369906487a4d8c0feb7117", + "additions_lock_sha256": "bd1f5f3d787b28024af16e6f7344f468a1b4aa48a69c0820f21a0a8f340e28fd", + "runtime_lock_sha256": "7c167018ee6bd61a3609971e3f834082c798e7bd2a3fa0c5c2564887da4fde4f", + "wheel_archive_bytes": 146218037, + "maximum_worker_seconds": 300, + "maximum_command_output_bytes": 4096, + "maximum_python_unpacked_bytes": 536870912, + "maximum_wheel_unpacked_bytes": 1073741824, + "wheels": [ + { + "package": "aiohappyeyeballs", + "version": "2.7.1", + "filename": "aiohappyeyeballs-2.7.1-py3-none-any.whl", + "sha256": "9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472", + "size": 15038, + "url": "https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl" + }, + { + "package": "aiohttp", + "version": "3.14.5", + "filename": "aiohttp-3.14.5-cp311-cp311-macosx_11_0_arm64.whl", + "sha256": "a2c473a355f9239efcb72c92d5abfd8fcdb0cc78c8e9af607e72ca12dbb36593", + "size": 552588, + "url": "https://files.pythonhosted.org/packages/3d/fe/92838944e601f0fe195fd3f3ada37e29fbee3d19bfd323fe15937ec79d36/aiohttp-3.14.5-cp311-cp311-macosx_11_0_arm64.whl" + }, + { + "package": "aiosignal", + "version": "1.4.0", + "filename": "aiosignal-1.4.0-py3-none-any.whl", + "sha256": "053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", + "size": 7490, + "url": "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl" + }, + { + "package": "annotated-doc", + "version": "0.0.5", + "filename": "annotated_doc-0.0.5-py3-none-any.whl", + "sha256": "117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", + "size": 5302, + "url": "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl" + }, + { + "package": "anyio", + "version": "4.15.1", + "filename": "anyio-4.15.1-py3-none-any.whl", + "sha256": "6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101", + "size": 132079, + "url": "https://files.pythonhosted.org/packages/12/b8/4bd346e22b28902df4d651910f5242c28d84e4a5c2435ca5c3f797ed7e2e/anyio-4.15.1-py3-none-any.whl" + }, + { + "package": "attrs", + "version": "26.1.0", + "filename": "attrs-26.1.0-py3-none-any.whl", + "sha256": "c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", + "size": 67548, + "url": "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl" + }, + { + "package": "click", + "version": "8.5.0", + "filename": "click-8.5.0-py3-none-any.whl", + "sha256": "255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360", + "size": 125251, + "url": "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl" + }, + { + "package": "datasets", + "version": "5.1.0", + "filename": "datasets-5.1.0-py3-none-any.whl", + "sha256": "86fd08b280485aab82894d2c81842c4151d93b575efea0054315104cb7bc3cec", + "size": 604432, + "url": "https://files.pythonhosted.org/packages/24/cc/7f624629846c709ef1c68aa61bfb8d0ae2638cbe265104a1c2f7a630949e/datasets-5.1.0-py3-none-any.whl" + }, + { + "package": "dill", + "version": "0.4.1", + "filename": "dill-0.4.1-py3-none-any.whl", + "sha256": "1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", + "size": 120019, + "url": "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl" + }, + { + "package": "filelock", + "version": "4.2.4", + "filename": "filelock-4.2.4-py3-none-any.whl", + "sha256": "301241438c57936ef93f4243a94f2559022268bfcf9b375a9a72eb0f8db8358e", + "size": 162475, + "url": "https://files.pythonhosted.org/packages/e0/1b/5e74289a1878e666bb4b65fe5805a04cafcac2aa5f519ef1ead67ef91c56/filelock-4.2.4-py3-none-any.whl" + }, + { + "package": "frozenlist", + "version": "1.8.0", + "filename": "frozenlist-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", + "sha256": "fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93", + "size": 50119, + "url": "https://files.pythonhosted.org/packages/6e/ef/0e8f1fe32f8a53dd26bdd1f9347efe0778b0fddf62789ea683f4cc7d787d/frozenlist-1.8.0-cp311-cp311-macosx_11_0_arm64.whl" + }, + { + "package": "fsspec", + "version": "2026.7.0", + "filename": "fsspec-2026.7.0-py3-none-any.whl", + "sha256": "b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279", + "size": 206583, + "url": "https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl" + }, + { + "package": "h11", + "version": "0.16.0", + "filename": "h11-0.16.0-py3-none-any.whl", + "sha256": "63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", + "size": 37515, + "url": "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl" + }, + { + "package": "hf-xet", + "version": "1.7.0", + "filename": "hf_xet-1.7.0-cp38-abi3-macosx_11_0_arm64.whl", + "sha256": "59fba37039233c7fcbe196817d6cdcf1b40dfb17b410f229d85b0cf0a1848da4", + "size": 3560352, + "url": "https://files.pythonhosted.org/packages/c3/47/a26ebdce7056a61e931f228439bc0ab08cbec239d1690f965e5e637cba79/hf_xet-1.7.0-cp38-abi3-macosx_11_0_arm64.whl" + }, + { + "package": "httpcore2", + "version": "2.15.1", + "filename": "httpcore2-2.15.1-py3-none-any.whl", + "sha256": "730f603058c260a3288378217b464713350c23e3b8fe8bba753701054032d98a", + "size": 85055, + "url": "https://files.pythonhosted.org/packages/0d/9b/e2b556e26fbc67716ab1dfc18832eb6fe196d88363f7094cb1da3d7c0d3e/httpcore2-2.15.1-py3-none-any.whl" + }, + { + "package": "httpx2", + "version": "2.15.1", + "filename": "httpx2-2.15.1-py3-none-any.whl", + "sha256": "75841cd83d16b417f7d6da8f2089ce8120806efdbf96af37d46c5129c49015be", + "size": 96411, + "url": "https://files.pythonhosted.org/packages/7a/bf/56068f774620330c8330e4b4915fae889a5b2d4b52667b3b7c8f66c4a9ad/httpx2-2.15.1-py3-none-any.whl" + }, + { + "package": "huggingface-hub", + "version": "2.2.0", + "filename": "huggingface_hub-2.2.0-py3-none-any.whl", + "sha256": "1667f145dc56dc210d60966069397df9ecfca9607a5d43db88b308c89dae56b3", + "size": 839884, + "url": "https://files.pythonhosted.org/packages/eb/b0/0f7b430fd100b3a3b037fdbb314878200241082e607b3383c63d91a13a72/huggingface_hub-2.2.0-py3-none-any.whl" + }, + { + "package": "idna", + "version": "3.20", + "filename": "idna-3.20-py3-none-any.whl", + "sha256": "ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", + "size": 69583, + "url": "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl" + }, + { + "package": "jinja2", + "version": "3.1.6", + "filename": "jinja2-3.1.6-py3-none-any.whl", + "sha256": "85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", + "size": 134899, + "url": "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl" + }, + { + "package": "jsonschema", + "version": "4.26.0", + "filename": "jsonschema-4.26.0-py3-none-any.whl", + "sha256": "d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", + "size": 90630, + "url": "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl" + }, + { + "package": "jsonschema-specifications", + "version": "2025.9.1", + "filename": "jsonschema_specifications-2025.9.1-py3-none-any.whl", + "sha256": "98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", + "size": 18437, + "url": "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl" + }, + { + "package": "markdown-it-py", + "version": "4.2.0", + "filename": "markdown_it_py-4.2.0-py3-none-any.whl", + "sha256": "9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", + "size": 91687, + "url": "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl" + }, + { + "package": "markupsafe", + "version": "3.0.4", + "filename": "markupsafe-3.0.4-cp311-cp311-macosx_11_0_arm64.whl", + "sha256": "7d3391b2188d18737cb2fa147028b1096236eaa7e156446c650a489fa2cadc91", + "size": 12063, + "url": "https://files.pythonhosted.org/packages/6c/14/0b05f79b4733e264a18d08fe08fa1df7347630ff32a6cb82180d9dccec55/markupsafe-3.0.4-cp311-cp311-macosx_11_0_arm64.whl" + }, + { + "package": "mdurl", + "version": "0.1.2", + "filename": "mdurl-0.1.2-py3-none-any.whl", + "sha256": "84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", + "size": 9979, + "url": "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl" + }, + { + "package": "mlx", + "version": "0.32.2", + "filename": "mlx-0.32.2-cp311-cp311-macosx_26_0_arm64.whl", + "sha256": "e0828c96a6b988afbe6f07cc39adc0470eba9121f7e356623a8fb9d9a6397533", + "size": 625981, + "url": "https://files.pythonhosted.org/packages/16/f9/15c5b7313a73c33014e8082992a868b95b742c1966a57b684b7a3cffe597/mlx-0.32.2-cp311-cp311-macosx_26_0_arm64.whl" + }, + { + "package": "mlx-lm", + "version": "0.32.2", + "filename": "mlx_lm-0.32.2-py3-none-any.whl", + "sha256": "5c0991248cfff5872503d74f466d2532dd8cb7fb3670cb9be7a06d7305dadc04", + "size": 546040, + "url": "https://files.pythonhosted.org/packages/d1/4f/000eefdeb41388782b47e7f39904cf2d9e7e49a60df5e172097c358dba8e/mlx_lm-0.32.2-py3-none-any.whl" + }, + { + "package": "mlx-metal", + "version": "0.32.2", + "filename": "mlx_metal-0.32.2-py3-none-macosx_26_0_arm64.whl", + "sha256": "e6abeac9ac5265830c9c1541b6f96e9be37a85c2446763a46ad466c63a3837ab", + "size": 64367860, + "url": "https://files.pythonhosted.org/packages/dd/cd/4e50bf325100e7165e13d025f264362bf0009196269f9eaf87f2c6e738a2/mlx_metal-0.32.2-py3-none-macosx_26_0_arm64.whl" + }, + { + "package": "multidict", + "version": "7.1.0", + "filename": "multidict-7.1.0-cp311-cp311-macosx_11_0_arm64.whl", + "sha256": "85cb3ced4fa84949cee12bfe78208b6ece7baf3cbd242b26dcaf773efff8d206", + "size": 98743, + "url": "https://files.pythonhosted.org/packages/30/5e/7d18eb5a75cf4bacb6b3ee689929dcb27d7ed8ca2d245e80e9b8056fcbf5/multidict-7.1.0-cp311-cp311-macosx_11_0_arm64.whl" + }, + { + "package": "multiprocess", + "version": "0.70.19", + "filename": "multiprocess-0.70.19-py311-none-any.whl", + "sha256": "928851ae7973aea4ce0eaf330bbdafb2e01398a91518d5c8818802845564f45c", + "size": 144457, + "url": "https://files.pythonhosted.org/packages/86/c2/dec9722dc3474c164a0b6bcd9a7ed7da542c98af8cabce05374abab35edd/multiprocess-0.70.19-py311-none-any.whl" + }, + { + "package": "numpy", + "version": "2.4.6", + "filename": "numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", + "sha256": "4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8", + "size": 5469159, + "url": "https://files.pythonhosted.org/packages/ad/40/40a40ee0ddf7ceb782c49af278894b686e586d65d8c1889c8b5da01a3d7d/numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl" + }, + { + "package": "packaging", + "version": "26.3", + "filename": "packaging-26.3-py3-none-any.whl", + "sha256": "d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", + "size": 129956, + "url": "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl" + }, + { + "package": "pandas", + "version": "3.0.6", + "filename": "pandas-3.0.6-cp311-cp311-macosx_11_0_arm64.whl", + "sha256": "d7564d86a94c2eb8ab290b07f63ddaae5c032fa53897c29a2ff2197d43aee8af", + "size": 10023718, + "url": "https://files.pythonhosted.org/packages/55/a6/39db5d41f3eb5d7846626312cb30e0cea48e988fc47a3725698bc931ad3c/pandas-3.0.6-cp311-cp311-macosx_11_0_arm64.whl" + }, + { + "package": "propcache", + "version": "0.5.4", + "filename": "propcache-0.5.4-cp311-cp311-macosx_11_0_arm64.whl", + "sha256": "3fc24f209c1b7f7f688b66b98293954f5504279760999b58920ee12dd8471c1d", + "size": 52116, + "url": "https://files.pythonhosted.org/packages/05/cf/b5248180bf056cc76acc60c9c6e8c0ebbfdbd1c6cffd31fd14996927b7c8/propcache-0.5.4-cp311-cp311-macosx_11_0_arm64.whl" + }, + { + "package": "protobuf", + "version": "7.36.2", + "filename": "protobuf-7.36.2-cp310-abi3-macosx_10_9_universal2.whl", + "sha256": "cbc70b17ee27e28894c7fee8bb04be1abead49e936bc70eb60052531eee2079e", + "size": 456039, + "url": "https://files.pythonhosted.org/packages/32/72/98342feb672507c8f3a69e34b4fa8961f608edba5c1a48a6f47156d92cb5/protobuf-7.36.2-cp310-abi3-macosx_10_9_universal2.whl" + }, + { + "package": "pyarrow", + "version": "26.0.0", + "filename": "pyarrow-26.0.0-cp311-cp311-macosx_12_0_arm64.whl", + "sha256": "fcdd1e04982637c6042337d3e24d472f938f01fdc502e2b994844b726d12c3f4", + "size": 36370896, + "url": "https://files.pythonhosted.org/packages/07/68/e0707097cee93be7f693e7e89495fabfeb8bf95ee30619063f8b30fffc29/pyarrow-26.0.0-cp311-cp311-macosx_12_0_arm64.whl" + }, + { + "package": "pygments", + "version": "2.21.0", + "filename": "pygments-2.21.0-py3-none-any.whl", + "sha256": "2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", + "size": 1250147, + "url": "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl" + }, + { + "package": "python-dateutil", + "version": "2.9.0.post0", + "filename": "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", + "sha256": "a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", + "size": 229892, + "url": "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl" + }, + { + "package": "pyyaml", + "version": "6.0.3", + "filename": "pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", + "sha256": "652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", + "size": 175577, + "url": "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl" + }, + { + "package": "referencing", + "version": "0.37.0", + "filename": "referencing-0.37.0-py3-none-any.whl", + "sha256": "381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", + "size": 26766, + "url": "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl" + }, + { + "package": "regex", + "version": "2026.9.29", + "filename": "regex-2026.9.29-cp311-cp311-macosx_11_0_arm64.whl", + "sha256": "066d0e3dbfdd739bce2bf8c2a41dd16f73e3d8adc2eb06dd803a36a307f56075", + "size": 292123, + "url": "https://files.pythonhosted.org/packages/b1/07/9bf3607d8d13a12e436ab9d63f9791e10706827d535695b23964ad79fd79/regex-2026.9.29-cp311-cp311-macosx_11_0_arm64.whl" + }, + { + "package": "rich", + "version": "15.0.0", + "filename": "rich-15.0.0-py3-none-any.whl", + "sha256": "33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", + "size": 310654, + "url": "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl" + }, + { + "package": "rpds-py", + "version": "2026.6.3", + "filename": "rpds_py-2026.6.3-cp311-cp311-macosx_11_0_arm64.whl", + "sha256": "db08f45aecde626498fb3df07bcf6d2ec040af42e859a4f5040d79c200342911", + "size": 345513, + "url": "https://files.pythonhosted.org/packages/4d/dc/323d08583c0832911768663d1944f0107fcd4088704858d84b5e06d105a0/rpds_py-2026.6.3-cp311-cp311-macosx_11_0_arm64.whl" + }, + { + "package": "safetensors", + "version": "0.8.0", + "filename": "safetensors-0.8.0-cp310-abi3-macosx_11_0_arm64.whl", + "sha256": "c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25", + "size": 484562, + "url": "https://files.pythonhosted.org/packages/f5/b1/fa7c600e7dceae12e9606c7578cbc9ff1e1ed55844883ee5c92205e86226/safetensors-0.8.0-cp310-abi3-macosx_11_0_arm64.whl" + }, + { + "package": "sentencepiece", + "version": "0.2.2", + "filename": "sentencepiece-0.2.2-cp311-cp311-macosx_11_0_arm64.whl", + "sha256": "443ac618c7a2a1377cf5c82581fbb849591d14e656d5e5a3e4682d4e36a34e4e", + "size": 1346997, + "url": "https://files.pythonhosted.org/packages/2a/92/3a6ea4a2c6dd9e7062698a5a33534ca0e20844883338ae9c6b9c122c1a9f/sentencepiece-0.2.2-cp311-cp311-macosx_11_0_arm64.whl" + }, + { + "package": "shellingham", + "version": "1.5.4", + "filename": "shellingham-1.5.4-py2.py3-none-any.whl", + "sha256": "7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", + "size": 9755, + "url": "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl" + }, + { + "package": "six", + "version": "1.17.0", + "filename": "six-1.17.0-py2.py3-none-any.whl", + "sha256": "4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", + "size": 11050, + "url": "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl" + }, + { + "package": "tokenizers", + "version": "0.23.3", + "filename": "tokenizers-0.23.3-cp310-abi3-macosx_11_0_arm64.whl", + "sha256": "68649e97d5b43c44c031d8d848874a6eecae8f8fe40ea989aa777a5a83aca716", + "size": 3101381, + "url": "https://files.pythonhosted.org/packages/b2/72/01e49f032bb346e5aaf06c10c74fe8aeec847173adbadd66eb7c53054bf2/tokenizers-0.23.3-cp310-abi3-macosx_11_0_arm64.whl" + }, + { + "package": "tqdm", + "version": "4.70.1", + "filename": "tqdm-4.70.1-py3-none-any.whl", + "sha256": "c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73", + "size": 80199, + "url": "https://files.pythonhosted.org/packages/a7/03/921a3d3c75785aca9ebfbfcabfbc3a1be12e2ab5265deb026d55a5a3f83e/tqdm-4.70.1-py3-none-any.whl" + }, + { + "package": "transformers", + "version": "5.20.2", + "filename": "transformers-5.20.2-py3-none-any.whl", + "sha256": "7ae75dbc1fb3922aff1b6c97b9e5479d6cf26fd86a8866bfb71324190479f807", + "size": 12844921, + "url": "https://files.pythonhosted.org/packages/96/b6/724faefed06913dd3e42e0476fe2c42448675c42f687a12bf810aa098005/transformers-5.20.2-py3-none-any.whl" + }, + { + "package": "truststore", + "version": "0.10.5", + "filename": "truststore-0.10.5-py3-none-any.whl", + "sha256": "9aaaedaefaf06d8b206278cf8b5012bc897f485a874503501e12d776df78951c", + "size": 19017, + "url": "https://files.pythonhosted.org/packages/51/e9/3a7820be2bb0fe53b6bc9c3be26d3d1158004e4c3ab953aa6840b955b1e9/truststore-0.10.5-py3-none-any.whl" + }, + { + "package": "typer", + "version": "0.27.3", + "filename": "typer-0.27.3-py3-none-any.whl", + "sha256": "e50022f28b82a86313e54501317a1db64bf8f8d036ff8cfe5ca7e47675454aff", + "size": 123312, + "url": "https://files.pythonhosted.org/packages/07/ea/2e31b67051e91a133189e9c000c222502ddc6969856416de0d095de4c0b0/typer-0.27.3-py3-none-any.whl" + }, + { + "package": "typing-extensions", + "version": "4.16.0", + "filename": "typing_extensions-4.16.0-py3-none-any.whl", + "sha256": "481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", + "size": 45571, + "url": "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl" + }, + { + "package": "xxhash", + "version": "4.0.2", + "filename": "xxhash-4.0.2-cp311-cp311-macosx_11_0_arm64.whl", + "sha256": "f2b2839a83b9a9d03ddf89f440bb43627832e77b9a4276988d8aa0bb49aed0df", + "size": 36206, + "url": "https://files.pythonhosted.org/packages/3f/28/d581b2308cb9c731644a86f25b33cba6e9e540d1d05e3c51fcc274f2df69/xxhash-4.0.2-cp311-cp311-macosx_11_0_arm64.whl" + }, + { + "package": "yarl", + "version": "1.25.1", + "filename": "yarl-1.25.1-cp311-cp311-macosx_11_0_arm64.whl", + "sha256": "564fdc7085d2245ab84f88882fdb1d6ac0723124bff6ded35bfb1c00f812630d", + "size": 104010, + "url": "https://files.pythonhosted.org/packages/ce/7c/fbb40fe2d53747c40aa36a9e2bd2178a202f942bda0b670f3306d4aefbce/yarl-1.25.1-cp311-cp311-macosx_11_0_arm64.whl" + } + ], + "maximum_private_tree_bytes": 2147483648 +} diff --git a/Trainers/mlx_sft_mac/runtime-candidate/requirements.host-additions.lock b/Trainers/mlx_sft_mac/runtime-candidate/requirements.host-additions.lock new file mode 100644 index 00000000..b0874dcc --- /dev/null +++ b/Trainers/mlx_sft_mac/runtime-candidate/requirements.host-additions.lock @@ -0,0 +1,4 @@ +jsonschema==4.26.0 --hash=sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce +jsonschema-specifications==2025.9.1 --hash=sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe +referencing==0.37.0 --hash=sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231 +rpds-py==2026.6.3 --hash=sha256:db08f45aecde626498fb3df07bcf6d2ec040af42e859a4f5040d79c200342911 diff --git a/Trainers/mlx_sft_mac/runtime-candidate/requirements.runtime.lock b/Trainers/mlx_sft_mac/runtime-candidate/requirements.runtime.lock new file mode 100644 index 00000000..0c60a69d --- /dev/null +++ b/Trainers/mlx_sft_mac/runtime-candidate/requirements.runtime.lock @@ -0,0 +1,1894 @@ +# This file was autogenerated by uv via the following command: +# uv --no-config pip compile Trainers/mlx_sft_mac/runtime-candidate/requirements.in --python-version 3.11.14 --python-platform aarch64-apple-darwin --only-binary :all: --generate-hashes --output-file Trainers/mlx_sft_mac/runtime-candidate/requirements.lock +aiohappyeyeballs==2.7.1 \ + --hash=sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d \ + --hash=sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472 + # via aiohttp +aiohttp==3.14.5 \ + --hash=sha256:0133c3c3b54a0bf1e71fa5c1ad95c93f07fd54e24ef1fe182f5122e1573d2bf1 \ + --hash=sha256:038c2c7e8caa26b6c8423779b5eaf1893904048a512c19b32fe841ffa5592b50 \ + --hash=sha256:0684952aeae1f5dbfe02d46039338513b94009baecd15d8e4098a357c4c4a2a6 \ + --hash=sha256:0790ec66fa4013e83c53b9025a45d454723da1a2fce28b3208c9b32d08af162f \ + --hash=sha256:083673c7a94c3ea035caaa5ca04288bdb44887abfe1f5ba23294e6a4b03efd2d \ + --hash=sha256:09e0eb18c7e0c8777e2f9149de63799195b9b3ca1b5c81ba6f32f2c6b8628210 \ + --hash=sha256:09ec102b4b8c9a920275733bbc11fdbb615efe6f9231a06007c0218d336fb77a \ + --hash=sha256:0a8ea271867e360ac985ae607f4a23ad9a38414b9aca1d49ec98839ae660e49f \ + --hash=sha256:0e6f16f5e49c4b8267988c05ab07760d7064cea57d077c3d068d04b0fbb992cb \ + --hash=sha256:1061b364556e8172e8d46b0b183adeeb73e8c42d30ebc745591e1bd89acad52e \ + --hash=sha256:1220353657ad49493551f089ce02f1a348fd57ffd585bfec77f2f3c4fe3a7346 \ + --hash=sha256:137351bf20bbed9a65e839f4a4452ac377389bdb2f2857d2acffef38f5e9f2d1 \ + --hash=sha256:149fb56caf7acb67073126f675d0958d9c4b3125fcd3f6d4877df98aa8a97ce9 \ + --hash=sha256:14f04769cfefe4734016a856a83af36133cd17779cef9ae817f812b8ba9d6d51 \ + --hash=sha256:15a310d3c71398e3d7bfc93a1a73fbe664315cd9e9016b8efc1cff85eeab7155 \ + --hash=sha256:1612fa5857b37bf32e5c1eaeefb96e3b01e9c70679eec81f0934e8a600080863 \ + --hash=sha256:16c8abd5bca220a47efe667d26f8460124c81810787e79ee87b242677563d9dd \ + --hash=sha256:19e2ba471507c34f8252402ab50f5ab512398b9ea8c8f1cb26beb3f75793ba30 \ + --hash=sha256:1aead151c3abbac6b32942e452020cb66d7efc099d253cc6c20f748e926c858b \ + --hash=sha256:1b438b73c38111818d0c9d6a5c2bfed8584c8e503a49ef085d70e874ec846738 \ + --hash=sha256:1b5416552740edf07234cc9437d0706f2acb67b93c198670b1a68e1b2b587dec \ + --hash=sha256:1d2d981b53dd09a319e3570ef8cc3bbc3ef86f5a7abef0f6b2bff3867db3a9e7 \ + --hash=sha256:1eb8167961ec4dfcc8cb9dd50bd0ee72519f7ef496be95203e49e27b01618382 \ + --hash=sha256:1ffa3a523a36d8628f98c06492ae16a31a23d14c0b4ec721757b477319f656d6 \ + --hash=sha256:20064a177a070d789ee64a50b01a9161d3468e989baacfc6c714aa685c4b332f \ + --hash=sha256:20726f9782d5c2744c1c66255842d1d163bb3edcf768b8de25216bf47f7b6ccf \ + --hash=sha256:20f085697d7e911f1f73c43ed03fafbed1e7121797e2eb5428efa80398060584 \ + --hash=sha256:225c579c23b68b343cccea27a7e06e3bd8ec23a09c30b427eb3f1e4ca6239b20 \ + --hash=sha256:24409db442e2fb6e766bc7f3943851a8381dec3098140e43bb2e843b79e31b12 \ + --hash=sha256:248d779ad720b49d4fb355720e60c9e5f444f95887bc16974fea48fc56c41789 \ + --hash=sha256:2528cb4c6b92008c76ac9ac6298624069bb2db91ff4929905512d1d84485f658 \ + --hash=sha256:276a4fc00b1d9ae492b802763a789c5b86328b989c5ea169f2faa447d6a11c7c \ + --hash=sha256:27c2322e03f66101acb09869ce1cf1efc04994ee95e1735b69827bf8c8b9d781 \ + --hash=sha256:293d3ae7c6a0ed176a42e59a1b5fde825ead65c835360f734148e96729f928d2 \ + --hash=sha256:2cc38a4f2b516bef1714e690df87a0e043faf1a7693c82d860091684453d5111 \ + --hash=sha256:3093b72c215bda16ce961a6d073f6e71d46e022962a9d5d457c5d4d421c78b57 \ + --hash=sha256:32e8fa6644e541fcd7e02430588c7fc93b602c1778ea0bc345505db76b61cfb4 \ + --hash=sha256:330900acd0dc4cb8b27f9c127fbaad770964845338493e7906ae3822e82dbf8d \ + --hash=sha256:33f706574e32c6e694f352a856e05caf18f7f2c871b3e87b41c55ea452b409ab \ + --hash=sha256:3ae800a20947e2c2e53088047d021e6bf7d51560cc49f6a0737a1f79d2e3a13c \ + --hash=sha256:3be7dd397d64ca3e1869626fa9318aaebb54b7bf93bc72d7a205448d83e4f748 \ + --hash=sha256:3e0eb43bed3c6801a6cee315195377789e90b2a72c2277a475b578535312488d \ + --hash=sha256:3e51a27980c3788e6e6b3325d694fdd4898087fa8a86b2763af77b39353da41e \ + --hash=sha256:3edbece0379b8b4aaa67619b8aa2399bb66fce372cd5911098a434ea77220aa0 \ + --hash=sha256:3f2dcc00191fd563e9075181a14ec31d7dd63223ced7582cc70a15a499de0c79 \ + --hash=sha256:42b5e616946dbaf505e2bff18c9af2cd4ef9e7ef300ee58a6e951a5b7cf147ae \ + --hash=sha256:42f320d4a5b00b9af0bddcfec5407dc6f2d9816f006b2f79ebbaa31f16895df3 \ + --hash=sha256:43351bdb5e4c3cb7d1772368e988534e869a74db7778079a83782c11c69535c7 \ + --hash=sha256:43e1b7994a8b038125f722bff07492ef501110722c2727c408995d9fb864c421 \ + --hash=sha256:4887d130a7bbfed3a85493bb5a25e5b5b558d40c1d986dd16970d2bb26d63793 \ + --hash=sha256:4f07fe3ac408d8b3f768be471dc3f56d43843c47d97c66120534467a15ead197 \ + --hash=sha256:50343c1757b4b6f6708eeaf24534b32f19dfb99fb1b762c00420867a62fc81e0 \ + --hash=sha256:50983e3be33d8c0942ab88cec3905b10602f64c469b20153c48c5d4e558dd016 \ + --hash=sha256:50a195903119008fe9cc68710535eb37f556ffffd6a7759afe70a2c145587045 \ + --hash=sha256:5558a7f5a05af9ecf744af91e5baefc436f93c9333e656c27ec253f9a6bbe178 \ + --hash=sha256:56572c42e3ecd636de8d2c3dd54cf5fc939cb5c32eb56297f176a0d366fac622 \ + --hash=sha256:56d9828f204331a5ca8850fcfe2bcce95a149f1f223f60cc7216e5524978e480 \ + --hash=sha256:579f97d5120f2971876d2ddca2968135f6944d00c44c3a6590ad7d86ca9b403f \ + --hash=sha256:5af42135fdfebdadbc2bcd9c0842a48ccf0d62794c36a260b21dc4b94d1e0119 \ + --hash=sha256:5ba14a839fbe87cf7c12a6b5661c05f324a296eb8363141edb3944ba63d4c9d3 \ + --hash=sha256:5c76f1802bab718a68ac3cce447160605c734551f95c67ae90fa1132b215cb29 \ + --hash=sha256:5e8f97c0488ffda3082766ac0f2c8150a9a58c4d05788330e479cfd449b37939 \ + --hash=sha256:5f3e96071686755d9cd3600c3880183eb94b012178e92746d68101800f0ed8a3 \ + --hash=sha256:5fb6a6e919bfb703227bc1ce6579281b84b1a2ba57deb9794dfdbec7dcd1e40c \ + --hash=sha256:602c1e9b718a3275c580149f947e7fac65044c0a20e599553fb12e9700da9eca \ + --hash=sha256:657291433bf4dd3142f3abac495764cd47d0c7c92087751e6666c6447e65fcef \ + --hash=sha256:6774814fd5c338e72ee0da5cbb9432816df450e69c019f72b5d29bdec2a1792d \ + --hash=sha256:6da32b5ff3fd78d244e37300463434c7145162bfd2b6e9e915ab164da37f7343 \ + --hash=sha256:6e1d8637cf73eebc92eba2e11d4cfff98a3b562f2505bd75bba766d908926e8d \ + --hash=sha256:6e4251c0ba4624a68a2c11471a1ac54c3306876c21f0ae86de085cc9241c8905 \ + --hash=sha256:6f275c11d1aa6d4c458e05a68be084efe3c55a113d99e3f46a318098e52948fc \ + --hash=sha256:6f967dde489ca6a8c02d093ab245d2cbf50ccb5c36adf0188b17b0ca39d24b67 \ + --hash=sha256:6fdcd6af7e2e51d1ba1b4bea16e97b074bcb7b5dd0246a9d8201341bb28085a0 \ + --hash=sha256:70cb4008ac2ed1e0ca9e824deb4b53d3aa0d939109698ebf1e723a84337bd794 \ + --hash=sha256:7457580535e019e1247ea35d6a02bf081ad30c26d0cbc210c93f6c3ab67a0835 \ + --hash=sha256:74b0a9c8270f9b0a11410e124ff8d4f18bfc1f1837440ec84da5ae7b50927b5d \ + --hash=sha256:74efb69332b85675b1eabd760a8cfc2e2cf42c60607c66f88014c1bdfb40942d \ + --hash=sha256:755933b107ea7a6a9ac916f635a70595a5b1a32fac10a8ff0b9f2ab88555550c \ + --hash=sha256:7779cd97e61ebe583ec2f1c5616cdd038aa08a4453b1848c67842176d054948e \ + --hash=sha256:788ecaa9c10533b786ce5ba70c4f2df78ad41819fd00a6c99d92b66f9a32e1da \ + --hash=sha256:7bd8ac754ebd6733a3e2a0dd1674c4d8ab086196803fd8dcd776f07b4e2607d9 \ + --hash=sha256:81c2b3dfd56c62bee6108e4852d5970b4cf9086390b6983f52b666e878c1f115 \ + --hash=sha256:823c910f046f23f4c713b8d99a2242dc65f591cb45ee86418fa11762a3c2963c \ + --hash=sha256:82c7583cd3dfdc7dcc927835b4f6c7faae7ecc1ba3ca5879321621ae2e6f8e84 \ + --hash=sha256:8966ecac808dd5f473c9c4cefd10cd3ffda71c18a4d3493b7c7d2ae1803bf2cc \ + --hash=sha256:8df7d481654ac96fe1ba9a02a9f67770fdd367823e0d5ef01b922725c4bd2cfa \ + --hash=sha256:8e317e0fb6b16212c881d2205a7d87414c29acd69320b3aa6dce9d9c7b86fe4f \ + --hash=sha256:932ce7e694bbc29b2bf6f64f2343c27d148d4997c771d01bdade4639b6749ff4 \ + --hash=sha256:939042d5cda21d41a6f512e7cc8b8e33a2aebff863352251da495fbd91b673b5 \ + --hash=sha256:94684b879ac1d71e4238850c99b62dc1b28d9086b156a2555f082010b85a865c \ + --hash=sha256:96a2e584f0b9ed8f1fa33211397dcf67bb7069866402cb405d191c2f0defb9a3 \ + --hash=sha256:9ad7e6aa38c20da1be697874349c4c273c8a03b7887169665081706398d0439a \ + --hash=sha256:9b42db919715e91eb76acf3bc492a9a7ccd8bd9adc6745c1412b689735269f14 \ + --hash=sha256:9bab2045550c4fe0f7baf89574db1b455c195750702ba96fef1f16972b146617 \ + --hash=sha256:9bf1d5dcc15204d9ec8b8ea4c18fd66e6b80e5de1f4ecbafb3a2f2740f8039d4 \ + --hash=sha256:9c061aa954daaf57d2a4b8374f9fca621ef0e1b603584431c220c22458c59b6d \ + --hash=sha256:9c428eb2bd8817588d16a0ab898aa4eb5d141f896aa2b394cc79a4cf61d9a8e2 \ + --hash=sha256:9cc882cf8619109583c906b4d4a85d6a111a98afa34b7a450d1e08118d016820 \ + --hash=sha256:9ce66feae6ac65327379460380549bf1b8df8e17c4e25df2a2bcf168272e3bed \ + --hash=sha256:a23fe35d776bc03cb495938b9594450d047e3bc08c5255315a82323e9cb7d2dd \ + --hash=sha256:a2c473a355f9239efcb72c92d5abfd8fcdb0cc78c8e9af607e72ca12dbb36593 \ + --hash=sha256:a63afd1f757de949028387e65a7127b61ad0f775432dbb0e62816ae619fe69ac \ + --hash=sha256:a6d02b4c38de03d9c7617813433e6a0fb6b522797974177d69d9dad431900833 \ + --hash=sha256:a7d470cf7b206e6359fc77b1b860632fde400d5a2ed59cd0181b93a686bc81ee \ + --hash=sha256:a95529a92a446db351675f4aab518feaf5e99842f63f5dd17160c2b74f382db3 \ + --hash=sha256:a9918e58faf62ba2c7147927d06057aec78f42475aff5048047ec47e7265a600 \ + --hash=sha256:a9d3983bd6ab7aa1cfd573544ae98df9b6cb6912a5185a198263e024a636861d \ + --hash=sha256:ab52d8f1fc1b64821c1fbad64a647ed6203627004059a6d1ed4f0858a1499703 \ + --hash=sha256:abfda5cb094a829f7bc25216a32f7db2e85cc65bd59910f8e7b40b3d9b224764 \ + --hash=sha256:adbeee7d6fd4cf5fe0aece2fb3edc4243615d3180430ba8149d01a90670cac99 \ + --hash=sha256:ae53924aa853a7a2ca20ed4142c7c6b56338e4d4cd999e2980075b9efc2e257a \ + --hash=sha256:b032a0023eb41d768ce77d83210ab2a3c389bc0b09313273c7e1eca48c10a755 \ + --hash=sha256:b1b8ece1e71132d2afba4dbc0c3d62c766e25165990b25db1196c04969eb3d84 \ + --hash=sha256:b2966998927d7bed9db12c0a4647b0c7b179755878fc9c357fe1ffd3e3b0c1a5 \ + --hash=sha256:b3cc509327c7b27f6f4727a8830f4004f6df7766e179f2f4b8e54e65c0bec5d3 \ + --hash=sha256:b7806e804889231b0e06469fd4a5c06313d1c0a3377322b6d9237fa5e0fe4167 \ + --hash=sha256:bea559ad70218d230663e4210875735076a9bfea5994cef34a55a25faeaf2544 \ + --hash=sha256:bf163cc701f3d4ac43ba7d97771bf5fd955220ef5500ef3ee847bc0ecfbf4ec1 \ + --hash=sha256:c147451b4a58e7050f7f7394e6c467867c84161560001f9ad4fb2d1446743946 \ + --hash=sha256:c172db893e516e1358e65a95ee20b7ce7173963eefe318b6ab2a2220688b999e \ + --hash=sha256:c1d60eafd9c7e8e74abd03a5b00df44e7febfe6d9b89b559c0a6551eef0699d4 \ + --hash=sha256:c2c30484dd1417ef98b51021ffa2cc0d7f3c78918adaaaab7e70817335ab3e02 \ + --hash=sha256:c32e26310cc10e547f53cd13d39a369034f69dcb7d749d5cb0e5f67bc196b6ba \ + --hash=sha256:c5ed596aedb9c42afd3fe0aae3117725378ac73d2cc5ddc735056fbdb96c5d02 \ + --hash=sha256:c8859a013ae0de1074660992139a1a440df3e6b219b86cf0d3f11c2692bb4fe3 \ + --hash=sha256:c8c4478bef6d57fcfda15dae461ea3c9f06aa7b257c58df3f2300174ccbb185a \ + --hash=sha256:cb11a971a3aea10f9b8373be628f1df932964fc6c6b174516d318a48c3ac4412 \ + --hash=sha256:cb131d775a1573c1aee66656bd78b023577bbdb6cb8349a07773bd4f73e68a6e \ + --hash=sha256:cd88b01f3d37b7a2a34f91d98f14720206f1ea3d540843fab2d649dd5fb91fec \ + --hash=sha256:d05e94cdfe0d15d0206f970722d2554780ce562787b21b218b275447f8751319 \ + --hash=sha256:d079c0a0135c36e7beb6f1c88087c8f108dc5891cdd0b5eafa778421bda70ed2 \ + --hash=sha256:d3112585250b199296c26ca6e0131640b6a8d01bab8b232d2eb3763ed469de11 \ + --hash=sha256:d418ce2af40c6bb685b3f663e9e8de27cb0a22431d8e88a167348d7f01878073 \ + --hash=sha256:d51db97c96384fbfcaf8f4c65922183a68b94f891c3c10c862ef5f6df2adbb1f \ + --hash=sha256:d94e44be379e569758fee8a9a58431cfc3c2598c708b92b1cfe96c66b4c94aef \ + --hash=sha256:dab9ac5a67c8d1f070c00fa8fccb7cbd1b8dcc1a8d6b42f37540df9b3d4cc603 \ + --hash=sha256:dbf53ae2601b7fd5a93c3944deea3a78d40f495226d582c35ef7a433425ce2b2 \ + --hash=sha256:dca3fa8d8a0a26679862eccb0b1a9151b2b9f1cd2c212e7a6335778faaff5833 \ + --hash=sha256:df37b620684e19b5e25724412518ccafc3b1a49cdac706fdbd2f983fad943450 \ + --hash=sha256:e1cc2bfaee8c214f06080a7c7d5772419b8a1108e8e5349236189811823fb02a \ + --hash=sha256:e29347c142cf6e99e0dff5e2995ead1d50fa3b51bf37a7c726a7ccfe5419745a \ + --hash=sha256:e4f5cf4dc72a71c4cfa9751b4950be22f733626670230d46e7d606592aa22d59 \ + --hash=sha256:e724a7b6091f0b1ac064f9d1b15ff9ec52e6033a86cdae649e5f086e32a3c0db \ + --hash=sha256:e87046c8ff77a8decdb6a41d8ab25824b47531b2da933aeab0c1e21c7acff329 \ + --hash=sha256:e95c8def4b81c5d68d5cf1f54c07acd7c0d2577af244e5b6da802120825737c6 \ + --hash=sha256:eb324e2009fb54db30a071dad7caf6998ee2879c4704007efb244514dad1fec1 \ + --hash=sha256:ef60869969180ec2464f1349aff07138ae35ca2200f0946cb3552e49e8f301a8 \ + --hash=sha256:ef692a24087a699c0a4a26af45e746e0c1eae2116f6d8a5ff91d8aae2b867b45 \ + --hash=sha256:efc21a454892828368b11c2c780de0ff8bc991f73f6b99c6b66e56205470929b \ + --hash=sha256:f001b571ead90ca1770f1e616db255351a1703317f20374c361ef22f12c06d09 \ + --hash=sha256:f2a7966bda23dd85051f1661ce0ace38d6890e05ec6c357ecae9d2479cba377e \ + --hash=sha256:f2ebb54b3f932210503072f09974b4fb574d823e497a944adfdcd140a6a00255 \ + --hash=sha256:f2ed8b64dc0c651c0f5a9c926777719770021251b8f336d97c4b80b660836ce1 \ + --hash=sha256:f375db73a39f5cf83696d500e21a67f418dc9a988955756f254be8f03b7b3651 \ + --hash=sha256:f59c7673465908cbe506117176156c127f29f917677afceada34957179221d91 \ + --hash=sha256:f8d40ce41991e9d56fab4f5dc4a51fe59bc3b5c77c27f4b148963064d00232e8 \ + --hash=sha256:f9033b43f511f27547c557dcaba0177649e10a3725336ccd2cce0fdc1dc4850d \ + --hash=sha256:fba47bc2c3d7303c3d027c6cf4d07626c37b1314ac81f5820c31032e0ca1f677 \ + --hash=sha256:fbdc5ec49f9ca3cd24955cf3520b10a4d4c901ba2572094c84274e9e7eb30534 \ + --hash=sha256:fce9523df31cea6284f3e2c479876750d7687cf671d7b25d32b19effc0e86441 \ + --hash=sha256:ff75a7537413a86e7cafe98e0e1d6e3dc4b15c6349896e7d5c6b881bfdb6d550 + # via fsspec +aiosignal==1.4.0 \ + --hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e \ + --hash=sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7 + # via aiohttp +annotated-doc==0.0.5 \ + --hash=sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101 \ + --hash=sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb + # via typer +anyio==4.15.1 \ + --hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101 \ + --hash=sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94 + # via httpx2 +attrs==26.1.0 \ + --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \ + --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32 + # via aiohttp +click==8.5.0 \ + --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ + --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 + # via huggingface-hub +datasets==5.1.0 \ + --hash=sha256:1561347daa8841b51351ef2ad1647b4ee4af406852c0f7bd1df587ac1ca7888e \ + --hash=sha256:86fd08b280485aab82894d2c81842c4151d93b575efea0054315104cb7bc3cec + # via mlx-lm +dill==0.4.1 \ + --hash=sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d \ + --hash=sha256:423092df4182177d4d8ba8290c8a5b640c66ab35ec7da59ccfa00f6fa3eea5fa + # via + # datasets + # multiprocess +filelock==4.2.4 \ + --hash=sha256:301241438c57936ef93f4243a94f2559022268bfcf9b375a9a72eb0f8db8358e \ + --hash=sha256:c6d8663535e1650ee430507b4ffbac7b464dd64b7647ea0112a9f406499d163d + # via + # datasets + # huggingface-hub +frozenlist==1.8.0 \ + --hash=sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686 \ + --hash=sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0 \ + --hash=sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121 \ + --hash=sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd \ + --hash=sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7 \ + --hash=sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c \ + --hash=sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84 \ + --hash=sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d \ + --hash=sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b \ + --hash=sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79 \ + --hash=sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967 \ + --hash=sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f \ + --hash=sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4 \ + --hash=sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7 \ + --hash=sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef \ + --hash=sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9 \ + --hash=sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3 \ + --hash=sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd \ + --hash=sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087 \ + --hash=sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068 \ + --hash=sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7 \ + --hash=sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed \ + --hash=sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b \ + --hash=sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f \ + --hash=sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25 \ + --hash=sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe \ + --hash=sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143 \ + --hash=sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e \ + --hash=sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930 \ + --hash=sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37 \ + --hash=sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128 \ + --hash=sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2 \ + --hash=sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675 \ + --hash=sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f \ + --hash=sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746 \ + --hash=sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df \ + --hash=sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8 \ + --hash=sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c \ + --hash=sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0 \ + --hash=sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad \ + --hash=sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82 \ + --hash=sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29 \ + --hash=sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c \ + --hash=sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30 \ + --hash=sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf \ + --hash=sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62 \ + --hash=sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5 \ + --hash=sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383 \ + --hash=sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c \ + --hash=sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52 \ + --hash=sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d \ + --hash=sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1 \ + --hash=sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a \ + --hash=sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714 \ + --hash=sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65 \ + --hash=sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95 \ + --hash=sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1 \ + --hash=sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506 \ + --hash=sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888 \ + --hash=sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6 \ + --hash=sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41 \ + --hash=sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459 \ + --hash=sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a \ + --hash=sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608 \ + --hash=sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa \ + --hash=sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8 \ + --hash=sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1 \ + --hash=sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186 \ + --hash=sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6 \ + --hash=sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed \ + --hash=sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e \ + --hash=sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52 \ + --hash=sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231 \ + --hash=sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450 \ + --hash=sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496 \ + --hash=sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a \ + --hash=sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3 \ + --hash=sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24 \ + --hash=sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178 \ + --hash=sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695 \ + --hash=sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7 \ + --hash=sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4 \ + --hash=sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e \ + --hash=sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e \ + --hash=sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61 \ + --hash=sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca \ + --hash=sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad \ + --hash=sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b \ + --hash=sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a \ + --hash=sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8 \ + --hash=sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51 \ + --hash=sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011 \ + --hash=sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8 \ + --hash=sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103 \ + --hash=sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b \ + --hash=sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda \ + --hash=sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806 \ + --hash=sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042 \ + --hash=sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e \ + --hash=sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b \ + --hash=sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef \ + --hash=sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d \ + --hash=sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567 \ + --hash=sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a \ + --hash=sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2 \ + --hash=sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0 \ + --hash=sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e \ + --hash=sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b \ + --hash=sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d \ + --hash=sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a \ + --hash=sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52 \ + --hash=sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47 \ + --hash=sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1 \ + --hash=sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94 \ + --hash=sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f \ + --hash=sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff \ + --hash=sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822 \ + --hash=sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a \ + --hash=sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11 \ + --hash=sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581 \ + --hash=sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51 \ + --hash=sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565 \ + --hash=sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40 \ + --hash=sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92 \ + --hash=sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2 \ + --hash=sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5 \ + --hash=sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4 \ + --hash=sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93 \ + --hash=sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027 \ + --hash=sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd + # via + # aiohttp + # aiosignal +fsspec==2026.7.0 \ + --hash=sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279 \ + --hash=sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88 + # via + # datasets + # huggingface-hub +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via httpcore2 +hf-xet==1.7.0 \ + --hash=sha256:0a9e802f33bf50c851abe45fc5380e61f959e2d369647d6742b79ad9d6c27cab \ + --hash=sha256:19c0e64f14175ccb6a1aff69e0d2ab9ec5269a560e6687abaf2b3fa4f73de7cd \ + --hash=sha256:2814a6e999d13464c4d679b788cc5d784eb5a4edfc638a31f10e9a11ab531ef8 \ + --hash=sha256:2b7bb5727889b0f2436dbaaad8fc4c3e66b8240d992716989e0c086b4278b1bc \ + --hash=sha256:4ee5e05a627f5ab5bad7a86582277d645556ea1e199903aae19e033a392aa13a \ + --hash=sha256:57bc157b8b7fe3bee9dcb9af7f3da8de41801c3b31a9ef68a77a33c6a6be382f \ + --hash=sha256:59fba37039233c7fcbe196817d6cdcf1b40dfb17b410f229d85b0cf0a1848da4 \ + --hash=sha256:757168feb5679647c0bb13ee5d0faebe799c4dff9051419885a566ebd79f949d \ + --hash=sha256:80f79dae613ce9e0ea1fd1ae15616ca9ac74aed4c770aabc199c4f03ebecc863 \ + --hash=sha256:87dab080f8f7d32781c2586904e3603f4e60d09bfc727706c3ae419e0829beeb \ + --hash=sha256:acc3851cf2576a8fb2ae926da863f4efabe21303cf292e9a44332802ab0dcc6a \ + --hash=sha256:b01fe18dbbd151a2403d2c64ed30dc6547b00d6babab9a617d77c7acdb81ee66 \ + --hash=sha256:b91569d5f1b61c34b043687da02c05dd3604f3d329e7868510bf3f7971599006 \ + --hash=sha256:d406ec79053c0871817f700c2ac8c36ba0d87f9c34b7458b0f0063bb218b0466 \ + --hash=sha256:e3e88a7a75d7d95cbee1f37dc31341d6201124cf21c6c4b1dfab8ccba9b09e0f \ + --hash=sha256:fa029678be1ba7f953c409b0b27bf15cc69cd1c9b3a674fbd78856ebefca1052 \ + --hash=sha256:fcfd6c22418e57dd5b3aea649e813b2e2cfb2aebf317b210d90f1fe4b3018b52 + # via huggingface-hub +httpcore2==2.15.1 \ + --hash=sha256:730f603058c260a3288378217b464713350c23e3b8fe8bba753701054032d98a \ + --hash=sha256:a94b6b3e6260e1a3f035915408d3dcd7dc8e844c2b544b642e227cf98a747af0 + # via httpx2 +httpx2==2.15.1 \ + --hash=sha256:75841cd83d16b417f7d6da8f2089ce8120806efdbf96af37d46c5129c49015be \ + --hash=sha256:ff75485308fbe441e75b1218546e4e78d884df41e1a9741c1f79335ed8fcfef7 + # via huggingface-hub +huggingface-hub==2.2.0 \ + --hash=sha256:1667f145dc56dc210d60966069397df9ecfca9607a5d43db88b308c89dae56b3 \ + --hash=sha256:5d1b47537394e4215cb858aa12fd493d0f7ef7f58990f5dcd24bc173107b2871 + # via + # datasets + # tokenizers + # transformers +idna==3.20 \ + --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c + # via + # anyio + # httpx2 + # yarl +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via mlx-lm +markdown-it-py==4.2.0 \ + --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a + # via rich +markupsafe==3.0.4 \ + --hash=sha256:007e1ffd9bf65bb6ee96df7b258fc632a4868dd5566037986c64781f35a36e98 \ + --hash=sha256:02fa4acbc6a3fc5c693c34d4dd8c1130b7fe99cc915181b0ddd6f72aeb296002 \ + --hash=sha256:03470d1a8268e692ecf79ecd565593e59d44219377a7ead61f1f1b94c1f7ff6b \ + --hash=sha256:04e7902ba80ee4bac1d50a549606527a1dcf0476cd81403db41099d3b60ec653 \ + --hash=sha256:051417f74bcaaefa316276e0ff723f541616ca51043d070da00249d9bddd3e3c \ + --hash=sha256:05295589e619b9bed252a86b532b8e27350abc372d18ba89b59375325e91ec1e \ + --hash=sha256:06de8ef6331f6e822c28d577dc8bf43fe398800477c49498f38fc38b67ff33fc \ + --hash=sha256:0764a13d34cae40db7bbf3a09b7e9b491bf4603e20b263a7a9d6b8e324975d0a \ + --hash=sha256:077293e425f28ec737dbcad442a71752e28f8ae27cde3d68acd1fb212091cd92 \ + --hash=sha256:0930db9bdc62d22944e10b066448bb65dc9abe9112880c7cab8da54db4284d5f \ + --hash=sha256:0cee7cb0f9a1b6892ea482237d9403b3d1b4603aee057d0ff01f0fac2d019a97 \ + --hash=sha256:0d9c47709875fdb321452056622e930c52afbc07a7d780762fbb8b4d91ce6fa4 \ + --hash=sha256:11935df9bf455ed0c04eb87bcd720f02b1fe5e02128a9430f23aed6f93336fc7 \ + --hash=sha256:12a606a492de952afcb43b59a14aaaaad120e708d3663dd0fdf2d738d427a691 \ + --hash=sha256:14bd2d845d62ab678eaf81da89d7b621b51756c72346745c1a594c09d49207a2 \ + --hash=sha256:15ba9e28640feef770374b116a6f019c21f52404aeabe516aa7f800587b98cfc \ + --hash=sha256:18a801868a884f216e784d7d14db2a4077143ce7610440aee2ce8f734e7cfcde \ + --hash=sha256:1c0df495a977d10460a94941799c72d5b5ab03d3858d949b55b5a66c8f371c99 \ + --hash=sha256:1caa2fa5a6184fb233153b35f654e6687bd555476f6170f29d8ee9be1a8b0af9 \ + --hash=sha256:1e1451fab512d1bcc3dc26988ec1edb0b82c2db909132872cd9356070a6b63df \ + --hash=sha256:1f1f9477e174582b0a1b583d60b66e1f2cf5d3fe12cee985e4aedf44766600e5 \ + --hash=sha256:2628d3a8cb648ecebb3c5d6b0a1052d400e4d8b7ac0fb786be8d285b50040d17 \ + --hash=sha256:26e9867520db70d37f7fb421a7f0d8adb40171011fb84ce869afa1a83370dfa8 \ + --hash=sha256:2a6ef68ae94aed8721934072b27a3b654ea2100b97e4ab864cf1489c90926fbc \ + --hash=sha256:2b2b1e18af909b448bb3cf9e3433366f7a8726271fc214e8b10e0f62a78c724b \ + --hash=sha256:2cb3dd71fc6be918ad4264346a8ed69485f9b7ed7bf35495d8e22807cd6b8bea \ + --hash=sha256:2d1b7d9308288661f56672b1b157d75fc536714d3638487bbea17b6318a78248 \ + --hash=sha256:2dad610540cb2e6272855c178f08ae9a1c7ac258a7fb71660553a5f104b42741 \ + --hash=sha256:2e5a7cd7fdd14fcb1ae5d7d8bf23d24fbd1daefd1fbca2580132e1ea75f098b5 \ + --hash=sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6 \ + --hash=sha256:340cbb1957ba99929cbf19a75626d36ba1ae21d1730b287d1cf7f824a20c4fc7 \ + --hash=sha256:34bdde374c5932765d7dc685c4a1d191a3207852d67e8e0a9eb6ea85156181f1 \ + --hash=sha256:353bd63081912ab8cfa6a0c7d185934cdf8426f04c618bba6bc4b394f2069b67 \ + --hash=sha256:387d8cd30e69b3f0a72877b9ae717033396404e19095b17fe89753a981fda44f \ + --hash=sha256:3882fb412298575bae3b9c46868251f15cc69307359f87bb1b382e53d6e5a2c9 \ + --hash=sha256:38fc55594dab834470b6733dead2ee9e3f657fb0608c769dcafa0ba5ab52f45c \ + --hash=sha256:396ec4e65cc889f69786b3b89478b471cee5a3bcf468b9d9bb03e1a30fb291fc \ + --hash=sha256:39dbacefc411633db5b4378b066a9aca70a3d7e2922c9e578d825f844026eeba \ + --hash=sha256:3a93d9616ddecfb393727a0041a562cf0b15a244e20f2bd25efc7949be4c4f17 \ + --hash=sha256:3d23795802fc8bd72534836d64489bbf0f67c088959091bdb22e10735a5107bf \ + --hash=sha256:434139499bb20b502ed3baa1f169e618f924a97e7a777fea1a49446d80106cf6 \ + --hash=sha256:436e3ffc6310d3c41878c601db29098102fe5d8a467c49da4a4125254e0980f2 \ + --hash=sha256:489505b03f692c3f376394e49194fa7a7f9e8558d6e293a7056a0032b0c38163 \ + --hash=sha256:4a540e2d3192792fc84eced57bef37851ccb2b41f73291bb17408eea77bcd278 \ + --hash=sha256:4a7cdc2a420ca01058182da4253329764d4bfa055564d1eced90e6ba1e8b1d3d \ + --hash=sha256:4bced6e2a6dba6a28f7dd3c6ce14df1b2dd495923f16ea484cad03decd463b2b \ + --hash=sha256:4cf3468d5ec187ffffcaca8e61929a37448f215dafc1386a12c750a72fe53634 \ + --hash=sha256:4e2c4809c14559aa7ef426f27fb35afbb38104c349a903bf8f3600456764bb38 \ + --hash=sha256:4ed644d75aa94a2baf7ec3a96eaa160ea58c742eb9d27c6506053c5c40fc84ed \ + --hash=sha256:4f6e0852a0283b1b1fd776eeb7b766a5f440b3e2bd31ab51af3b400585f3965c \ + --hash=sha256:5066b244f576f91afc8ee3ba029a89f99d39c79b1853fe9d39bea9f0afbec148 \ + --hash=sha256:5086f9975abb1ab531ee6afca1761e4b59a19b446f3f6522ed776963228cfe5a \ + --hash=sha256:50b5bedc9ed8a94fc8857a42ef4f84a81ea88f8d4f05dc8705fb23ee6d8dcca7 \ + --hash=sha256:52704c5d36eb6dda8866493decd61111fff86244c9b1ad225ca01b9e91e5970f \ + --hash=sha256:55ffd6ce583d97dc71dc92e930324c8c0d25aea7e3ade6ae54ef77cedb096811 \ + --hash=sha256:569d65055d367e3dcdf30c3f41119467b73d9ee9faf332bdf40402644f5ac08e \ + --hash=sha256:57f9947a7e57a081c1e3e0a2dd0d2dcf290a4531450e6f611e30084c222a7295 \ + --hash=sha256:5989cb26b2e1efc6a42216a9f6b5ee495ce5ace2e5b352a9af489976b32d1ee2 \ + --hash=sha256:5c22873ad1f0532ba40fa1727f3c0fc1bbbaab6d373d4cbe3f0dc74b2e2521c7 \ + --hash=sha256:5e8b3d0b18fd623afa12ecb2ce8d8becef69f9b5440c6330c7972200e0bb84b0 \ + --hash=sha256:61631e08084be9e21a8967ec3139c7616ed7c5e9368e05c86d1b39562c8a57b6 \ + --hash=sha256:64511c54db4e4987aef4c41923235927428729e8174c5dba488429be70a998ed \ + --hash=sha256:6669c1bf34080161ce49c589cc512ef24d4c704ac9d2b2d3667f519c60418378 \ + --hash=sha256:672d207103e6b16ca098611b0f9efad6bc00afd47c03d6ef62186495ca677dc0 \ + --hash=sha256:6768d67d1bce64270e0fdc2e69309d68b9b18ae56ddf6c711d168e9d051c2cac \ + --hash=sha256:6a45c3d514f2436064db00d7fc8778d888f0236ebfed649b53d13a59e69ad51b \ + --hash=sha256:6bd9e1788e15bfcf6a9082de42e30387e7b85d211ab21e57a939bb8cfaaf8d96 \ + --hash=sha256:6d2a9efe686f9de00d0d1ea32a4a5a86d558a2277501bd78d964214eab625e59 \ + --hash=sha256:6da83a088f8ef93b2d483a8232a4dbf4d69d3d8496b568a03c56becac43e1808 \ + --hash=sha256:7018d4af1cd272e847aa5917983ab5e83e4f6579f9dbfecd4a79c0ca80b144c2 \ + --hash=sha256:71f88e749ea29f67f21f3b36433c1dc54c7729ed2a6d9e2da2e0d9e0d7b224eb \ + --hash=sha256:737c9c3981998eba27f11786f84fddcbabc74068b72a4a1f454ea02094b57b65 \ + --hash=sha256:73e77980c7207854f00fc4e71fb1626868d5740ab4012623d55c7a99ad122a72 \ + --hash=sha256:799c39bdf5e2f1292fedd3009f7b3c9e760f10b2420cb9638d56920840ff6db8 \ + --hash=sha256:7a83aa6e4805df46fed18e989d3d16f86ef60cb50bbc8d9ce3a6be89165fbf6e \ + --hash=sha256:7d3391b2188d18737cb2fa147028b1096236eaa7e156446c650a489fa2cadc91 \ + --hash=sha256:7e1636da3d8dfc220b6dd10264db5f2b165e4888c4518594898fbe381049af8a \ + --hash=sha256:805c8b84534fa10891890f0e4be39f3a99e94615d93e8836bf9fa1fdca2feeb2 \ + --hash=sha256:811d02d5122171c1941357efd8f9bf4ffe907b7f0a1a4e729a880e4be3f46e3e \ + --hash=sha256:8138eb83940ec7299024d92d4dee45f601b9e6c5ffde9d25f4e35e326203c707 \ + --hash=sha256:83b3944fea42a8400edf92fd1770fb8d0d4f7de651353bd2d8525a92dba69a21 \ + --hash=sha256:849dd2bb0e5e4ab2b71c7191726a4a8d5aa8a610daa584728cbee0b710ddc4ef \ + --hash=sha256:8698d70a8081ee8c090dbb394768b5789a1da8b131b5499f89d071dd3cfaf6be \ + --hash=sha256:8781a792a070cf2bd1b86d3aa943894115faaba6e88122a7bf32d62072742453 \ + --hash=sha256:88d59b473bfb03259722600839af9bbd7fa13a2eb514beefeedb95997882f69a \ + --hash=sha256:8909c2f1c6dd65e054ac4b573a91c8384d1492281e55d82d159d653f7a13adf6 \ + --hash=sha256:8965520ac587c94a4ac48b729be3d8b8de00af39699b17585dfb599babe77977 \ + --hash=sha256:8b5d563170ff8ba3181caa967c99a3c804d1dedb702c7cb93a6a7c32247da978 \ + --hash=sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581 \ + --hash=sha256:8f0fac8b13d14bb06c68195f849371924ae53dd7b1c00fed24650f704383b692 \ + --hash=sha256:9240187afb63d2f9ddc3e032c670356fe941f6e20662ea168a5dc3f1f317e1b3 \ + --hash=sha256:925f929d6b59a8b3f8b8c6ac363cd0af7eecc81efb3071770b3c6717c450a369 \ + --hash=sha256:9348cbb300d224fe3b89793262cb093504d4ae927004468463f745188a193e4a \ + --hash=sha256:9388003072b95f2f1e3fd908604194d653ba21330d811961a78b7da1a77e9e36 \ + --hash=sha256:9438a2648b2195980cb2dd8e53ed7b8df91319e2d0b70ae61a9e1d1bc8d3bec9 \ + --hash=sha256:94e4c421742086aeee4c32a506eec8859d7634aad943f7e6aacf70f813478768 \ + --hash=sha256:94f5407f7bc64fa6463906b896f9904beeeb7dd8dc116ee8e9056c8714ff9916 \ + --hash=sha256:971a3bbb75d97ae4e2e8f7d4834236f86f85f0c85e04ab2e191db1123b04f80b \ + --hash=sha256:9e227f3dbe6bde7491cf0a9965d00b88c6b1a4a95d11480ddf88bb96d397c19f \ + --hash=sha256:9e25feb9e330b63edb0278a0acdf85e50d0cb0fbf49c3084abbe4e24ae195346 \ + --hash=sha256:9f098115c247e11d138ab83a28fa0323c77015007ea2df73ba5fd714dfefd67c \ + --hash=sha256:a18f38cafc329bac5e3c2b96c765b4c96d3d103421ed22ab7988c1e3fce27464 \ + --hash=sha256:a4bbd2d87dd233b9fc5812160c3d0ffbe42edc22a26ce0469f58479ede633fe9 \ + --hash=sha256:a5fcffb37e602b0b3c1638a97746b9b96125caa9bcf6fa41d337a9261de231ee \ + --hash=sha256:a8e9f292fcda89b324f2f5c91d13f1424a153e40fc2756f38ee23b15835ff300 \ + --hash=sha256:a9f54054101545a9a9cccefddf54316aa6e4491611fcbef9e91b3b6bebec04f6 \ + --hash=sha256:aa2c838cc024642cc04c6854232f32b43e5e22833dd11119c1766c7873b8370d \ + --hash=sha256:ac0c7c9f1609b0c4c114feb1d7a3409564c7fb77e360bed9e97e5d25dfeaf868 \ + --hash=sha256:add96447a86d205ab616665d53b2950ee81083757f56e6ea833c8b2917646b46 \ + --hash=sha256:ae9dcb8fbe244cb82f8a6458b455b927a03685e383d9bacf1ea5ce180b96dc97 \ + --hash=sha256:b4a635a0487774f841cb1fb62e907e7195cc95bc761e053184b8acc3ceb20733 \ + --hash=sha256:b4d12837e0203bbace818ff4a7461afdcd78bcd782351cea148139180d7bcffe \ + --hash=sha256:b61687d0828e72bf5cda24a2690188f37170bd31c9359ac97e4e66569f120a16 \ + --hash=sha256:b807e598953730f82e4eae3bd30f6a122cf6b31c398c6b504c0e04c13c170429 \ + --hash=sha256:b8cd1f918b26fd7b1832ece557cc18f2d8747309ff8b3f0ef9d4250c5ad67a39 \ + --hash=sha256:b91cc9d336957239ff200f30097e6fea2dc6d6fb3c81e853eaa09eac904fd894 \ + --hash=sha256:bd3ce56ae2cbae3ba82b683bc425cd7e48d2ed8b10f3e818186b6f5646d9271c \ + --hash=sha256:be6cb0c799abb0e2ba3e618e6d28ddddf7e485f6c2ce938dfa237daf3905072c \ + --hash=sha256:befb4158af32106b9a93db8d6d1d1cbbd418c0d5aca0cabb7b1780abf0c89169 \ + --hash=sha256:bf053da3c97a4bc5ecfbb218cdd2983febd91c617be8367d139882aa11e490aa \ + --hash=sha256:c02e8f18bdedba082cef725942ac823b9b60656db07f7e265cb31618dfd00d77 \ + --hash=sha256:c1bc67752d5f21013cfe430df4062441714eab79f65a6a05e01505957e9c35fe \ + --hash=sha256:c61750fadcd119d0825bcb7d7d675dd264dcc89cc05292aab5be68ebdbb374ad \ + --hash=sha256:c90d5b3d4e944e065a301d741b3c1d784f6bd1f503aa68b4967e32b2ba313d85 \ + --hash=sha256:c9a7f43c0b202b334cc9184af09bb8f21d3a209e038efaf106936fb69e6b026e \ + --hash=sha256:cb96e6e088d6cf71c1ea977510948320234824cf226e32f6f6e044f7a9c82b34 \ + --hash=sha256:cf63c214fe879a65e69a386f915e36104fc84254ab141240f8854602d8e0be2a \ + --hash=sha256:d1aca03ede943eb80ab3d63bb082c84b7aab85ea83bd0fd0c200260945fb49d9 \ + --hash=sha256:d2e56fd3b00222722abfb3f5f0759ddbae4b90811b5ad4343c64030ad1bde70c \ + --hash=sha256:d5f93ebbeb8032d47e349328ec8662d973d9b05a70b3c35df1f91fe419b84749 \ + --hash=sha256:d882a373d8093c2941e01291b7ced96e9cbe4781da9a7751ca7e6c70385e5214 \ + --hash=sha256:d920abdfa61279ba1a2ef9484aab07bf03331f8c08a10120fa332353d06e6932 \ + --hash=sha256:da2af0d7aebfc2074080d72efa6ab8317c62481ef1f896f65d9999c1c01f4494 \ + --hash=sha256:dd8ea6ebee7aedbf7c749fa80521d9ccf1ba473e0d1e14805caafbaad281c889 \ + --hash=sha256:de8b364c423ef0a4bad9069657d617f9a5d2b2062457a89b1fa16ee199c399c1 \ + --hash=sha256:df1ae86ff54725a01fa1a0510b914ca53a161b7050be74f6204e24aded5971d0 \ + --hash=sha256:dff05cb7016dff1e9fd68f4122c127b65dfc59de5306cfb7ad92f956f230bee2 \ + --hash=sha256:e1a622f13970d81f95d0c72f9dc090dce9085fccfa4c9f2174377ee32bd15786 \ + --hash=sha256:e49fb0d1ce92cfa0cb198cc5b1b11cdf9d0638658e2a2db2687e39db7c87fc78 \ + --hash=sha256:e5c802729725bd07e2bc3ab7b76dc7e0bbfc53129d8f1eb1c002c24cf774717e \ + --hash=sha256:e841068dc0be4cb6dfb5c890eb88cbdcff2f4a332393c7ec94e8e618bd32c1a8 \ + --hash=sha256:e916035e3e9930cbdfdd10abf48861340221857f45509565898e012263f7b289 \ + --hash=sha256:eba154571c16e032112afac0dc2dfe9e63c2ceb7aedd07bb7eecf2ce26d4dd4c \ + --hash=sha256:f03460ff076f70ab595bb45a0205ccea1971443575b6920c52e755dec2b3fbfe \ + --hash=sha256:f0ec3b750b59375eab5b0fb2b9254810c00a3375be6d789899f1055a1d556237 \ + --hash=sha256:f291bcf42ae98eb5107edb162c3c998b4a89648fd8e99ed4cbd12705292788cd \ + --hash=sha256:f61efe1d2fe0de16158a5fe1d1cf3c14bdb6aecd54d8938fd26512c525c1f624 \ + --hash=sha256:f68edfc67aabac33708941f26f22a7b8e9f81429bc0cf249fcf7d66b23af8d19 \ + --hash=sha256:fa95848c929b6a75f6848d3c9793e59db365ee436776e57db835cdbfa79ba977 \ + --hash=sha256:fd9f8797427910198f95bced71ddfed61130d7e349213bfb8466c9c99e2c46a8 \ + --hash=sha256:fdb4ca07ab75ffadab4a8b135ad59cdbb3156b99310f3d565370da74a15d6bd3 + # via jinja2 +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via markdown-it-py +mlx==0.32.2 \ + --hash=sha256:0007e4a071756c07c79f4762aa6ccc1d646da2621f5221f4467ee9419733e0e4 \ + --hash=sha256:1a76fa10040ceb6b119f015e5edce7678387f2c73ec51818cfc1a22441910795 \ + --hash=sha256:2382f3be0f254eb44f7e8571e9e7ff60f88d55ac2f95d0a141d8f997e4f487f0 \ + --hash=sha256:238b50d2ee3917c836e73f9446011518b79ff094940eb0107fa6cd17d02a2eca \ + --hash=sha256:3503617e3aa6a8e411df53236d5bca039fcca97556f1ade1c4e5cc8a64de52d2 \ + --hash=sha256:41f436cd8dd2e052ed0eba0cbc3b7aa65069aed294c74c636a00d14e85352fca \ + --hash=sha256:45857fadb381fea3db57d9681b04c987813fa883f3182ec9fb05bc56f468978a \ + --hash=sha256:48e8738b078eeb7bfde74931fc02d8aa9c32f05d3c0dbf97b5c3cbe0470fbddf \ + --hash=sha256:4a1738951f6209c7c6a7d45a1c0192217ddd76cf7285433e22677b7a98fc3f62 \ + --hash=sha256:4bc6709cd92a46de44caade61ec7831647005e66113243f681168f67001155d9 \ + --hash=sha256:50ced716f4ab860cbda9d65adf74877923fe26036c4e28d8f572ecec57621cfc \ + --hash=sha256:52e2912df37d5712947fafa1c9949da6a15c8cd5fa6eeb837aa42900a3619489 \ + --hash=sha256:5304e5db83d982d88cac54449b34daf4e928646c6ded8799a171f6966cb253a6 \ + --hash=sha256:583111ec13fedf63ddbfefda77dabd57168978f06474f5e5d2637180d154fcc4 \ + --hash=sha256:65beb9ce75153072808ef18913ecca3d929b87be5d7c1f57d9df391c04b6f957 \ + --hash=sha256:65d3d29b66045ed8dd2d8e437c8770de325843c364f7b7c38cd8ae90a7eec854 \ + --hash=sha256:68560fd648c5bb900aa6f6765cd74c5a8abaf092d97d73584a57b7545966c227 \ + --hash=sha256:6c615ad1c6877d7d38affe8526446955cd48d6796e871150f59145cf0d1a265d \ + --hash=sha256:71865f82b36768f9d6f7d94a1e19a5609e85abe8e90577ab027e1b13da03c777 \ + --hash=sha256:765af966cb92abceb9d50b382908bea606d0796c04a9b4957f98ab630d030c36 \ + --hash=sha256:77217798a2b036bae9f213b851d4cde4581893787c9964458b7d471f86036bd6 \ + --hash=sha256:809ae9c6f944b5c60309eab5d6c042e4b69fb67bd9e87be86b8baa3f31df66e0 \ + --hash=sha256:8668d79d528753ebcb05c3cb8c4fb7a34800bee3f67f216202402f1301f71866 \ + --hash=sha256:8d270ade1e48e006383a6b5f33a3a4ec5c265389299a60a60bf55165af1770d9 \ + --hash=sha256:8fc433e35a7058e30f7a225c39fce06ba41394fe8e9fd67383b70f0b06de398c \ + --hash=sha256:94f4bba2183f6875f5f6de3e78939da1471626f2458c16a7ae6b494df8899d8e \ + --hash=sha256:9d21abe340403b6bf445e8494590a7341b655739088c92f178d7bd4241ba0110 \ + --hash=sha256:a1d43722f22d95435ce344bacbaed09128b5b982b12a1b7d5f0b695696e7a84f \ + --hash=sha256:b4e34c337403e788900dea304453b52760134716ce7ef9fef4553ffe6c6c456b \ + --hash=sha256:b50c3d6724ad36a7ed173043a78d2c2ec5fc1b83df8ccac562ccb049d80d8601 \ + --hash=sha256:b868f8e9c4f6d143c4977ae889960e7e5a5b07103ed50bcf00889be95a130335 \ + --hash=sha256:bc69bd1062b97028ae6b79522ed0bd635a2b133fe3c398c03b66b060c20d9240 \ + --hash=sha256:c0abc76090607b3522faad3e73580c5c5aff12158074297647508d8ab2ec035d \ + --hash=sha256:c7670ffb854c11e6776349a3797fa076d1a2a00290ebe2e9cfed8b60ca4a5db0 \ + --hash=sha256:c8b6ac97e9b0f0c3df50563af6a6e9aab72e9bc2872c8962fcfa84792013a1fe \ + --hash=sha256:c95a384de1a0c0ba18425344cad8ac87180e3c5c1921a42e2621475be5966bcf \ + --hash=sha256:c99e3b6451bb4a55071cf19594f564435bd88de77008f377e611a40a8bbfb328 \ + --hash=sha256:cab5889fac45696cf2feb0e6157404cf814c0a54f1d44f1f6879d94b4d712b81 \ + --hash=sha256:cf63fd5c32258ab07523b06401c20ee2280bf56e26e991d05bf6fd8a4d42d1f6 \ + --hash=sha256:daebf84dfb857d70e87b1b98189a057e691e99a4a2b9f6f64cadebad917f8964 \ + --hash=sha256:dc5eb3cc30d4285f2734c368442f717599e97294663441cb42959e3b856d6898 \ + --hash=sha256:df8c75e509de868fca148dfeb38d92ce956eed386569c87caeb72bd16d2d6962 \ + --hash=sha256:e0828c96a6b988afbe6f07cc39adc0470eba9121f7e356623a8fb9d9a6397533 \ + --hash=sha256:e507dc67d268cf78254e17756f25057198fe74791508fcbd1e6d967673a8d374 \ + --hash=sha256:f6071e4973927966c12b3894deb75f4fcfc33300e0705312a80fdbbaa92a9c4f \ + --hash=sha256:f77e47e6c1e176a61ec1bb1f9e74d874482eea6eb86ffb162a1f3e789c806453 \ + --hash=sha256:fe813a4dee2daa6d5ac496ef25117034728c77fe26f713983a3b406eb6f4e8f0 \ + --hash=sha256:fe8c1ecdad687259944b76d67621b00536b00194557588a4604a45efcd238d8e + # via + # -r Trainers/mlx_sft_mac/runtime-candidate/requirements.in + # mlx-lm +mlx-lm==0.32.2 \ + --hash=sha256:1af8a92f6a141c3e2bf5b0369ac4464203a8e2189c47b9e40d3e921e5b3a1fe9 \ + --hash=sha256:5c0991248cfff5872503d74f466d2532dd8cb7fb3670cb9be7a06d7305dadc04 + # via -r Trainers/mlx_sft_mac/runtime-candidate/requirements.in +mlx-metal==0.32.2 \ + --hash=sha256:3825fff379dbc107dd3413e564a06caeaa24819910ec49c0439e454c06a1b9b8 \ + --hash=sha256:55a369250d220b2cf10213a87a2ac1b1a420608c5b35b1df4e7147ac8e32f121 \ + --hash=sha256:e6abeac9ac5265830c9c1541b6f96e9be37a85c2446763a46ad466c63a3837ab + # via mlx +multidict==7.1.0 \ + --hash=sha256:0179698c3c913eb64f32397083747fad20ed0f0a2b7469a08cd1a8a95d14d90e \ + --hash=sha256:034b0dc1b7fb8279599c5d8563f86abb4d2454735b06544ecab23c54572ad2bd \ + --hash=sha256:05d12b4bac53abe0c65f3163af2b45894e2e1c0cc55493ac784d52a350047d88 \ + --hash=sha256:0604ff025497a050a2b2dcc4ae0e5cb6477c525e57b89825152c707e88d74d28 \ + --hash=sha256:0631eb5f49f67de10bbdc3f64141326dbc62e8d319900966648381ce0845d8ca \ + --hash=sha256:0747a83e7ae617793181a4763ee8b84863cec5c0bbbde70c4394e4c0276c36de \ + --hash=sha256:08834fb8b20e1a985c70e8380a10940234b4162de62694458727330376e58b33 \ + --hash=sha256:0aa1ba3ff7cdda05a1242490612976b2ae1c90fc6200903ef8f53815dcb35c5d \ + --hash=sha256:0ae91de396d5c4ac97cb24dbada3d5c91a51454781e0a70476b008f4e879e4f0 \ + --hash=sha256:0e79ed92b1dece6bb57e9b46effd74d7a5d3d00187c85466d880ed184239a698 \ + --hash=sha256:0ead852a5e906a43fcb6784eeac480f6a67919a51d480c1f80d32ddf9d615475 \ + --hash=sha256:10202ba98cfb3f7eb60da7ca87a2c458a69b7d0d6e4d4388cd6773ebbce89085 \ + --hash=sha256:1101aea5c3eb1d26e090b931c693488af0db9f3d52e68be8d4cdd807dad9841d \ + --hash=sha256:128ea4142f81a79d430f3d0eb55206093e5eda03a12abbc7b03c34748ff6116b \ + --hash=sha256:1348ddc076251cd542f4a99ccda4b7c1f8444e8ab489d3541a978ca5901c7c1f \ + --hash=sha256:1401caec21fd7f002e79ab6806bbfd1f54bb3de6d5e12bd91c6685dce16ad2be \ + --hash=sha256:14b1ce8579a43dfc0e592d93fb1d63dea693e4977980ac4166f26d494cc7a358 \ + --hash=sha256:159976f9c40f96e3fe0952b708846a43a76bacb114e9cc828816f5080bddd5ec \ + --hash=sha256:160bdb3520fdadcaa21e1b98aab2e011265070814ecab3804eb61674becbd400 \ + --hash=sha256:16b21164797bde6f417066d02775975cc2e15ab8abf80efa55fe85e0b4894020 \ + --hash=sha256:170ba61761f59ab92afcc86ce5534a3f3d0b07c38b339b950a83213f22dd86ec \ + --hash=sha256:18a447d46a3a2f1e61b365cbf5627db7030fdb707dad70c4f2760e5144166ecc \ + --hash=sha256:1df055e51fe7491120cc84f3362bd43db186be78d0e4c476acad45e435af9ffb \ + --hash=sha256:1fed3d721f75c25a9fcdd0e362af53f4b20acbcdc63081112f85419ba0ce3444 \ + --hash=sha256:2128f3358335e0c83688ecb40c19d9d6606cd60784dfbf2e24e980ac2ba87b0d \ + --hash=sha256:23f6d325241b0db006ca2841309ed17622137e134930a740a8f1331ec4404791 \ + --hash=sha256:248dabb89b5aa90b2f7e43e045f048f7e5392ec77b6446d80853ba7117d7bbdf \ + --hash=sha256:24ad4921135a1410d95b1f1504f4901e1c64cea680014ce2c3c7a825f4f259fc \ + --hash=sha256:274023bf952f849e0d05eba28a4c1f65f9796430d2b09ec16539386c0f76554c \ + --hash=sha256:2a964dfeb2aba3663f0536c809aa1ff385f065e89fae57e883fb7edfb4067c2f \ + --hash=sha256:2ba6611fc93c4b169d0e0ea376ebf4b8a529933d1f5f2c2ec7d8f8b93ef58ec2 \ + --hash=sha256:33376418ab2846b931a72b36cfa16810befc4f49485d0b3f4dc054a4d6d00038 \ + --hash=sha256:33389fe084e5426d9fd85d7d9ca91a29cd0d88a83c7c96e411aca49a3f9967bc \ + --hash=sha256:34a35be8fb82d37087e8176aba907b9459f03d0e293c80f574c6337a436f4eaa \ + --hash=sha256:34d2ee98e15d5cfe782a431bc913fce3b58cf3fdb34fcb437aeb275cdf9007ab \ + --hash=sha256:35534b366410a36bb3d6f788691e37a76e4d1da48326b0ada3e5032580dd76af \ + --hash=sha256:36b14886aa3e0b8786ecdaa196374422c7b1c1dcc8764d02b2409f74d47914bc \ + --hash=sha256:379f477b98a1e9a77ddc3ccaa8c709d3fb4a288ff54b96e171e637b55b4adbae \ + --hash=sha256:396ba9917fe489ec3a5942ae3e29e91324c8b9956f371f7e124c971c71379e7a \ + --hash=sha256:3dbaa7f7c2f0ca8578895fc61fb8c8e50ebb405dad8982f92f4343285c7a3fda \ + --hash=sha256:40f586bc8a084a3671ddcae9e5fbd3228a596bfb63d9f0380f153f9a65b69f08 \ + --hash=sha256:41e0c3350d08994ee8640c39884e16514e282f70ba40f5b2299582509a327774 \ + --hash=sha256:41ff3202cc23c800507777df5a4805b402f262b31008c60fdc652aeb6db2f278 \ + --hash=sha256:439a19f7fbbff232ce96682c57e27030b8ac3a4b8121484c94f04bf99d08bfff \ + --hash=sha256:44f7e5dd83a615636b80182bdf446ece57ed61d5d51854acc5d9840631136d4e \ + --hash=sha256:46d4af0afc6eb9867b3ae50605787c80b868e2f52eac3801246034925fe578b8 \ + --hash=sha256:4b5c41e44da74383c924cc5d75ef0a268f301d69305b3c42bd17af685d55e412 \ + --hash=sha256:4b87ad54e8d4adeb0a1f04889504d6ec7f04fb02609220810f51f1b6c66bc1cc \ + --hash=sha256:4cba2b0b9235fe10e12301d6b4cfba0f353fa668d635f6e988b03623c2cd42ba \ + --hash=sha256:4e11e7299079718c78f8147e7206c22fe35bab4466d38992420795288a0b8096 \ + --hash=sha256:507151e1e3dee95e9e8159e329aed4f75aa5205ecd6505a4f6be546890eafbe1 \ + --hash=sha256:50acd7ee7096949b04482cd7720cb6b85eb9cd9dd5d7ffb6704bfda250261a22 \ + --hash=sha256:50fdfcb03be719d9573597b095b1175d2e9d0b30d065791dfd9fca727c499442 \ + --hash=sha256:5129cc1f5fec6888e2db0be936dab67242e32c738811c8769aeea93aab4257a8 \ + --hash=sha256:51d7f33be9a4a1a2801430846d72841deea0894eae8381a07e7d90e0f71b3c4b \ + --hash=sha256:535173fbcc3933d84f9929d49d7a59a0faec259ee07d07c34c7d2a980b4e3683 \ + --hash=sha256:53daa47dd176db64bb35170e3d5d0ae2388c060121201883696278f055a0e70c \ + --hash=sha256:542429c796430de924d03b68a6173bb6d79d5c4967d4e9a18de3e501cad55593 \ + --hash=sha256:544f2642a456fa264614e975d921540ee8c3b368b04d5aa1ddbec33241b13e08 \ + --hash=sha256:55392202cb374dd1a1f89a8ce1586644870d9e936752059d053e576acc50bc89 \ + --hash=sha256:5c6455f2c11daeee40665c67494cedb426f67dba7375710524071c0c56d739a6 \ + --hash=sha256:5c8074ad4d67067c87bd0663dfda654f786336078c8fd7d2f6c1aa41de8494cc \ + --hash=sha256:5c93473d0d7cd9bbb370973a9679a62f381c7050d7dff4ad6aaa92e8650f5a79 \ + --hash=sha256:5cc58ebb731200ddb64d55f1b345630fb5f7a8138cdbd242af9dce964a7cb03d \ + --hash=sha256:5d19bb1ec12e385c09215d5d53a243c060c7e8a0aacdba16d933e22902ee380d \ + --hash=sha256:5f21fda91bd6c34455bd5c312e42aa1334da46cdafb4c533ecd01e0f7f19250b \ + --hash=sha256:5fa1484f74d011addf2e5f5a0378ec41521989839a05d6051d8067d8ce732423 \ + --hash=sha256:5fa296f14068538fced53c6eec86520a2ef3d3d27a0fb134640d03e067986d5f \ + --hash=sha256:6120aab922bb3e15800b6655558cf8e0a5cc79518e954d457f064e5b3d5e9bf6 \ + --hash=sha256:61a4e5d81b8d4e4ad61964b230129e7a2b914793d96289029078fc9009f074ec \ + --hash=sha256:67fcf28db77b385820881521db7435e9f1c607cfaf07db6eb78aa9d1146bde86 \ + --hash=sha256:6ab323f0c5490abaf35a78563e1043c7a772eb86d93f359ecc0fd286d1cd3807 \ + --hash=sha256:6ad60de1f4c702448fc8f1449f05e810f6b7957c08a5b3950c8a792dfb13b50a \ + --hash=sha256:6b7cd1cb0b363cd43ebf499beca26d201dd8b89eee49fae60205c82ba13ee03a \ + --hash=sha256:6c2144785e42527404bbd5cfd11981fee4abe59a22aded0e498eb711a831d3f3 \ + --hash=sha256:6c9fd50f636a8fa9cb6324cd3eac962fec2bc5bb432452a3b583583a1059acfc \ + --hash=sha256:6e7f70d912a589e30290ed926f90ddbc3160998359cbad7c9ede1bcee481748c \ + --hash=sha256:71196ebb8d523148e5975396a444de02367f204b53b14e26794c96b2be0ed742 \ + --hash=sha256:71acdc6eded0f4b86b5e16c96314887cf2572a8eb5d8038b78583d0c0eb3aa1c \ + --hash=sha256:77024596b9046572c4e90b34c1ff212346756dc48933f90c53cf6e233660788d \ + --hash=sha256:7a90453a79423cd7145cc08fc92322dcd7aca4862258f533e03f473226d4b835 \ + --hash=sha256:7b25c335fc53acf29d4d21dbc19fe39d2824201cdda0448623152cc5917bd259 \ + --hash=sha256:7d0b4fec6a8d02d7e95de5cfa913261820f1ce04bd4c0381924de0da523179b8 \ + --hash=sha256:7de54b49e6da811b0321e412d14efdaa1ee0c0b6609296ea5b9022bc5b2bd843 \ + --hash=sha256:7e0bfa161df365ba3c88899ee3b7c94755200967284bdedef8c1b8b43e2c0f2b \ + --hash=sha256:7fac4250b37d994e3fe42b46ba3c8bfa1614d1d7d8cf1cf23f303099082a9565 \ + --hash=sha256:7ff8dd079e7b5f3438332499233a2a5acfca0741fd0eb3d4ddba0c2d9bc04d19 \ + --hash=sha256:8090c35199d6b7bc6426bb8bdaf341e64f295cc2624a1fda7860c0837f1acc03 \ + --hash=sha256:81a0e08c64dfdad27dab687b96f572b23bafa1999a39d1b6f70b3ddbb73e8bd0 \ + --hash=sha256:852c921217f330b3e81a822647ebadeae7e42cf503ec1992d0bfbc90121c09fb \ + --hash=sha256:85cb3ced4fa84949cee12bfe78208b6ece7baf3cbd242b26dcaf773efff8d206 \ + --hash=sha256:86bc779a0896e59e4be30a5be5cd6eeffd0b40b6f0e75e730218736b7bfc6f5c \ + --hash=sha256:88811f890db240a1c82bf0bcd52973763707a552c8113ac3fcebca183afb2fa8 \ + --hash=sha256:88ec4d16e9f58071c9896ea01c4da97cce9d01418fe844ff06eebb00e0a1386a \ + --hash=sha256:8a844b8b1685f38a2e8b2f3213b286e2a7abfe67508381780a0d4599ac337c1c \ + --hash=sha256:8b8429361241da973e594d15344a0989f44fd288ea58d33a6221fb7cc0daf27e \ + --hash=sha256:9161eb81b8062da824426d3700d4b0d287f0cb0b05923713adfe3bd25e7937ac \ + --hash=sha256:9267bf8261a779abb2a6eab5f107f5db85b2d1745f2494081c731aaf28738ce3 \ + --hash=sha256:939d8cd2d8c35e3956f6bc858390b6ccb611e6152b4920d64ab5e98f3fcf39e4 \ + --hash=sha256:943a9bce22180ad0f4d32d1b402a0949a4ecfe5a1257b47f54a1b51981d81b86 \ + --hash=sha256:966ae0588ac9959a040220063733b33f321d04eaf4e60349b42cd855d232202f \ + --hash=sha256:98beff85392ce435b28a0971ec21cade61ce8be8b632c9d855475a28ef92d31a \ + --hash=sha256:99cf27791129d37e191ff013bfc29bf6631c29edb21680c00978567b91fc5d6b \ + --hash=sha256:9a8c826caeb7c08264e0a556df1267531c6ed90cc70506e7e5f4119e2d09f3d7 \ + --hash=sha256:9b24e1f93b9b586ec03bc7bea1bf021ec90bf2528c729195028a3ca1c266b3f9 \ + --hash=sha256:9bc5e7f843d14a167cdc26fe2d22f6f3aa2feb57919cf3ff034262a57d8d95d0 \ + --hash=sha256:9c10791e9f5ef132effc8fdce2009482c1cfb26618c5fc1b7952a47dd5eb632e \ + --hash=sha256:a177a0ee5cf19931dcaeb3f662bc562754cfa4f4ace2351d9da24a954ef7db94 \ + --hash=sha256:a2e575129c048bc286d696ed8e49ca148591768b2d77debcc6569f6fb64d0668 \ + --hash=sha256:a5f0bebb10aae010d3c9ee3abaf83ab2069c718457aea09c15532355dd7e061f \ + --hash=sha256:a5f721a2437390ab69c10c6df5c142478d399af8dfb02e6d823cf2358e8a4748 \ + --hash=sha256:a60b720c329c0007feae692b7bf91cf17b3f9bd3727be96cc6f9a3336651041b \ + --hash=sha256:a8bba9d1f6db4ef2a6ebfc937a65d36e80e3aada00b382eaf56fea8f639322d5 \ + --hash=sha256:abeec7a89d698aa1c9b4c36bd5e3c746faef0867076e6a2ca27fa5077c4ece26 \ + --hash=sha256:aeba2c750102051aa51e087c2ccbc79f2724a41c94168f8731e36f54c453551a \ + --hash=sha256:afe36ca503c2ffe30fb6df82b20389fa3c4035b5d65888a61310921cf3ae91c5 \ + --hash=sha256:b0e0040b0d8dd89bd0af9ab18901981e344ffba68bb30b8eabb4eab6c303279b \ + --hash=sha256:b117ed1cd1a23df0902461c38093408b95971833dcee629112acda25b603c8d0 \ + --hash=sha256:b4674b12701c3fcbdf7f88b9e4479701c93bec5da9eb576140d5fcc0092990af \ + --hash=sha256:b4908e17867930b7ac77f89a18dc67308c67c511f037d8580489be86fb585912 \ + --hash=sha256:b57d4d7021bfd159db9f8f6f862a85a7a6027934643c512f028d6e5c60c4cbd2 \ + --hash=sha256:b5ed78742502b8d90ff2816688d407a097c8b5cc6af4343fc5ad7a98df53a7cd \ + --hash=sha256:b78de22bae456a976f33df34d598dfd16edc9a03df8f4cc8b7c17bdba4c97b4a \ + --hash=sha256:b7cc5333fcbfb27327d12612ed72322f221b61c2b69deb1155078c964f86e1a1 \ + --hash=sha256:b9d9b7d72975521434368fe8aed3f6b522060bf271adabaa5ca6c87c0c08e168 \ + --hash=sha256:bbcae7a54050b7ad7bc7bf425ba63dea7d2cd31a92246ba787a2ce69a9b98dbc \ + --hash=sha256:bf14cfcc30b097583d698a6e2b8b68c9bcffab277c485d481881958360c2938d \ + --hash=sha256:c39dfcaa0bf23443474c0cb58d8d8aea9529c1841d99654cb38e4dada7b1948a \ + --hash=sha256:c44ca6d3cdf4cfcbcd4f928fdcbe87af5fd7319f6ad4169617b7fd6b4527c33c \ + --hash=sha256:c44ced5e5168cdf677f0ae39900863bf2bda7d14a5e13502014005cfe040b8b4 \ + --hash=sha256:c45629c0049fbdef932dbe408ac2b271fdc8c7d9962ca31160f4a0fc3455fe4f \ + --hash=sha256:c53be0dd676484a660acc56e4f1cd0dd74bc1255d12fa285e86a3fa9d5f22bf9 \ + --hash=sha256:c54ae1b89e582aa25f213cd8b5eac0bda1724e79299f486baeb3f562bbf82ca5 \ + --hash=sha256:c564d0758748f38aec56a6b98c6801a427b3a63f39b7cac538b2b2d18ca32740 \ + --hash=sha256:c5e4a362a95b85301d262ef6bed06cc8e4a144ac7e2be874cb4c3c46ae89d754 \ + --hash=sha256:c6b67f08014bfc4aedc22cf6a21010c2530cd5fbeb655730406827fe196296be \ + --hash=sha256:c7aafa4dd2f702ee2198005d6cba4309c1e25ed1c201d77beddefa47411bead8 \ + --hash=sha256:c81062e947f4b5a624135a843f6ac4b3c7fe6508300c9fb27347f022ba0c513d \ + --hash=sha256:cbec738d2ad551c6f70955d7eec95e339380ee1564e2afe86bfee05fed52ceec \ + --hash=sha256:ccf98ee859fe29f874ddd8e637f14ba59108a333492b521acb885a9095244a9c \ + --hash=sha256:ccfb950359a80de0fcd2030ad60ac1b1a861462de3e2ef746697c9256659af21 \ + --hash=sha256:cf606cfe3f67984b4064ac605d71e1eba12515fbabf5bd5a34a8952b8800dc66 \ + --hash=sha256:d02cd23b5af182a49d635ee72be38053767711987a9fd82625b16b93828a0d8c \ + --hash=sha256:d1b1b32f3c32f734dde8f36ac1df8e275e768a7b333241cd637cb2538628a4b4 \ + --hash=sha256:d7dd46a8fcd7653c09ebe67eae9d4cb6636c7a905d9cbaf587dabcbd4eca6013 \ + --hash=sha256:d9ef29cfd98e17085b4f91bba8fa1570bec6787d5c52ce653ed33a58785585d0 \ + --hash=sha256:db77888081431aaa69f3fd3480891746ddce6c2a571f6201869a24e2f06cf423 \ + --hash=sha256:dd8a6b3e8f9edb07fe671b02d8c3241c8b641fecce7eb1e36432db3e55e243da \ + --hash=sha256:df03e392cae1e05462918abbae06d6100f1e53f67db971ff0ac6c07d9edf7321 \ + --hash=sha256:e0d91a4bcb59ac0d7af0d8e0da737332e1b7fe6831e53e47819b1b5349d431b2 \ + --hash=sha256:e2e718fa9d1d900decbc240a533d5d0baf0947ef464c78a8cd4fa32b4e8f590c \ + --hash=sha256:e4ef15d0a29fc2da67fe8ba2301ecabd6f8733696cc2bf0a0cf96a144a20328c \ + --hash=sha256:e50f7775b66c7802f4cb697e986c5acf30ec07301efee95b396c08114e890d67 \ + --hash=sha256:e6906aa4bc62cde2c8aeb8a99a7b4401b241e274ae7b11df67d863d61ab3d5de \ + --hash=sha256:e96d67914ddbf5466e4476a1cd7ff30a332cbab85ed895207acc3e58c979b6a7 \ + --hash=sha256:ea027bdeca1d7e498237634ee4e3a852e2723eef39996dec0ff0f77dff8a2336 \ + --hash=sha256:eb0228c809b2e7eb47921876050af0bc4214b351bad8d8112f70b6ed4288763c \ + --hash=sha256:ecc68f5e47bc6f6f889bbed5bc657b22bb2237ad9ccab8229cb5a0d64f4cb536 \ + --hash=sha256:ed6b7f402f3dabd1d72c798b96cf947005ddd796a5bea7b041bccbd517859a42 \ + --hash=sha256:f16ac8af2804855d3cae5fc3c5ab609c9fd0fc8ecacd92579c05ed3c173396fd \ + --hash=sha256:f376224572d1f5da1c871f969ab04765727f180e70d012d93e07bfc08442c64b \ + --hash=sha256:f76ceb623f7ff50df46ac57e1587c479d87a5766319c4f43d0c0a5158896afab \ + --hash=sha256:f79def86aee67b5ba01b2565f1610f262bf88ae53c379f93e5fa29c50fe793be \ + --hash=sha256:f8e95c95039eab6a2dad8c83c38ab87fc5431d28849e0c8a7e2a4e70ba38710d \ + --hash=sha256:f979a077d1c0a9a36dd4fab0d3a36b8de7b593bf935e13df85a380395b2c11ad \ + --hash=sha256:f996b19ac89e0dae65821ce65f788619e4286f78c62d005ecd3b75b5d9c0892b \ + --hash=sha256:fab380fcff8b3555eb2bd04304fa4330909a771a9a9b0dc07666cfc23148a711 \ + --hash=sha256:fabfdd4cf97db033196b51af46b8a681d4785c2a66347f2a5af1b4bbb1182629 \ + --hash=sha256:fadcc96cd6155f35e6d85845fa4fcd37b35885dc8fda77b9f851cdfa538194c1 \ + --hash=sha256:fed6b7705d49dd07e5e0dd5f5c873fc44047e92d714299b13245b5fecac49d01 \ + --hash=sha256:ff15531a376dc6f35984443fd1429e4b150c36ce27633e7cc52a9e5318546e20 + # via + # aiohttp + # yarl +multiprocess==0.70.19 \ + --hash=sha256:02e5c35d7d6cd2bdc89c1858867f7bde4012837411023a4696c148c1bdd7c80e \ + --hash=sha256:0d4b4397ed669d371c81dcd1ef33fd384a44d6c3de1bd0ca7ac06d837720d3c5 \ + --hash=sha256:1bbf1b69af1cf64cd05f65337d9215b88079ec819cd0ea7bac4dab84e162efe7 \ + --hash=sha256:1c3dce098845a0db43b32a0b76a228ca059a668071cfeaa0f40c36c0b1585d45 \ + --hash=sha256:3a56c0e85dd5025161bac5ce138dcac1e49174c7d8e74596537e729fd5c53c28 \ + --hash=sha256:5be9ec7f0c1c49a4f4a6fd20d5dda4aeabc2d39a50f4ad53720f1cd02b3a7c2e \ + --hash=sha256:79576c02d1207ec405b00cabf2c643c36070800cca433860e14539df7818b2aa \ + --hash=sha256:8d5eb4ec5017ba2fab4e34a747c6d2c2b6fecfe9e7236e77988db91580ada952 \ + --hash=sha256:928851ae7973aea4ce0eaf330bbdafb2e01398a91518d5c8818802845564f45c \ + --hash=sha256:952021e0e6c55a4a9fe4cd787895b86e239a40e76802a789d6305398d3975897 \ + --hash=sha256:97404393419dcb2a8385910864eedf47a3cadf82c66345b44f036420eb0b5d87 \ + --hash=sha256:c6b6d78d43a03b68014ca1f0b7937d965393a670c5de7c29026beb2258f2f896 \ + --hash=sha256:d6db91ca6391eebc139c352f34578cea382df6bfa03d3b4146ed12b18b01cc14 \ + --hash=sha256:e5e7dc3e3e1732e88c07aaec17eeb9917f9ed1107d9e60d5ab985cdc14bac43a \ + --hash=sha256:e6c0674d34b8adac22533f6786576b3de4e396aaeda9e0c15378af9b8ada2702 \ + --hash=sha256:e8cc7fbdff15c0613f0a1f1f8744bef961b0a164c0ca29bdff53e9d2d93c5e5f + # via datasets +numpy==2.4.6 \ + --hash=sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1 \ + --hash=sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4 \ + --hash=sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f \ + --hash=sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079 \ + --hash=sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096 \ + --hash=sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47 \ + --hash=sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66 \ + --hash=sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d \ + --hash=sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1 \ + --hash=sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e \ + --hash=sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147 \ + --hash=sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd \ + --hash=sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75 \ + --hash=sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063 \ + --hash=sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73 \ + --hash=sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab \ + --hash=sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4 \ + --hash=sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41 \ + --hash=sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402 \ + --hash=sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698 \ + --hash=sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7 \ + --hash=sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8 \ + --hash=sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b \ + --hash=sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8 \ + --hash=sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0 \ + --hash=sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662 \ + --hash=sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91 \ + --hash=sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0 \ + --hash=sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f \ + --hash=sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3 \ + --hash=sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f \ + --hash=sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67 \ + --hash=sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6 \ + --hash=sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997 \ + --hash=sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b \ + --hash=sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e \ + --hash=sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538 \ + --hash=sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627 \ + --hash=sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93 \ + --hash=sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02 \ + --hash=sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853 \ + --hash=sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c \ + --hash=sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43 \ + --hash=sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd \ + --hash=sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8 \ + --hash=sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089 \ + --hash=sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778 \ + --hash=sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1 \ + --hash=sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb \ + --hash=sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261 \ + --hash=sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb \ + --hash=sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a \ + --hash=sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8 \ + --hash=sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359 \ + --hash=sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5 \ + --hash=sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7 \ + --hash=sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751 \ + --hash=sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8 \ + --hash=sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605 \ + --hash=sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e \ + --hash=sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45 \ + --hash=sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2 \ + --hash=sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895 \ + --hash=sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe \ + --hash=sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb \ + --hash=sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a \ + --hash=sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577 \ + --hash=sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d \ + --hash=sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a \ + --hash=sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda \ + --hash=sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6 \ + --hash=sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20 + # via + # datasets + # mlx-lm + # pandas + # transformers +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via + # datasets + # huggingface-hub + # transformers +pandas==3.0.6 \ + --hash=sha256:0704044b676496b8350e023b09f174a26772456c974a2b11c36bebb558c9490d \ + --hash=sha256:085e3786ae6b2e82b406266bce36690f72b9dc1421903ba9296b2981a9fcf586 \ + --hash=sha256:097090508a1dd335013d39106fc10b20f4fd4a171638e47b77d55798ed9dab6c \ + --hash=sha256:1bcb3e9ed29e74a7439cedff9e2aefd3ea65de84d7de9ccb6c194192541bd60e \ + --hash=sha256:1e7c0afdcaf6661d795fcefc2f647ddd1136f62cdc153fba177c685d97a87808 \ + --hash=sha256:1e92d9fa834c7d877130027cddc0cad8dcff97c1f6cca26bd6310f847228b658 \ + --hash=sha256:22172a92e7ee678ec0140c7af4fc9366b55413834a1cd86af78b3caa0b0574de \ + --hash=sha256:253e12cb9081b0afbac607920f6142975966bc315135e09de275fdbaa415d2de \ + --hash=sha256:265f562fdd1079f69f3de96dd425c3405224038c0af4f920c54bd240ee2c4640 \ + --hash=sha256:2a8fc94be2ee5f1d86f97aacd8cc566f81680b6498e76f3007421bb5d98151bf \ + --hash=sha256:2e5fa32ff162dfdbc280157d664f44d23049ae414725af9676df339c501d82cd \ + --hash=sha256:3ef908d28590b3f42d7070e7ad8f9b34b442b260b7f3c1afb57e0040c58cdb1b \ + --hash=sha256:429d9df32731ab01383ed98f2baa7a60368090d1a94fc06019a12062510e8630 \ + --hash=sha256:47121f9571503f724c9b93e297ab6254ac99c77adf5e9ed085ea419fd585c258 \ + --hash=sha256:4e25e2e1adee99ddfada6f7206a79ae8e9c8a8861b0e3eaaba165006d3eef18e \ + --hash=sha256:4ff44b2cb51cbd691c91f92c4ea6c71e34003f239ebd67c2e857dc898466b49c \ + --hash=sha256:50c44cbf5820b6b91a5f74aae04972472aefadd3cd9fbd1010409d85528bd570 \ + --hash=sha256:569e114072b24fc4970c12e2b4bab252671668a40b324318903380cab0254c0c \ + --hash=sha256:583be68728a31d0d750d5b8d9e00f02b153df0d4655f858bde93cb84cfc4227c \ + --hash=sha256:5e75072773c1b2f7cb63faa3a6f562aede11f3976f68ed34cb538bc091a28171 \ + --hash=sha256:5edd0a7abb0986ecce1ac81f56d99b6763f86aa6946dceb6c661224f90af5a19 \ + --hash=sha256:60d81f9e1799b36f3739e7fff44d1fbb2e8fd5a271b3863e03de9715fccda0fa \ + --hash=sha256:62f51d7f651c8054c5e82a69265c98082e795d1442df7ca6edc3a545d61214b1 \ + --hash=sha256:654aae059295dbba6ecd2328ca12712a2cf1676214c8699f1c29213f7ccf9c34 \ + --hash=sha256:66b07ef7315a31bfe1089cd3d71a7de781c9dca986762d0b4fe7c0ef17465d10 \ + --hash=sha256:6ff482fa91fa2bafd92e8fe66ce3645c851824310f295c1f0a2f96e928fc4541 \ + --hash=sha256:77ccbe5057aece6fc172b9b77f19c04335af6882bc2e10c8f3ee4e6bfb3da553 \ + --hash=sha256:7dac2d65e9087e8e7b5a45fe15c4920911a221df061ab629943ce016489145c7 \ + --hash=sha256:83e91d15738d7783c050197cef2f2cf82fc6353dae9865aa87ed1fa16aa4d55a \ + --hash=sha256:86fa853a12e0b70927e2b1ee00d56d2224ec9cbb4b9d58348b5ad52d2f21150e \ + --hash=sha256:8fe77b408d82e2615674dfed62533b95e18a03610573877422aada4f625d4947 \ + --hash=sha256:963ca21199097a84c7827c4678b04e30833084fbf8ef44fde3fa7180a29f8fa0 \ + --hash=sha256:97274c9adf6255bb48c620cd6959805efa7f09ea2167f0e0ae006a448cd2fca7 \ + --hash=sha256:994a79608263fe1c14cc48ffa7300e2b834b7d1cb406ffe96a08828cb0cdd79b \ + --hash=sha256:9ae8073aed8e21d1a7fe263dcdc6840743549722a6738198a0a46000fa9476f2 \ + --hash=sha256:9dab635a549e58a053c7b0fa054dc0bd7be22f0ed9a720f4a85d5fb993276172 \ + --hash=sha256:9e492cd4bdba6778de4fe0df7f4590c012161ebcf9902dce01b01dc683105514 \ + --hash=sha256:a3a22e07fe75347eaacc75b0e85297947af4fba6b4aae23916bd8b6828d0bba3 \ + --hash=sha256:a4dbd4dc65cbe645b92b8785d0f96dd7311010dc6606cf620e51b07b8788a12a \ + --hash=sha256:a77a1a44e4d88f1c6a2a64d3eb12efec8420875722e14279800b173a7c7c2804 \ + --hash=sha256:b27c8d890e4aa2171437ae2a39de1d215e674158e4865c4023a8b31c932513b2 \ + --hash=sha256:bd75ed0c840f709fc2ae26ddd9534ac77ca1a48ac0cce521a74acaa85f3340a7 \ + --hash=sha256:c6e4aae3e9bea26c6c9a20d88d96c86ec4a99b4db5fd516bcb4e829ab2c0ee36 \ + --hash=sha256:c826e9babb7790142c399f58599d8de679bea059d7b39c5b6efa2096fac37266 \ + --hash=sha256:cc39303913e2ea129915670de5d1c9fbd647f543bb72e5543bac8baa94e9e42f \ + --hash=sha256:d7564d86a94c2eb8ab290b07f63ddaae5c032fa53897c29a2ff2197d43aee8af \ + --hash=sha256:d7dcd21238cbb4828ff148481ba01cac8946dc5121457b5aeba28636f8f99a60 \ + --hash=sha256:db7ec631f26223beee8e5c9e0b8f23c24d8197bbd1d982421d4e3188bea51965 \ + --hash=sha256:e3dccb584123b399c07562ac4d62543e90ede49ddf8ce3c13ffc64cbe828c281 \ + --hash=sha256:e7c1905ef02c3d6d43d9dbd5b6ccb4da4870a0b0c821bbc103fbdb6f3ad2707b \ + --hash=sha256:eb6900de08ac85f93ac4948aa6b80842eba555875337b8359035ac9c43e92d34 \ + --hash=sha256:ee913a91669056c1de1a6b733fbfeab711de9e54e3bee2dfa5fe79d9457247d1 \ + --hash=sha256:ef738d71d1059245b6bb03e312be06d8b3821326a83486c1ad03b9aba3710e44 \ + --hash=sha256:f3ce8a6968045481e91a3990e797e348ce13db45ee164a7095bbc824e26c09dd \ + --hash=sha256:f4e7c52eb108d752e7592268108fd3e98efd76d83a3125cdd06c621c2e44359b \ + --hash=sha256:f8029ec0f1f89e4f985929ce1f6626dabf3140d61a4e9c1215afdab34eaf9a5d \ + --hash=sha256:fb625f426b375bcc96e3a04c5d5d266cd7be6ae5d6866e0e703382ab5164068c \ + --hash=sha256:ff51a4459ed036e93d1eb1bb5e6e7b28685d3cb6b7c12b91c05b31024e234729 + # via datasets +propcache==0.5.4 \ + --hash=sha256:004e685b315646c410771836e72a44f143bbe624f29653a42687815069a303d5 \ + --hash=sha256:02c0a34f16889cf800f10f0247a564d8ce6eeab6ffcd7c87198f769067eb8432 \ + --hash=sha256:03969626faf0783a592dfa17e28eac06018bd0b44dafae6943d53b92421a7f72 \ + --hash=sha256:03b229037d25b801e7af53fd52b9fc49d9439b036fca1e087e02780631adfa97 \ + --hash=sha256:0951315a6b3142ee2167404d707743f0157c110091342b1aa0accac5cf0e4acf \ + --hash=sha256:0a095db8e15a6020db149ecbed6461939fe74f6acaa3ae8b702a1fe8c38cd983 \ + --hash=sha256:0c889f6fa84957bc7e8b4eab71fd16a0455068d5045e3aa40c733071d2b2fd77 \ + --hash=sha256:0d21d0d2c82bbfeb1677a9711f38df968f9837576102bb4add1bd449d28d88f1 \ + --hash=sha256:10ef33a68a61ce317e095fd2e202a592ea92392b90944a78c993f0d9a73ab06c \ + --hash=sha256:12682126712ddc19b70ff819debbd279e58adf1f0c8f8f8138c18ade2044b284 \ + --hash=sha256:135036c5cfc93864affb0f9af9a27e5d7a71cb7bd745e7b6dbfc2d56cc30e827 \ + --hash=sha256:13e52b6e0bde97dee98ab66552dbff2931649c96f1ac432eac299fe689ec373b \ + --hash=sha256:141fdbd73748db0cf7636035030aaac383d2efde8f34e7bc24594cc776d225b8 \ + --hash=sha256:146f48a9e4812611a7581003b1a39de56c34967046310c4171a68ef908c9a745 \ + --hash=sha256:174507f82d3594622acb1dd2dafecf2d899d6d506335494e7107767bf05f3aae \ + --hash=sha256:1783582065a1f07f9d9ee1e992e13f15d7dc8fb1eb3a7476d43eb3f2e69d26bb \ + --hash=sha256:17a7400cec0256f0a71ae71f9da398f9894c956ff6668a1c9d317b3367316320 \ + --hash=sha256:1b2f3bec4261a94019575481c726c29850f72e27907773c75b1de421e20e9f9d \ + --hash=sha256:1d759d05634f1b038fb625a66662a8c85e5a8fec912da381b5149ddac107482b \ + --hash=sha256:1df8d8561b21465c5dd56110a01caf897e026d065b4b84e98a488209094272ec \ + --hash=sha256:213bb68d9ced5cf2bf717b1071bf2b09b4b04c426256f9fe6d054c60318424c4 \ + --hash=sha256:23278f808cd81d5ada7184a76606b925fb3389c60e1077b2cd7da7b1fcf0553c \ + --hash=sha256:251c63dd46a0659bb875cb254dc4c1e79ee91a847c737cd62373295afc2235dc \ + --hash=sha256:279655a16973f1ee2bd2fe79973137681642fd9ae0d89215bba263726eb0dc3a \ + --hash=sha256:2814ecd8e818f487bee4b0f921bc4d1c176cc5fc71ac0f072d0fa67eda4ac14b \ + --hash=sha256:286867fb156488c251a3721766e380ac4495e4fd6b51aaa1403d89ce7f4359d9 \ + --hash=sha256:2dba2f02d2d5c09ef8a0e6c1a42aeaa451f4be9898cb00b04fe98717da2eb23b \ + --hash=sha256:30cc1cebaf9aef49db06357a50398323ae04d70460c0491837d026ab7d6452ea \ + --hash=sha256:31eb43ba2edc704ab2ec27815315dd8a19def0fb16215be4cfe8d32fe78ffd51 \ + --hash=sha256:350b272b2279f4135a64fc0c304a5d08e28a137c9573442c606152446638a831 \ + --hash=sha256:36c0d9db44b523ef93d03341b1c42d69ff01d673c053d1b1c6c3a363bcaa39ba \ + --hash=sha256:3af0c8642b2da4815d86e631232ac8286e17644fad907c19508aa8e7cb4ba8ad \ + --hash=sha256:3cd3a7edb6b95b9b33998135ebfa18d709da82290fb8f27c858970b5a12c8b56 \ + --hash=sha256:3d605bb239b796e82a81c6709548b2bd460ab73b4590cb0c83de8a2dd9694d0f \ + --hash=sha256:3e413d7a4a9b4866b7a761d6060d434b64d23cd35122eda3b026a0bbe8196b25 \ + --hash=sha256:3eb2e820e8e2101407da93f17c57cbb7d225461955fc60105daaba14cd421ee2 \ + --hash=sha256:3fa15757fea1dfcd5b7745cad9f4638929605531bd4018ab2adff7955f1a403d \ + --hash=sha256:3fc24f209c1b7f7f688b66b98293954f5504279760999b58920ee12dd8471c1d \ + --hash=sha256:4054acf80d40456a0537f2913b349718649d8d6458a14ab7f48d0ce28c30869d \ + --hash=sha256:40e94adb1e7d39ff28a8bd8d8b8fbd1df6b9f40976dbe379134f1ce058e532dd \ + --hash=sha256:420162a77f94eb1cf5ef7893f500016dabd548e73de956785a1dd899cc73006a \ + --hash=sha256:425f8cc86ab5018b4b8d4a23bc8e74d964bd3d757c3702e301aa79be76c53f6c \ + --hash=sha256:44149f46500a0a41b95b4d99c2e586a77319539730607b9892974a092788b111 \ + --hash=sha256:445ee3bfb46e85838387fb3c536a73cc0b994dc192b004e40e170adc54aa2a7e \ + --hash=sha256:45488d1a5f9ab5bd90aaa1ca20f50fe1922b8ffad71a2009d2adf41355897aac \ + --hash=sha256:45bebbe252550fec975ba3b62bc6f931643cfd3b5464ef47619cf3fef154e01c \ + --hash=sha256:45bf2e730ab8905d0527fe05a86500f406e64305c34cc81ebe64b4617cab9760 \ + --hash=sha256:48cb48c5346a97de792254af77715aa2529c2a1ebc5f586aa0aae44a02f1fe57 \ + --hash=sha256:4a1f4f5ffa55dce6307631f3cb2948e117e665966ea512e0d502b16c24f567e7 \ + --hash=sha256:4cfe0a92ae30151869e67a4b5f5e105e4e03ad30b3f38e5211b5bf77d0881993 \ + --hash=sha256:4d86476a935c88963d9b8e1a9a0d38188790e9622169bfbafa173046846709d3 \ + --hash=sha256:4e985382be6d15da8d0c2710a6fa7b9070fc9ecdeefb7f580e88373984ec8be3 \ + --hash=sha256:4f2d880ff60f45898f4acfa152aac8d04e3ee627d90ff4003491bf92239d5757 \ + --hash=sha256:4fbc1a15dc8cd1689508758d626b372b1f09d28d9577667feaf9e6bfcd8efcbc \ + --hash=sha256:50e337653721d20ead710da33bf44487fbe8a0db8782714b60306481e9f95b51 \ + --hash=sha256:53eaa697c4d0422ff4cb714d00231b43352064d97b944033b30c1d57cc506ec0 \ + --hash=sha256:56fc3f7599528db40b1efa0889a620116e2704144495273d66066e8164e45838 \ + --hash=sha256:58134228927cee6c047d626c08e60a81be604a20578a12ce752cc5c9a84d4826 \ + --hash=sha256:594eb4c6ec35e7179b058481f4e9f02521b56de16fa577c4b85c76fb1bf8a9f8 \ + --hash=sha256:5cacf3c9efd09df409dc33654dd077e1c245ba8fb747b0f0236ef41b7c49b589 \ + --hash=sha256:60a64cbccaa11b7760ce705a14ada17ba459e7ca9f23ba587eb013821032d7ef \ + --hash=sha256:62530ca89187827e4a4fe733f971abe81a7542eeea48ff61995f19b64d7199c8 \ + --hash=sha256:62c60aec739ed00124573cce1178138fd690c7676352d67a37328c1cf51d7468 \ + --hash=sha256:69fc35c0779522da366c563e5faf203ffc1f8ff0021d5b1337fa4efa5be73177 \ + --hash=sha256:6af4693716bfb03f1752ef1b30faa593db2c01d5272e9b8564a1549452a979ab \ + --hash=sha256:6c7599df2b57ebeea8de011b5f2f7b85de95e76037d43d34b95e328430275487 \ + --hash=sha256:6e9368e87a3efc285e559131092c5db643eb8e56de4ee42064d5baec22ef2bb5 \ + --hash=sha256:6f0093ac3e9daada202c2082439d414a625c57184727a46e112a3fb2a81cb788 \ + --hash=sha256:7177c43eddf10a0893c4fec52ebb408fdcd7f7d63962caace9180d8f81b14ece \ + --hash=sha256:720cf832eb2d0b0dfee129cb3335a26f6ce3cc45ee1187e8f0731758caa16792 \ + --hash=sha256:770e8209d018175fc0063936fa9583b6d27e88c5ad31543f3383d66080efdd62 \ + --hash=sha256:7a8d5ff04eb1f85698a78d20c62a14676e7b960dcafde09a388d60ad377d355d \ + --hash=sha256:7b9100a93b372418d8688f3f2a3e5b45c64d70ca4d6176e121aca1e3bfc1e32f \ + --hash=sha256:7cc528e760a8af06f2b13e9b9f362cd90c7c718ea61228a96dbd31ba16ed7f47 \ + --hash=sha256:7ffafcbfc7b549ab940047e505c831eabac5e67de53e1bc174adbc5285c55944 \ + --hash=sha256:87a3caecf8095e48dc72f84bfa42e23a848cf410cc9cc13031fba4869b706a21 \ + --hash=sha256:886b59c4d28ca97dd23b025fdfc50a0356be934efbbbca89ad26230067f86fe5 \ + --hash=sha256:8876b39961e33d912afe3c1bee18ee564fdad0206f873cc15d522756b7f50737 \ + --hash=sha256:897d1ddf6716e8f47200f7aad9a0efa6cc7586df66c6defa572f9eab379c078e \ + --hash=sha256:8a1fc236528c457cd739c88abe823da851b7ab645d72792f88658114cc340c12 \ + --hash=sha256:8a235f73d6e020855dc29dff012d920c02ee0feab8d73a24185a7569f4be1161 \ + --hash=sha256:8f911c395cef73c510bac566da9507bb6a43e7763d0c79138dc60ee53f11207e \ + --hash=sha256:96f7c5c15656040ddcbc51e56dc59b58aa25999d743c126abd425b9766ab43e9 \ + --hash=sha256:978f28401afbc76cdc3df9e1717b4229a06b626a1dcc75db4e1f2beb3884c3e9 \ + --hash=sha256:98914de2c4d7f0f9f4a8c6ea4bf05841f4175796941e3ef7d47eb718f22311fb \ + --hash=sha256:9a2a8a50a93dee0268a860a07fa3b4bd968f8ce4dbd794957da772f395368526 \ + --hash=sha256:9cbfff4423eef4cc6cafc021469641a2b835f610b2647a6c5281903e21b8670d \ + --hash=sha256:9e9ab13760aa8b6d0881ae7cb04fd891d8d490cd2554ea8e79bb278399169bcc \ + --hash=sha256:9f3551b8a35c1df3e7ea4d2d86edee15f0dde1bddd434a71744048683544d0ef \ + --hash=sha256:9f86f7259efe2c951f43e57d471c9b41daa5bfc7db9f67189059cf1ae6d77fd9 \ + --hash=sha256:9fb0a5be8d9aa213150e8d8148a42aca4984b285bcad1e69587dc4298edd929b \ + --hash=sha256:a219f0ac59817a9114dd2aa57c13180f993e819ba658c7ddab4b66ed1ee0d370 \ + --hash=sha256:a419ee85e654927baabda3929c03c0cc1112bf472ff0dfd6142f4e3a81ca4162 \ + --hash=sha256:a4d7a54719b67338a305dca2ce6aafe366817df94ddfd4b5514374356f5ca546 \ + --hash=sha256:a5793c7698a53f56f4a1889a4737c7eeb1b7ad0842fa6b1abca22913ff79c8c1 \ + --hash=sha256:a5e8ef588c109725dc713ba69aadcac00a1ef90c2ce9c0a8c7075128f569f47f \ + --hash=sha256:a74bfa37147cc08fb29df10bd9c16f40fa7f860cd3a6d2fff853323a94f6e17f \ + --hash=sha256:ada748108a43d29b7c328ba7db3755327cd94f028bcc1a7ee3f0addcfacd9c38 \ + --hash=sha256:ae58f361bd5dae942717c65d3413b478c70aea9c462599e7b9adad3731db3894 \ + --hash=sha256:b28f41fa3b8c6900457f858ec5b03998f3a6d535fbc1bb2edec5961ea05ec429 \ + --hash=sha256:b3083bfe87f95c756e610bd8025f26cbd1cd4aaa03a422f2d65efb7a97cd53d8 \ + --hash=sha256:b61805357d966680acf68b3b6d49772631ed9df44ebece10ff1460e117a7da8a \ + --hash=sha256:b77c313314524ca9c38fbd70f73515d04597ac58c40c939bc0e71eeb4abff680 \ + --hash=sha256:bee7d3aed13d56f54e681df38c3a23031bc9e3863f687d9d598825c9146acd7d \ + --hash=sha256:c02c0e570c5c7e077b0181a9f3cdb7d4c3617d1cda6b5c95bd5d34022923d82c \ + --hash=sha256:c174bfd1c48a1b51a3078e95586dde718374bac79719ab3541ec9e74aec40574 \ + --hash=sha256:c2ba30a89035b57b73e00475de948521602f543d79ce01db10b04b36c4c76fc8 \ + --hash=sha256:c3e98c55bde2bcf7db3c70d1aed7ae9aa8aebbf19a250c66645cde44cdb8b867 \ + --hash=sha256:c3ef2818d63bc86071e9d2989ae75a1bc32b8f7059cfd9f5abbbee70c32e2ed6 \ + --hash=sha256:c83acbce9f2b5e3f5f5eda9e53d2001fed22fcdfef81274a9e02d8fd53b70a30 \ + --hash=sha256:c9281e922c072158c91974d4589f1dbe0fee6d467f284c28e463f9f5a4d933f4 \ + --hash=sha256:cc07876cfb079b6f6f36d21ce75784ad6c2c6b563eeac0ed26c2fa2669b85df9 \ + --hash=sha256:ccf4f7a79e26bb7efb06ecd50c177833b71df05cbc748701372325e6bcc17f6f \ + --hash=sha256:cdee8205a44d0be91bbac4c41b95d86641b72dfc7aef1279400e4fda3f26a937 \ + --hash=sha256:ceb3e879afac028f93d272c957814695dc5569e4904262dbee92f6c41bd5e4a3 \ + --hash=sha256:d1f5a500bfcbb2c0ab85e98a0dcd70f5899d34efe365a0187700369a79603031 \ + --hash=sha256:d42a9a856a4a6e2f6c10f1318c07e7daa498d6593abe745c71dae4521a26ca39 \ + --hash=sha256:d83b12902eb8bce151259c86c03ba746600b2d994543de46e370cecf96c452f2 \ + --hash=sha256:d8e017eeb7482bed34cdb0d61cf2bcfc88d104bbab296a17cd16a6af8aabc70e \ + --hash=sha256:db3ae52ccc150dbc84704e9d642743897f3e1c54742ff34cacb661e52e3818a9 \ + --hash=sha256:dbab5f5ff6897c81f355d079010cdae85b02e5a0b518b5251523b8ad8ae9ac3c \ + --hash=sha256:dc4242ca653c9b30ab51c5f8193323e7bc0928f897ee9103201e59a43abcb72e \ + --hash=sha256:dcbf346a318a5e30063f547630b02bb787ce2f45b6368d5da143660b6a3835d8 \ + --hash=sha256:dd2ac8f5b643454c2cc6b6118b13da16e88f4a6434fc3ba61aca384029f04f36 \ + --hash=sha256:e1d52a05dc417279f7e5c7618c5dfbbc29923aaf9bc0a5c1802ddcebf54c61a0 \ + --hash=sha256:e6720ba44ad7e72174314d0e1fb0172494cff5c73a3a8a2159c3d2402ff15565 \ + --hash=sha256:e738ab81179510ce79b2eac9a6ecf47feffd9e76d1c72e403005dddb6e36c06c \ + --hash=sha256:e904d4d01f36bd6e197590be1533c44e06058771e0746dd073a8ebb3ef880858 \ + --hash=sha256:e9f165403b81fea7e89c932d89046a1e3d9a3a60e8d7ef2f249dccdcb0982bf5 \ + --hash=sha256:ec6a85f424afa8d23e0d9a094e5dbb6eda01da91c92b9183cd433768247ffc97 \ + --hash=sha256:ee19113bce2f3acd46432050688b70f61acd6857d75abb9ec96341b7e9ced123 \ + --hash=sha256:ef3b928d9c984322b5c44e6964d8dbc653da87d2d8ee1647fa6da43072e650a9 \ + --hash=sha256:f273dcf7149a50527c4fd1f55cfe9eac0f60753f5af544b4c9352578e20c0874 \ + --hash=sha256:f5470694918830da62fac9e69133b53d23b736d7070e587b27a4a2be37e08e68 \ + --hash=sha256:f574e460d1c8a08384a016fdb09ccf3543433263ed6b2f97104f979e64ea57c2 \ + --hash=sha256:f85915e00dcb1cd9f2f890ead064ed40a27df06f0db65be427b29482ae357572 \ + --hash=sha256:fc2461ecc45f17893f8207e73b46ea8ba93e33630e51cf4af3fbc21d47462b1a \ + --hash=sha256:ff6b113f50bc066a698db5d944d2c6dc7507168dd3341e255a8892fd0715a558 + # via + # aiohttp + # yarl +protobuf==7.36.2 \ + --hash=sha256:497d0463ff3316681da6c0b9e8d06cb465d61abce00b613ab42226175644d1bb \ + --hash=sha256:89f23aa53c24553a2416fd4fd1ec06f74fa42b14b546d8883128813f775bbfd2 \ + --hash=sha256:912c1221170e16c08d1f086762f563dd61ff83c18b5fa6652952dfaded66f728 \ + --hash=sha256:a300819d441e078a5608c0d3c709796bb548136058fda017ae51d425b44fd353 \ + --hash=sha256:bdb3a345d48db958e6ce1f18e508beb0cc981d64f24088427549c866cd039f1e \ + --hash=sha256:cbc70b17ee27e28894c7fee8bb04be1abead49e936bc70eb60052531eee2079e \ + --hash=sha256:e11e1f0180583a2af89db6a2ecd9e8dc40aa6d2988ca175bfd0e6d12ea72d74e \ + --hash=sha256:f4fee11ec330d238b34a05c9b675f693c20415d1c5bd7d5320cc2f8a798eb9cf + # via mlx-lm +pyarrow==26.0.0 \ + --hash=sha256:01c863a18bd9c8412453dd0d92de6d0ee7b2b3d6fb079d9734a4b2a3c8bd4453 \ + --hash=sha256:0cccd36e00ea3afeb52ded61f2721ce71f604853d70c45365c58324eb773d6ae \ + --hash=sha256:106bb9290fc6fd9a84138a9440038ef184bac86463543c5ff099229cb30d996c \ + --hash=sha256:13b0972a3dc71b642050d1bc72664a3916e14f59c943d8c1368154d6e4b0c2d5 \ + --hash=sha256:210cc9b83888b87cdc8f793eebb264f22b20d0dedbedefc73b9687a7047b4747 \ + --hash=sha256:240bd18a7487f8767616a948a69dd4e740a8bc36a1c9da49e4dc9a32c5c2faed \ + --hash=sha256:24f892fdf1ae1942d69d3f7742e2f49960ec95277cfb1a70b8a1d91f4a96d935 \ + --hash=sha256:290a74c48e9491b436fd5edacfadf357943f82aa45c81110bd83a69aab33d1cf \ + --hash=sha256:2b5fcd69c0e1107b79e55839877db5a6ed04651b73fd6fec581d09e230bed5e4 \ + --hash=sha256:2e4a413046eba9896e632925066c74095182200ba32e19ff0166bf64d2f936ac \ + --hash=sha256:3a4d235876f14b4136b4d616ec42eb469ea0d6ead336cae631aa1dd29b21c962 \ + --hash=sha256:3de30a7432b48b98b9decbd9e25a53bb9251d202c2e6c5a29a50869592ccb117 \ + --hash=sha256:41dd3661ef40790a78870052ad7a58ad827b27c67a4511f06962eb9e9b74d19b \ + --hash=sha256:4a5fa8dc70dd50808990ff36faf44088e357b353d86c7682dd92d4b78d4c97d5 \ + --hash=sha256:4bcba83299cb2b8f8e443d36c6ba6269a5034431879015fb0719495df8a14de2 \ + --hash=sha256:515a10dae2a1d236bc9c9209d0317acb6746ea63cd4f98704904af7156d90ed1 \ + --hash=sha256:5780d487ff6c6ed7b42298609680d87fe0036e529a9dc2e1105364bce9697f50 \ + --hash=sha256:5b827650e874f1f9f9392524ea3e9e3e8a245de5ba64acca1f81ab188090afb9 \ + --hash=sha256:5d5768d03426abe6526d5274adefa00abf00a7f81118c46e98b5a46390f5549e \ + --hash=sha256:645917e976671debabf854abab6e2b75c571ca4f82adc33a2d338697f7c27d93 \ + --hash=sha256:68cd662e9e2b00876a131950cf32336ace2d0865e1f9418763e3d3be8481dfa4 \ + --hash=sha256:6a628922ba20705fa964ca73e4ef959c2fb2f14b9bbec5589a6a1e68e6257c85 \ + --hash=sha256:6e89dee53aaeb50505ed6152ea55bc7ddfd4f4df264f5427ea255288d8f0e580 \ + --hash=sha256:6e949744dcfc2d379808f7013c5f9cafaf0f817656dff7d46c6931528dd1784b \ + --hash=sha256:734312d3d99088d9ec28c5b17bad40389bd8373a1afc10acb60b83fd217af087 \ + --hash=sha256:7aa12ab8e236789b1ecd2d6ecaef036b4e63d675ddf1864a43c6799d18f2d028 \ + --hash=sha256:7c3fda041e7078802589cf257750323ee3d0cd1e56e53a9b20ec845697fb3d28 \ + --hash=sha256:879331ddea2a26479fa18fade71e6facf684a6cf19f67daec3775c871569e8e5 \ + --hash=sha256:8e8e28c464552b5ca03e30d4504168c4425ce383884f8611b00e972f9fd933fc \ + --hash=sha256:90ddaf7c625307ad52f31a9b25c34fe5e4897c7529ee3481135822b2b6842ff1 \ + --hash=sha256:954d971b363b16ee41f89389a4053315dc71265f2ce5c2468eb0a910b1166268 \ + --hash=sha256:9db18a9dc0af52135c9eac549d80a7a882696efbe5406cf882b044525d4ecc2e \ + --hash=sha256:a0e4e92eeb088f1d7c2c04d6c7de8434c75abb4b4ccf0bbcd045aa7164c68d93 \ + --hash=sha256:a6ca849f90cf73fe361f08a5762c783ead9671e4548c1f558cc637b54c9103f2 \ + --hash=sha256:ab6914db225d7f399652ae1f08588dfbc9efe617612715701e3d9d5cfa5ca19f \ + --hash=sha256:c2ba350957076b1b3a22f549261dc3e9c67ca20816d8bd5f79d7b9c69be4c4c2 \ + --hash=sha256:ca77c43ca55bfc9a4eeb1f0cd5f093f08731b77c24cdba0829035f084959b0bb \ + --hash=sha256:cc903e1069e9dd5e9dcf780324c0112e27e051e422ecfaff574fb33ed65d9160 \ + --hash=sha256:ce28748cbeb0f29c3ce9603782979c7117580fc76f16aa3ca448b38a22281adb \ + --hash=sha256:d58798c4d8d629700058e9afc1e16b9801023f3ce4dc1c92d945e79b5ffe4e98 \ + --hash=sha256:e2a1856e9565fe2679863b372478c681806aebbf7d0a6e72f33e77f804e647d6 \ + --hash=sha256:e3b190ba1d3d22a5a8758597f797111b77d433473744352a184a5ee0a42d672e \ + --hash=sha256:e890816e5ee89c74a0f8b9379fe8b5ba83f46132b2a0bbb9b1c21359ec30dfda \ + --hash=sha256:eaf9e7cc7ab59f6c760232bbde18f64d559bbc50544841303bfb32be53533297 \ + --hash=sha256:ee341973f78a0b46e073d065e88e75026a9c584051e97f98a0d05d96c6bac7dd \ + --hash=sha256:f1c1b4263fd13abbc339a16f2bf19f3a5cbf2a620853d812b1256f03c5342cb8 \ + --hash=sha256:f7444ea6975c49a857c68f9bd8fa11acae96dede63d120ffb3bf0a603ea82516 \ + --hash=sha256:f800e9e722c145ccd18012d82a864cb21bfee4ba4ceffde77100d25eced511a9 \ + --hash=sha256:fcdd1e04982637c6042337d3e24d472f938f01fdc502e2b994844b726d12c3f4 \ + --hash=sha256:ff1e816af7abff71f289242e109217036723ce36aca74ad6691e52d964a74afa + # via datasets +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c + # via rich +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + # via pandas +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via + # datasets + # huggingface-hub + # mlx-lm + # transformers +regex==2026.9.29 \ + --hash=sha256:01000ddf0e3ffef97f2413ceb514f6313040106b6d18a03ee00a4fe35c1eb1db \ + --hash=sha256:0166844493626c5015c6088ee15c9ca2fd060ca15b7641d1657da6a58432ae33 \ + --hash=sha256:044265d77d94f5e3cb2fd72c76723807c429cb8c533e9d4672d0334a6f14f588 \ + --hash=sha256:0476e5bcbe6e1ba3d1c4cc7bbb1c3ba78e3b979b5c8a88d0a6a8cdd4992b8c84 \ + --hash=sha256:066d0e3dbfdd739bce2bf8c2a41dd16f73e3d8adc2eb06dd803a36a307f56075 \ + --hash=sha256:0b65c72739f981377c9c22e0c5c3cd7f42da7bd8a3c9209330fac772c7d893ed \ + --hash=sha256:0c992c19cd45058a4b92f68f139c93db168b48fb1f322c9a7cd620806afb6b51 \ + --hash=sha256:0cc63b5e47c12a48d90c7e9d7de6a035dd14f62868aaedbb4e0ff8ba2b8bfe7b \ + --hash=sha256:0dd8af32e9f7b56b7f95cc1fd79b23054c3bdc172392ae560acc24d57b7ffe71 \ + --hash=sha256:0def9fb6abac55492d6d51cddb7225d07d6f279e774e0adc08569a54a5fc8d46 \ + --hash=sha256:0fd2c901cc307a745ad4bc87f20060d7a0825a3371d1e93488af22e7a387f78f \ + --hash=sha256:1043aedf5917caa861bcb25a9c11460049656bdf0017a90a309fa8f255467725 \ + --hash=sha256:121a76a0985db80ceae9e171c337f8c927868e37d01b54e3ce87bc87f9c6a208 \ + --hash=sha256:143533cc4b6fbc5b95aca0a5b8d541088d374831593def000ec89322c220221d \ + --hash=sha256:14e953ff3607c92d7675bf79c4d4509ef6782aa8c08509f179f9b3d6d0679e86 \ + --hash=sha256:18ae8eed4526e35bdb754d61562b90bf5c00a67fdcf3cc1380dd59597486631b \ + --hash=sha256:19959129885356df0e97556856f77eb2888380dac18bed075a7c05c5128c618d \ + --hash=sha256:1ba8c6a416569ce0d37e83e28a254a61dc99a419084dfb6476cea02d997f74fa \ + --hash=sha256:1c2a0026062abcc321a53db4a185ceba0b59a66b5d37b0808917a88b55a5257f \ + --hash=sha256:1d9fe8091b2e89d470df68a9331111ed008ae8aae6bf1e8e1fba4086a495c84e \ + --hash=sha256:2089fe39c406784d90101c726755ffa1497bb74638fd434300d2b88006186de8 \ + --hash=sha256:23ae6fdad9e63e54038f5ef78aba2933faca61e24d432786589e737bc5522ebb \ + --hash=sha256:26ec4ccce55aa533fbd603d08911b01101a8fcfec987845ac3ae2c7087b2bde3 \ + --hash=sha256:2f7f7aa47b229f2b39a2ae2596d2ad5625d77b5eb9856fac2dab3eb506cdd0a0 \ + --hash=sha256:31b003f9a070335e2a8233ee9b14a3ca8e6d792012ae011f741bf0aaf11744c5 \ + --hash=sha256:32ab11df9677ca80bcbb5fe4eb1da9109a5019239a054836efc6fa1c64e683cf \ + --hash=sha256:33026515aebc0e70d1c89978e53e8d695d35d9e472f8d5b34465ba3c74028650 \ + --hash=sha256:34b6925af9853bf461950e6508910f179fd6e9b1a7ec8548e069606b7e51a26b \ + --hash=sha256:352cf115a810b357caa35193ab656ecf5ef41056855e82f292c99e8514f8d954 \ + --hash=sha256:39ab5894d971f9ac68baa6eca5c50387db579cfcacf36ae8df3feceb1815e6d0 \ + --hash=sha256:3a21a9509d0ee88e7a70e1ad228cd2f0e0fd1e187458db132e8a8d18c97daf9d \ + --hash=sha256:3c5c2ef13797466aa64170cbb66ad98a32351dd4127694cea7199f80f213750d \ + --hash=sha256:3e778bfccd63075167709136afbc251c1f683758d5bf49c803c60ac3f894ce6b \ + --hash=sha256:3f1e6cb402a89457582cd696f982559217d13484a193202c394015297968c86d \ + --hash=sha256:42e82e578c904445d4c8a35b8f28052cf567593215fa5db06266fbc6f77aaa2e \ + --hash=sha256:4408b2b27a95ca8cc48b7411945753773353b5c93b307754781086c99d3a576f \ + --hash=sha256:446654b29bfaa30500d80947eda42cef1449dc8a87f4e3cf061cc8485d3a1f0b \ + --hash=sha256:45010bcfe66df41522d56c9b6114e87ecc597a08970ff6a2ced24415c141ae5f \ + --hash=sha256:49ee178ca31c94621294bf9b8b676a92a2e6bba8af0529591753719e57edb621 \ + --hash=sha256:4d7d93613b01b0199961330e49cfc52d479b3d5776c56c691db31130c0a07d91 \ + --hash=sha256:4fb41211d2333eb930a51e0546a65999761cf1f572a4da56ef9b8a62966c06f2 \ + --hash=sha256:4fe97894d1b306c919b4e50def1e6f6c522f4d03a7283811f4d108f1ce5d3ac2 \ + --hash=sha256:554bffadcbcb6d5f4e5fb10a61cc52084b9a63d1dab5f10bcd2c4343972e8e2c \ + --hash=sha256:59b49507f47479e299a9e1bc41b5cb83a7afda0540625f1dbae886615978acbf \ + --hash=sha256:5eeb8edc6110d9194a4d0d54610f64c37a31c605b5dbb7e407fc6ec7fa34a4a1 \ + --hash=sha256:612b709381c0355b70d89cdb51b7f670591ed5cbbc0e3b5337488019dc667b65 \ + --hash=sha256:61956f074ecd123f55adca68ee3eab46e6a07ad3f8e64e6db95dfacb444f55c4 \ + --hash=sha256:6398d5145689503412cc1748895242598d8846b8967b851133b20dc2ed1e21e8 \ + --hash=sha256:65b408d8fcb273e3499e7ef2ce796810da1becd208c7fb4373692a242d79d461 \ + --hash=sha256:686ac5350fceae63830bb98805fcb8039325bf4c06d9f6f048ff65229d5bffa5 \ + --hash=sha256:6a1a824fbed817e0a891103886b68f063b1e83cc51bc97192a90a60195a9291f \ + --hash=sha256:6abb75ab16bc3281714a5b99548a2225db70dba1f995f6d7f7419b76eb5a8fbe \ + --hash=sha256:6f7121a8914ed13fcfe2099f895341bfb789f004d4c5a0bdece8fa667da10849 \ + --hash=sha256:7020ed44df30b3aa492c00ee3b52d0548c1f30c2c6c5bb13ae897680900d3413 \ + --hash=sha256:720537c7ea6f80dc61913184edb0ce2497a306b39ef19f28505b322553d52bdb \ + --hash=sha256:724184b4aafed865e4f13ca313fdcb43024300c028ec67319cfa16847d84685e \ + --hash=sha256:7c03031610e3e6ed1768a2b7a8fc84637c1257b50c5eacaf094c6e17a84fc563 \ + --hash=sha256:80a5ea3b4fd9d6a5b9a44f7976a9acaaab35aa3c1f6b29e5bd857dfabaded223 \ + --hash=sha256:80c7cadd3fd2bfde5df8aa0787e315812cad0c313a753095d02f4c2b6c01677b \ + --hash=sha256:80ea96f5c1a30bf09007d48466521d9c294bebe197c708c3359096e3e3691632 \ + --hash=sha256:864e9b87ac33c3fb9fb4ad48166d4fdb579c351d5c77deb0d34bccb36a775cd9 \ + --hash=sha256:87fb80cbe3557e27e7b28b995c2b2eedf689b8886f941ab93e0e288f0976518a \ + --hash=sha256:8873c4a11c50b9989168881aeb3f08859f469d809941866aa1feefd8be5431f6 \ + --hash=sha256:888d60953908dcf761aa320c3e390ab8556efbdb551ace63921de90f6ae0848d \ + --hash=sha256:8b5fcc4771732191b2b7d1dd68d8f0353f47f8d90b6150f6dce58bf1112442cb \ + --hash=sha256:8f39588af4731c8923c26810eb3b33f76f17633985e40f59c3cd45a33805a895 \ + --hash=sha256:9173db3be74a35cb6731701094b98120f7ee4876a287882a59cdea1fa7da342f \ + --hash=sha256:92f05c9c42bde5785dc48770bc2194d9f7442544156f951e19cd31b096cec562 \ + --hash=sha256:951733b1bbdb71e377cec567b409f1a7881b47cfcad84121aa74cb575fa425ea \ + --hash=sha256:957bb708e8057ab1649ba566456429d691ec9b90d1c9ad1af1ba7ffbbeaf05f2 \ + --hash=sha256:9916fda742cd4eede63b286f58c06718324265d727ce0856eb1aac86d0d150d6 \ + --hash=sha256:9e1d3a4cb7993b708f0ada8d0c84590efd853f169e7147d2202c9da503180242 \ + --hash=sha256:9e4482589065c8ecd761cff522dcd85f2d39e62f551e37e025d1c7d54772def3 \ + --hash=sha256:a5300757f8a68f5b6cc33f57338d72a0e3589c5cc9ad5f8504ea06f028be582a \ + --hash=sha256:a540abfab208e1b7ef2df231c40ef3b6cbb30a0aad6204e9b6a81c10a6794628 \ + --hash=sha256:a5758353650079898dc1b2b0e95aa51fa23a30d020e06f62c430dd08ee56cdd8 \ + --hash=sha256:a64b85a4760337cfefdb27d42da6ed8b58e8cde3f2d57b6ef43e76ef6ea9ef47 \ + --hash=sha256:a655d34b2a6943af32401f3d94f72e9d731f6ad16285815550bf2b4ee69d420a \ + --hash=sha256:a714befaacbd10092ffe4cea0d3c5f008fb9efe9bc322c715bcdfdee414b9a3d \ + --hash=sha256:a760da040b47767b4b873adfb7c3b691e9ba2fc60f113f9d0b88f1a62f323e85 \ + --hash=sha256:addd736a0547d553283adaf4e05d7104e7f2c7b0b092e9b4d28756825f14531f \ + --hash=sha256:ae4613d7d9dda60fcba95f846cc6f808017f1843f392cf9daad14a6534493d71 \ + --hash=sha256:b11b589e00095ec69cf79841a76360f9b079e95b0368a25b5ebb951ab0c157ff \ + --hash=sha256:b3e445b66c80b4eb4234e855ce94d9adc183eedbd632816228d89930b91b2c5b \ + --hash=sha256:b7b893976e7fe42053da64f2aa27239c24252fd2ec6df471e1be197c0addc3b1 \ + --hash=sha256:b84f186a7f0536fe4ff9a9fa12d06d007b9b71d4b5352ddcc41f59ad6522a312 \ + --hash=sha256:b89efc38431793d28b7cd91227e2f952ad7c48df19132b17f43a5fec3c14143b \ + --hash=sha256:b97a38fb4c732b6832db6bf108963adbcd82ef1268ba2025dce390f45af75efa \ + --hash=sha256:b9d74e4eee9ddb64c2e92d5d61472c59c21684c059eb7b68767be9628e977859 \ + --hash=sha256:bb90e7177944b6684738c1fc36aabd2dd00d1de3be7dbe09f91e196f1bc0dc81 \ + --hash=sha256:bec37990e3d6121f29ecfb594bd8f1bf009e9f7926daba2e50e3b27d3892a783 \ + --hash=sha256:bf3c49863c23a1ad6da9c30351aed6cff8d5ddbeb63c5c8420ae54e98c7d0138 \ + --hash=sha256:bf48516e35cf848390ea68850aba53e7c333720d2945b4d2c25b69fc5171723f \ + --hash=sha256:bfc71e6d970419c1309b3640305298643e2a734cad3f7cfb6d2ddee4175ab53d \ + --hash=sha256:c0094897d7d01f184b2d7fe8c56c66d64efe01b31f4b7d34205b391387df1111 \ + --hash=sha256:c03c6eb6ece86dfdcbb34799efaa339b093132e1aceed491ba5e08fe06cdf699 \ + --hash=sha256:c1a9a6651197fbed6f0212591418b9def774fc3f8324f78d1bf0e6a63e5f8aa1 \ + --hash=sha256:c3589f40749acce747510bf5d589d54e376cb0930ea58b35effac97e5312b0c1 \ + --hash=sha256:c4e38dd8f39c43a91d2410ad2b85610701b0979342c3df1d69eaf8e838c757d8 \ + --hash=sha256:c6c8fabf1dafc1f1ddcbb67896d3f93efb092e8c4b6322d7389b944e76a484e5 \ + --hash=sha256:c90fcf7804ea0a54b896ce0f2b9565350220b8d4890fd0db461a476a4c687963 \ + --hash=sha256:c9b602fae1e00b7c035d661ce85575365719192a7b46784bd71cf64c68053aa0 \ + --hash=sha256:ccb64d887a9db1cd76dbc0f92051a1a478a2a67e7f56c62d915cb881d7734704 \ + --hash=sha256:d06fcdecc10fc7954d7c8f27a03c96055fe525274dc84a7b0dbdc3d6b9e03dab \ + --hash=sha256:d0c3082bf79bcd6a614d55916590ad4b8f93200e10b97f463ea5d9d07c9b5f23 \ + --hash=sha256:d49c18f1ea294cf4adde2e5ac256e98c82ea9d708462ce4bf799dffa7cfe8a2c \ + --hash=sha256:d60030baaa7bfbb02d650c126cdcddcb6e33dbff14d819434c8fa2fdcaeeeba5 \ + --hash=sha256:d7cab119d0df0b9413f106b4d7fc34f2872d3574ed3806fb48959c830b1537da \ + --hash=sha256:d9b77b25b4f395f92de6099ab08e8ae2bc7e51dfe157f22900902243a5cc90c7 \ + --hash=sha256:dabee8f4935e731fb46b2a3091bdda0d3d94b3bbfb907d2b4f12eefce4009619 \ + --hash=sha256:db5e82ba15c142425b8406690032df89e39cca4a2e8afbbb9a3d84edc2373ac3 \ + --hash=sha256:dc79d36d0618752265f0d575915bdc5c5130ecb9c9f6b3bcefeae32e4bdfafcf \ + --hash=sha256:ddfa987262763c3c22a8367d2a49c244b018a74c3a8e3ab1a864119ad45c5633 \ + --hash=sha256:e1172147d28d8fbcf8cb8d26c41506169f5ad8fe9ec969cb116835a19d4d8eca \ + --hash=sha256:e11edba5bc344a32b029a7af9d4b3173982dd79eeafa0b9dbd787364414b0509 \ + --hash=sha256:e2c89e9b762c57f59d5e99ee8b20202adb892e35f8d3485741340999ca55058e \ + --hash=sha256:e31f72490b7c12f7790e1e25c3afffd20503ee1bfb43461d7838b871ff244b19 \ + --hash=sha256:e8c65ef3862a8ad6e86492b6ed9327805dd66904c012bd3649dc67d822ed6c34 \ + --hash=sha256:ebb8912f565b8cdbbf27debfe00df04202c20e2f651b9e32767930c5eace3621 \ + --hash=sha256:ed511a0708e2297e1d6431e7fb217e3402791e491e02da800658ace4973df1bb \ + --hash=sha256:edf06545875f3efa31560d94121e95c7fd70d98b1dfedc0157097d79b13b52ea \ + --hash=sha256:f0fe9834e5aeccaf19a0d8feb296d66a24be1a7c9922002f842a682cd5abb787 \ + --hash=sha256:f1a0d5117230dd46b399a30a38afa44f79c99f3168988fdc4f425c3f928b39df \ + --hash=sha256:f37964e4a5e993d2fd45147741e9dff7f34a2d8c00ab94c4ea0514a4677f959e \ + --hash=sha256:f57dc6b8fef170f105d2cf5cdce254f47b137d7755086cf7050f47e16582abba \ + --hash=sha256:f93bc1c3486ef3747e07c9d7c1d0a147b8fbaab975f80e348aed6f71309dfaca \ + --hash=sha256:fb00027a09a8f9f08028b40dce4c933cf73e4833240ed356583fdc9cfa721566 \ + --hash=sha256:fb99cc9d45f48895d9d67f6a0b8a57f08d39c174d9f25ad97a313e0470267b1c \ + --hash=sha256:fdd88ed5e20b1bcdd234421e454962c971aa44b653bdb7f1ea9ef683e90fb649 \ + --hash=sha256:fe3fa1dd453ed5c7f5ea23a26218329790ed7197a99b90e94330e313959a7f52 + # via transformers +rich==15.0.0 \ + --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \ + --hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36 + # via typer +safetensors==0.8.0 \ + --hash=sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358 \ + --hash=sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f \ + --hash=sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d \ + --hash=sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d \ + --hash=sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0 \ + --hash=sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc \ + --hash=sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235 \ + --hash=sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98 \ + --hash=sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4 \ + --hash=sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846 \ + --hash=sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca \ + --hash=sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0 \ + --hash=sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25 \ + --hash=sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452 \ + --hash=sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d \ + --hash=sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78 \ + --hash=sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774 + # via transformers +sentencepiece==0.2.2 \ + --hash=sha256:046b15ea22d8042e2e173561d464ec3b64a9c2081324df70ebce7bf7ebb3e497 \ + --hash=sha256:0e2aae42960392d6dcb9a72d8e1e65a97294c965071b43c7b3429a42f350250e \ + --hash=sha256:1120e0791540615e650b2e9bea835bf38a7362455d8ab62dee7968219c2d79a0 \ + --hash=sha256:1402d8ee36f0d851cea8eee4dbb85fea14643b7503cf4d00d102eec0fe3ca719 \ + --hash=sha256:1416b92f2f010333786fe6306ed2631121d5ea492219b0841e967b6765e64107 \ + --hash=sha256:16c84ddef8d3084a8af37208acd365b08092ca089080f1a71fbfdd911adda9b3 \ + --hash=sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b \ + --hash=sha256:201a8e0f55501a76e08dbf2c54bc45f4642b379271e89c667d517bfbc2191f2a \ + --hash=sha256:252908153eeec06c3ca3a32077e64a49d572e3d89881475b4e0f02d99d9fcc7c \ + --hash=sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838 \ + --hash=sha256:3ab3f1ae98970b5590e2209341522718900ba19bcc2c207ffaa6bd417ad960c5 \ + --hash=sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6 \ + --hash=sha256:3ec27c152a1f1b24bc9168b55a5880f3c16e2334e697da6f55a1046a22405a3d \ + --hash=sha256:3f5851441ab1ef8634963a5100b733a8bbeefe623e0c5c005b1f1f3880e574cf \ + --hash=sha256:3fd9ce2ab4460c713cfdeb4aca693ca6732a11538e05fb332d5af42e3d7fde25 \ + --hash=sha256:44284adc6fbe9d5bdd480541431a3d93f674fa44736714d3ad4bcee8283ace7d \ + --hash=sha256:443ac618c7a2a1377cf5c82581fbb849591d14e656d5e5a3e4682d4e36a34e4e \ + --hash=sha256:46ba07b543add034de0ff47ac5f907e9a06682f91d85121a972764628933be6b \ + --hash=sha256:4f0603267cd15b92b68c2c0e852a441507614b70dc7773659baa6b8c214a91fd \ + --hash=sha256:524e2a85c028a0d2f9935191fa751e5ef9d9bcc39616f70ab14b28d0369c9936 \ + --hash=sha256:54a83df9260a89c1734256e620fe1f1a6bfedd7547139d4dc1384efac11a3a85 \ + --hash=sha256:59d6588712101ccfcae9b03692be3aaae1514c2078666d7b05f15ba3a702e41b \ + --hash=sha256:63250cfab8b80a1ef82a614eb2b3cadfec2c405f870cedc139d08e2f063eb708 \ + --hash=sha256:64b656f025355cf8c51abe9fbe3848540756c6d7ca5e6791b1afa664bc24c7cb \ + --hash=sha256:65d84ec36888de4a848eee5f910e67fbc79b064685ef1e10a502e14520ead9c9 \ + --hash=sha256:69e9dc8078e128286ed3b975e37c837ba96e215a50c3ef9f3f8b7ab9e5a832a0 \ + --hash=sha256:6dd76f3e5c8b2eb8a3a3efee787bbf5b9a66e52a048fe09cab85eca33fec6790 \ + --hash=sha256:70d4ca6f4d06df7f0ccab6fe4f49c8a712c8c8b6847b4f0af9a0e1dbb0e0337e \ + --hash=sha256:72b7825b331b1b7e7c45be2e674b3e3c65af608fa376bad2d851b20aaf0cdc78 \ + --hash=sha256:741b4b367140e9b5c36b5a14c72179f2c946d991ea9a7c031a2a1ee6ad097b99 \ + --hash=sha256:74f0ee601047c0c12a783088b51be4e6214a62ecd9e02278c477433cd16e0ed9 \ + --hash=sha256:76ff5814db72e7462dece042d7593cdf102b8ec82c2b1cc201a2add34ee3050d \ + --hash=sha256:77c3ce990b23441e5ecfa5bce181fd6f408b564aeb6d7e1d1e7de9c5612501c8 \ + --hash=sha256:79bac5a251f23a7341e28fda9ce0d5319edf45328239ce037c0682936f137906 \ + --hash=sha256:7c6e7bf684dc12145bfa685d3060beaea55139134ba848289bee514ed42e7383 \ + --hash=sha256:7fc14c1585139fa6b68775e616a6b90cf622ebf219f9558c0aeaf5d253ee6c9b \ + --hash=sha256:89625fb43765cccaa1443b9adb61f283e5fe4cb1536728205d06bada730caa53 \ + --hash=sha256:8b2db2056c97224e122054fd794543cde5d24b7cae28424f6e3eb79bbe08e42b \ + --hash=sha256:8d44b20234905ff022b7d535f79d1f823ad7670c9851cc4f03cdc34787cdb3ab \ + --hash=sha256:8eed98514bffe5ecac37f493f91869c351fbb05629328bfdbc08502c6c094dc0 \ + --hash=sha256:8f1f61592e7cabd45d49ce8cc0ef42ca655c091e037153754fb3fa59725b5914 \ + --hash=sha256:b23fe17779834d3c27aaf2edac9486d04cca1a7deb8f5facda35150ac6263a91 \ + --hash=sha256:bc7b0b1da20f856bfac5f84b2673fe534b167e41980b27442ca8f78c2b7eb77e \ + --hash=sha256:c62bd361cec1f5b556eb8210264ecfff37486cd990c3386cc00310f26c54090a \ + --hash=sha256:c76c9b3324efd79029eeb0fd2ced1964bdbeca7d45e030b46fa3ef3cf74f8032 \ + --hash=sha256:c798f0b327bac10dc95cdac77b9a197ab2bd7dd1e60ebd7586a12d918d4be711 \ + --hash=sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da \ + --hash=sha256:caad9566e2ef0e5640d36032c69b0edc7ac6028277b93d93815898804fac450c \ + --hash=sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a \ + --hash=sha256:cd810878180a52950e5a61f25ada5248a453bbdbafe474f89514135fbc1f633d \ + --hash=sha256:d254c98ca6387655400b3959c33c83efd807f5edeb608e3aca45800ceaa77151 \ + --hash=sha256:d795c4ac689a57f9d4ba2288126ec7901d389ad5827d2f8b8533c883974fe563 \ + --hash=sha256:df88b0c34f2fa909d322f7b06b1398e1e81af4b2f42a7b8e3556f928b25d1811 \ + --hash=sha256:eb8da9d9a9b418422c21a07fd19b9d9228692b7a7468a45eec6b11642d3c808b \ + --hash=sha256:f7c06c751c19d923435a54bff4f7e66e728fad160e8da28254f133abc9725820 \ + --hash=sha256:fa9f5ef0e2a82233dd0b8b32ea3f5710e0c44afbc07ed3620219f32601e56090 \ + --hash=sha256:fd523c4992041faa5c2b3cde62253d11a96c30d73a34afe48a486e8e2254cd1c + # via mlx-lm +shellingham==1.5.4 \ + --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ + --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de + # via typer +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via python-dateutil +tokenizers==0.23.3 \ + --hash=sha256:114e2b55ed177179d59f4ab98200a4471e11e78f9e4b5a922d146740f96fcf52 \ + --hash=sha256:1554a6eed34d9d6a78d23360f4e06df8dffab1ae08c7e8488e0b3e3b36cc266f \ + --hash=sha256:1ebf28794e7e4954e20a7f70fbea410b2d1f0418f7dbbca97ca384fcfef38c25 \ + --hash=sha256:1f0823bb00c5fdc98e487354d54dd55a03848d61a1a0bf29a68c77f24f3b26c3 \ + --hash=sha256:2a89614730d7b80940a5d2ed9320e1ec8add5a745c6151d8d05071b7215505b6 \ + --hash=sha256:376851d22bcf9d650a5c3090bb83e6cf9e895fbf0595369fa4cd43c1f69b5f87 \ + --hash=sha256:5cc24bb457dd4a8af89c8fcb40074d570129ec473df2a866c276ee55db4749d7 \ + --hash=sha256:68649e97d5b43c44c031d8d848874a6eecae8f8fe40ea989aa777a5a83aca716 \ + --hash=sha256:7e48734d2de9260d86f03ab056d2cfeeff3869f61dbd49aaa15a2793b5f3458b \ + --hash=sha256:82eb480f6f1c21cea3349dec32cf1a6384c6c1e775f00f83b0d51197bc013687 \ + --hash=sha256:84513ef0aeb8bf8f4ea11a2e8a7ac163ec5288aa115e649a59b470ac5c3107df \ + --hash=sha256:9d2b5c97daf61688c2ad1803ca851800feaba50fb68d5821779e9ea5880d968c \ + --hash=sha256:a4fbb3662f9f59d199d61338e54b4bcc11d07ebbb1aeb3540dacb2be9c521cb7 \ + --hash=sha256:acd5c57b4bd3e56e246e2731a3a3a6825a7a7d89b7e3b761ba80bc521710f04b \ + --hash=sha256:bf501c40b72d2d5c8623620210430e9cac1ce47a46e45b34107b70a1557d46b0 \ + --hash=sha256:c64a0713180ff16829d4e7f39a658b77ea11443af4e1aa46523692943c9b1414 \ + --hash=sha256:cded33237c77caeef62944d32aa9a7ef42bdce2b3497e18d137e072a8c4be438 \ + --hash=sha256:d3407fb7b9c4d75dd68850ffd7180bc0a5d2dbaf0762d888e612f31fec3f9c6b \ + --hash=sha256:ddedfd4b3b4be6be24ff6ca645c4a37fddfd305f6f3e354c54cf10b715c48215 \ + --hash=sha256:de536665495cb4b409d25bade41963f801aff4225c19a6b804b048f7d14e34c7 \ + --hash=sha256:e05ab7baf7f47b406a95fea6f3b0a484b2ddcd9e1d14b68844c457eb755085a3 \ + --hash=sha256:e88646b8580c5ad7f4361477f1298e9cc01771a1ee9aecfe32c47b8ff614cc38 \ + --hash=sha256:ec82e80e65a862275b97c3d90b7a523df8d9519ee48aeb4e9625b2cc909274e0 \ + --hash=sha256:efa3d7318406b4d115dce61ad5061953f1f44b128e79c020ce4615d763e23b6e + # via transformers +tqdm==4.70.1 \ + --hash=sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73 \ + --hash=sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4 + # via + # datasets + # huggingface-hub + # mlx-lm + # transformers +transformers==5.20.2 \ + --hash=sha256:080871e061546d720be57f5846fce426f46b44d61574907cc7404c2cba49d180 \ + --hash=sha256:7ae75dbc1fb3922aff1b6c97b9e5479d6cf26fd86a8866bfb71324190479f807 + # via mlx-lm +truststore==0.10.5 \ + --hash=sha256:30d36967ccaded5cbb38d602c433f53600036c79d502f4533a49b60a03bbefcd \ + --hash=sha256:9aaaedaefaf06d8b206278cf8b5012bc897f485a874503501e12d776df78951c + # via + # httpcore2 + # httpx2 +typer==0.27.3 \ + --hash=sha256:d0396f770a560ab1b0a8504e13b5f254b728cedb05c61cf0359e944e50ce8901 \ + --hash=sha256:e50022f28b82a86313e54501317a1db64bf8f8d036ff8cfe5ca7e47675454aff + # via transformers +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # aiohttp + # aiosignal + # anyio + # httpx2 + # huggingface-hub +xxhash==4.0.2 \ + --hash=sha256:00329536c281e13725283421dcdf4633f5710123e06a608b910d6a6f3bc891cf \ + --hash=sha256:03b1e97d14fd1b185c75808f72dfc7d9ea5866df3f8661566f2a23d0d07dcf7e \ + --hash=sha256:043dcb74ba30a068a2821c136b61262db67f1e1d174ff4dbe83a3e24996861e7 \ + --hash=sha256:04fe6c267bb6a04d8e5f56d8058eac0621359dc8fc03faa0c03cf1dd4343304e \ + --hash=sha256:050c1d66587e8f1ce70ccb7f661ec117e01497d48da817c14543beba0b8c677e \ + --hash=sha256:07aa7644899f97e4ff208f54b9d57fe4c077bfdf2dea5bd5a4a165e9b57309f1 \ + --hash=sha256:07abbc84c3b22064294e8d7f8e8d71a3c0dd3ae9aca8dfda55ad1977d3a7af61 \ + --hash=sha256:0b96b2694c97084c5e2bff8a6a3dc3c2fb3b9a07178286e8d69869efe76eeb50 \ + --hash=sha256:0c9ce49f326c85958df18789486d4c28812b52890e74d833b121735200d04bce \ + --hash=sha256:0d57b09c8f01945ab789bf8dad0e78a82b47f8e09b91d740cbefb52480a45744 \ + --hash=sha256:0e1832b53b321f742f46d692ca3c25ef23298e422f146d34b0e0010944fdf875 \ + --hash=sha256:102b2da60cf1b76fa4cbed4b03acaae4c4be1238e534a00fd608ace5027e3653 \ + --hash=sha256:12d5a3c0687c990124a98a1eb027655075deebd985388b4e8ba4302992b90850 \ + --hash=sha256:131d73e1467e9f782048d7f4901eb5ced6997e55ca761b503b598618271da8d8 \ + --hash=sha256:13bc7f3ba5b048fb5191d01f3971ef4c1c04a7f8a22621d6f8e94eb3c7cbfd04 \ + --hash=sha256:144a93e0efa8dafaa2826200edd377680981830fd3082b414cf8a96ba12a578c \ + --hash=sha256:14873a759b512ff875e01e7f8bd187ff613362cf189c44ba604e5cff833a308f \ + --hash=sha256:16e30b4a045f1456af45016a888fda09a45b5adb5606da0463a3a0d58130852d \ + --hash=sha256:17c05e116555586f584c6dce34af2517b6648d56ef5ae7e4d84ea1a19d6743ef \ + --hash=sha256:180b349bbc89af61240adab8519426e0cb6546c2487e4b26b464364bf59dcbed \ + --hash=sha256:1837c43b06cd2823b44ad27e80e51bbceed172a4afd0c3cc2ef543f2be74155e \ + --hash=sha256:1ab636ddc4f257bb2713c078d3f6112f194e92d8a32442233ea957fdddb51de6 \ + --hash=sha256:1b0f2dc4556ed6466707b6343266d90ba6cd6707822f5d567a6f8b51f5ed70d4 \ + --hash=sha256:1ba7e798c28e327b221270af073f7319c771daaf5974d18856c0ed3d0fa1c6de \ + --hash=sha256:1c5242bcd736259c1600cb2588736c736cbbf8872fd7a091379c036bc59c5bd8 \ + --hash=sha256:1d549419687aac3f25c17a0fb534cc6828b5a4c3ae301544b9393127e413ac59 \ + --hash=sha256:1fce8df7958f0828bd78d2ecfd9b11510abff9edc0e28fa7f7ca0ddcee2f0a8b \ + --hash=sha256:202c18875cf93d652c506d0e7c8a50847713f963a8aa9c06ce5117bdd14ba771 \ + --hash=sha256:2076eee9f06fcb611754f98b9c272046e4a8a3fcfde353575dd8e6e2f33e80ed \ + --hash=sha256:250048412cb196e1f705134dadbc415eca399e74d2678b935660be0b2e4fc380 \ + --hash=sha256:26f8343b7a5d65f704029a0ebeb7e98a47149cf76dcab0072f0874307181bfb5 \ + --hash=sha256:28881f55fe85fef47921f4cbd3636bbd75506f1e616b2ae61e266d53c6c56637 \ + --hash=sha256:2889ac17629a786dec7991112b1dde2de377f8035dc0bd9590bd7bc640b01ca4 \ + --hash=sha256:28ae2c583d923aa57c0a4301ea4e071bccf1f84b5e16a31f86b854d9f8943151 \ + --hash=sha256:2a99c2d36df2d9b238a27e1fd8547311dba51c997e4c7681a383a98c0831aee0 \ + --hash=sha256:2acdb1cabc41140e97d4adba7422dbeb7fabef9caace27d56f8de662d505fdfa \ + --hash=sha256:2b5849d45e5c12ddd23316c999eafda73c3ad00b589c474285dcf4135205921e \ + --hash=sha256:2bc6cfaa4a39bfca4c31878ad769986d2d039673eaf527068f668e88ff65ab99 \ + --hash=sha256:2c2d0dbcf64ac20300508b5a7bd763924d58f19feb13cf693922fe7188bbf431 \ + --hash=sha256:2ca60825069a19e381f4ed03d0e036907b2b98460d33d4f8d1a93cf2d7e37c7b \ + --hash=sha256:2cd94476503f0d277e4188cc117f72e7bc92e2287e415931f3d91bf0e13a3d21 \ + --hash=sha256:2cf332a7b823e867cc0d8e4c69eb5c82ae96ed827677fe4d96f990a6a1fd86b8 \ + --hash=sha256:2f164b788017a4a766884aa9acb3490bbc5f03a79adc7d02894393df44675c1e \ + --hash=sha256:30b1798c6e6814504f32e81a0b87072f9b38628b0d610c22d53918e5059e086e \ + --hash=sha256:30ffd0deaf0e9ea112c38656922323ae23dbaa90878cebdb243f646d121e52db \ + --hash=sha256:318e6c3a31d75ccacd1f9b864e3b857d44c9f4bed5e9d5fb5db1dc2fec82f019 \ + --hash=sha256:329bf14bab110bf111aa7211c613a94e97e4554407d2cc964c01338278cd4213 \ + --hash=sha256:33f89d2a64716d7108f49b38d92ebe5344d3c50440f15f48f0f65cbcf4ffb781 \ + --hash=sha256:3473e7ac274a9a63c90acd365dd4b481659c6e061db153851f6e5d7b61e264cf \ + --hash=sha256:360343b1d61efb44be005c2180cb19f8c0351d7dc63c8ebbdd8e219ca5cbbb4a \ + --hash=sha256:3764bf2b5c139932b4951092103be2f7567950b1441cbda69c94107744d9010f \ + --hash=sha256:38ad1aef6d84d644b4a1afed0a7d226449f09cc2de1c9a0d0021c51f680ae2f9 \ + --hash=sha256:3a5529d43d09851dffa38936040f19e938e8368ecaddc646035a1dc7801bfa09 \ + --hash=sha256:3bdbde088f54a5cccb92f0d199e018bd676dbe2e5d2eabf1f9dc8148c6f297e2 \ + --hash=sha256:3c368344f91263272aaf0fc5c1420f0ec9196466d507f8b1ba52d0e29645d1af \ + --hash=sha256:3d7f484b338ddf6a8e58ffb73c4f9d13ea157f01152fdfdf9c44c9a59666f3ac \ + --hash=sha256:3d8b5da050138043edb95fbd4b79e240a0994d2f272e872cf074dc6acd68249b \ + --hash=sha256:3e0d6d7ec5072ed81e54de4fa778e6c1c8fe70cf8ccfc3907abde59147fac84e \ + --hash=sha256:3e229bf3c7fa330c84ca25bbc29a83a6d078ef2cf3b7b826a5c9b3828aac6f07 \ + --hash=sha256:40fab713c1d2f5c5e7450b8731615d8580757599ebf1f8683b211310ed8e5ad4 \ + --hash=sha256:42fcaec8c705d1ea362d4cf011aea0dad1ebe13096ed227f9cfecfcac396c72c \ + --hash=sha256:438fdfda0fb1198b7e9a5b4da6b073f36cec5dd7129dd486bbe72c8ecd623adf \ + --hash=sha256:45b10d7adffa2d5805279a123ad0cf5dc635b7e72542215d0e0f221cf49038e8 \ + --hash=sha256:46fb84bad2ca32eb68eb11882159d0a0af19576026946a185c957d8d306624a7 \ + --hash=sha256:49b93898657ba0dd4517576b5c94fb0aa6738200f6d1ec2dbc9d0981154828dc \ + --hash=sha256:4a5955614bf744ac52faaf535ff94df3e935ec057c6242a3d8aa4e1007efa0f0 \ + --hash=sha256:4c3834e32060ab46ad9cfe4981d48c306cd1f124b8efe0aed559a765f6e1d332 \ + --hash=sha256:4d1fa6f15a913dbafa50f3ee143a13440cd0b9849a1bde290c72ebde7838c937 \ + --hash=sha256:4d3a90ec615e72934be372bf734190626614929054809ac12e59950aca91d8a4 \ + --hash=sha256:4d5f0e2bf5257900f60339cf439a23b35fd29a45cb7effed4b9f1b87c094df20 \ + --hash=sha256:4d86d5aaae8312de06d25b91fbc901b891b3b5ce2f5c8359b340b4bcc3d01b8f \ + --hash=sha256:5065d7e74b7b3e3df03e11b47d9b14bdcdd0171603ae77cf39c89df140e3e948 \ + --hash=sha256:515a5f4405a29bc447bcee2ee13455dfc67ce41bf8f1da5258bc75bbdfb6dee6 \ + --hash=sha256:5223b3d133b81197fdbede79601b9b0f0bdf07818a71b751e9d2480bdba72a70 \ + --hash=sha256:52357b9fee08af3a5c81a793c97110d8be63baba91b88584beee400a2a0113c6 \ + --hash=sha256:52dff0cfe99e1f03614a40867e39d5c5a1d55047e114265db4ec2355d65fc542 \ + --hash=sha256:52fd50828b49caf80824fff67bdbd8cd2db7e033dd8b3edf024c98b326d2a8c4 \ + --hash=sha256:531a1056fdfae837ef56c81e62bb4a0ce3274b813a3164bb558151d913684cce \ + --hash=sha256:5359b14f7467972b50ee264f0a711ec0bdc7d0a3d05c11f5ed3fe462fcbba0ee \ + --hash=sha256:5424d3754239b3080d2c0719dae2e6fb0b0e74cc38125f2722942c054d172257 \ + --hash=sha256:56ee43c284daffc12d269b3fb36b1fc624c7ab007064a24d54099049ee71ce80 \ + --hash=sha256:570ba2952f1fe3b4f7a272314db64903e5c9121032e1dbe20800c7e0e5a13145 \ + --hash=sha256:5846aa6fe15c5b1cda6738ef9b1ba3cf49028b729507ce44a1e9fc2642586b06 \ + --hash=sha256:58d8fc67453d337a8cbeed4f5543493c3792cb5f3839c17bc08ecf307592466f \ + --hash=sha256:5a4e73f8a17c433640d452a94455414a9888b88be0443ca37541d673343b261f \ + --hash=sha256:5d51b2159376d98e99f7a33e19aed3d58d30d24ca641086e2ca0d54875d92ea6 \ + --hash=sha256:5da15e375188f4239277559e4efaab4cf4434162e622b2ba1dea956457994cf7 \ + --hash=sha256:5eb07be4bbf9b0b751576edc37423017e5666d37d24b993b476295ca0d2138cb \ + --hash=sha256:5fb7bfdd0d1136fb73e4efa9a1eaff71a72f45ff7409647f601bf714d7838922 \ + --hash=sha256:6010eab32dd57dcb8070b81a5d459c422d09de3ffd1bf6f11f0ee381582fe5a5 \ + --hash=sha256:60208a7c31d936045e911ff0c3170ac0e432611b6800c220cc66b908c33b16dd \ + --hash=sha256:603eac9723ac2becbf126cf0d237b6ac5cd645592809d697aeced51842ae09bd \ + --hash=sha256:6049b09b27d9226dfe0a8b0dee2b867063c8df9a7ef64748bf48110d8fc896c2 \ + --hash=sha256:605b582e3895fc240b972d9a28606c53fb5444c118c9f6392bffe8795a4c2467 \ + --hash=sha256:60f30791faec3e96c8ed5921045231f7d3355649af47809e5a75c8a17dba8b6b \ + --hash=sha256:61ded7dee4f95da69295c228fc18a65447f5741c45a315aa3f9a3adda6038611 \ + --hash=sha256:61f60d0233612bfe3502a6cb1e233c469ea074e027328d72ac2c63b58fbbbce9 \ + --hash=sha256:63522739ce94e4e73c1605aa60a52b3863e77e835f6292cc8a5e7e7b626ff00c \ + --hash=sha256:66fda3f72176bd7d1d3aa71de1f4bc116276e3ca3ac583bf06e5f561f2b1fc74 \ + --hash=sha256:68ed6108f9ef16bf90752582aadaef4c14caa00a50f8b340e282348365a351eb \ + --hash=sha256:6e736b549811153e3b54779708725c68e105a6e6860f9f30fcba896a5f28ef0c \ + --hash=sha256:7016f753d9c79d2ed53a1517b3eb1bd475f05d026fa3fbc3acec393d14ddd396 \ + --hash=sha256:711b8d7d6a4588c14a0874c3d4afdb0b9cd501a11705e231d79d6660ae937f60 \ + --hash=sha256:730fba89df435ac081bc2ae28e4fa0ca5009a9eb5f4947b648abc27acd96e7b3 \ + --hash=sha256:74a92b9104c4af158ce37021b32cb165aa74e69c39bd10bacf569f39604bf2c9 \ + --hash=sha256:763a9b3447ece8a5a1e9e27749ef3910a2f6e8ac251b8983a3da2e0801956e02 \ + --hash=sha256:768aba59c7fb2e3e976560c91223cb01f00f83809088e451e59f05b841486fbf \ + --hash=sha256:7779b0f388972029f0de4c83dee4523544b3ee33272425683a34c801c8afe6a3 \ + --hash=sha256:78219e648979877aeb1782486f598ca99c5788da388155510daf6527fee57ef6 \ + --hash=sha256:792e7c12ac0d300908a99f644e583f4091c3a5a0035ab7b1b76aadc908480381 \ + --hash=sha256:79cfd336c2ecaceef1061e7a7b3da079eab5920ba1352a28be690e857f67f2ea \ + --hash=sha256:79d832049ab43b88846a13161351887892f3d8cb0bcd3e7c91b3f125caada384 \ + --hash=sha256:7b244e3e1d6eadb5061ce777632a822ed75c2fb6174c65882f5829f31b46e8c1 \ + --hash=sha256:7b4855f9a4b7e2134d248af852b4e5b1381baedc01d70afa9736efb40dc5bc3d \ + --hash=sha256:7bc036ceb030d468473b9e19e3d49617c48b378f92c4cc07eee83966771a2b55 \ + --hash=sha256:7c1087b8dc4a08709e1ce358a79671f9aa4d029211aff537d0f821e4a6483305 \ + --hash=sha256:7c7c95dba490be3b4e5a2fa5595b4d4f088fe7b8726e5998b8c1d83b1f9eb3e4 \ + --hash=sha256:7d66a2b02c02cf5a165454ec62372bc4a97071c2652acc3f917814c2014ccc16 \ + --hash=sha256:7d8d913ecbd21dff2f47f458a8cd37beccb3e41504004341f42234829b973309 \ + --hash=sha256:7f290205f86f984507ab409a018986dddcb0724c4197705edafb80d21be85355 \ + --hash=sha256:8050eb0d892d11308f81b6980d13098ae065efa57ec68f0e9ec44b8bc6b9988f \ + --hash=sha256:8113a8cb51a5121b8017f275e9aff2cb9203d57e614c989f943f81937442896b \ + --hash=sha256:827580bfb04f69c6c40283cb299be15508e151af066aa012f07732e9579b3f30 \ + --hash=sha256:829ae333a35219c24584755b8f5c718b87e4713ac4233c8531a4e5d3d67fa7b7 \ + --hash=sha256:838dae9c2efa4f4cd0be082a1a4077aa3043b407d8bc72f82864cd0807bd9945 \ + --hash=sha256:846189d5cc9a4e6815a04c595148b2a82721da1fcdacfdff13fb0225d0945ea3 \ + --hash=sha256:86fdc2ae36f5d11446896c4456883b3740be2ec3ff56f700a7dc7aeb635e236f \ + --hash=sha256:87255720aa1a1be35256b688c5ee4151d3273894186cda5c8a5b82f1dc05debf \ + --hash=sha256:873ee53a7d8f020ddfcd3bc101707ae46fcd1eb702f809ff29a765d0f88b654f \ + --hash=sha256:8841fb5f071edf93655bd304e7d43005630e71c6aa58d345cb312a867aa1a6c3 \ + --hash=sha256:88fb30a81b5c7cb4b4dac3b4974deb00de06f5ed4b15fbacf5c7e14d670ec049 \ + --hash=sha256:892a66641657cf9aa0ae5c2c88258bc281221a33bd4af34bed786eea23e06d83 \ + --hash=sha256:89d02277a7d5ce8b3fe21182ce4e32cca54963768c4b9609ce29e1ac82922acf \ + --hash=sha256:8c24094617758a4953fbf56e96637262b56d5007fcf34756ecb8d0b60231f53e \ + --hash=sha256:9478e4ae8b1db3f6fd72a5d17fc5805fc0ad576e31b58e2acb6bdbeefda2a549 \ + --hash=sha256:97903f3df852ca5840a26cc27beef15f9f93b0071e2aa550a03d8ee5c65e7765 \ + --hash=sha256:983a14a70a018dcccd847e780ba630771d5525d7604b6f519c963634414180c9 \ + --hash=sha256:987a83e7a7762bd8d40d6ee75bb047a33f6ed34a4269dd6bdda561666c3e0c54 \ + --hash=sha256:99fec645143af9dc697620aebff07e99388c16d4208a2ee8b6a2a9bb7d68a21b \ + --hash=sha256:9a1f6598dc8f7ccdc9bd5ac51264c71431185194e8c88042502a04c1f53b5780 \ + --hash=sha256:9a23f0c1dd31aa4136ef8a678a40e73dbaf070b54ee6f2a907b78ba40331b3c3 \ + --hash=sha256:9c22ed1ea72cc23cec070fc5b9c3574844e67236f3f6e5bf7058e512abefe086 \ + --hash=sha256:aa60df2de8dc4f59ae4faddd77237514ff88d15c2afd6d501e21dbe82a4711db \ + --hash=sha256:aeda53dfb6bcae94adb1f5df4246fe13b9c4202f531dd9d39e0d55b36ba8db0e \ + --hash=sha256:af3d1b3cfa5280e84d8968e9f30c9eca75781db4915eb6d6acb0dd4fc93e0f47 \ + --hash=sha256:b3428dc057d1d483723d597e60e778bc58dd117daf23b866c1a6b5debb8eb300 \ + --hash=sha256:b4c405512c813a47cf70e47c9886b07702a38e991756216b65ac19f9cbb4f8f8 \ + --hash=sha256:b5fbcdf157caa642de731565310bae6a5c3b1e15f28507feb3eddf04bd4a56a9 \ + --hash=sha256:b7f62733495384b424bae45ac457fc80d2143c80de2564416a2a7cd8a4dded4e \ + --hash=sha256:b881b67c01929df867d30fed94d4221205639072a63049818732c1bae54e74f0 \ + --hash=sha256:b91b42f62e6cf6c1aaec9cea6b9e4b8ae18119517242d22ffbd967b275f9d12d \ + --hash=sha256:b97cc2b874d0656e88385fd7ba4c2dedd903f8e23767f69207fbdab79bd0861a \ + --hash=sha256:b9bff92881cfd1d4f98f27e1275f125f90ddf324571ac9a3e35d1dfcf6f42d19 \ + --hash=sha256:b9c7c18d235991edcfa2275889e84a1458dff0b9315cb7351f5368c5b942bc0d \ + --hash=sha256:ba2daf714d2c87374cfdab5ff22736b19cc12a200130b64610feb3b0b08bbe11 \ + --hash=sha256:bb99a2c5c35cad78284e057f8207856c911ac088ff89d0807dccf07b65775895 \ + --hash=sha256:bdcdedbee2922898f4457cf3a86ca857e442770d71e847db36d7df82912184fd \ + --hash=sha256:bdec072bdb457c65e4b96eab7610af72d1535ce38adb32b4e02fa6957d7cd2b8 \ + --hash=sha256:bdee724c52954f6d3a551adc30eac1555a6ab1444dd3e02bffcc51832cbcd5da \ + --hash=sha256:beb27fc243c6975ea44f74ea2e2c8c5014f9956ba0bf95ed6ee6dd3707e0add2 \ + --hash=sha256:bf93a290a49285f75d0d6ec801aa247592a16bd043892a7b48d93f718dfc99c8 \ + --hash=sha256:c04c1aeff5cbe56714dfa49aa5d2edbb69c1a1a798926307b8d6af86549e7534 \ + --hash=sha256:c06c5fd89304f2c274d5129bd3a555b78f7a8af5ea16edd5b1360010f3359bd4 \ + --hash=sha256:c37fcacb7f5b8590c822bbf3bc41690e40a7581d8c4bcea7b15148f0743e716d \ + --hash=sha256:c3f96730e700636db76860d2710147202470e3059d3d37bc018088ebc70b7573 \ + --hash=sha256:c45f489790caf482840d2c108d50b8b0d9e77f2f6d17e2b345e2ae3dc48849ff \ + --hash=sha256:c5d199912f885f381ac518d0bb300dc2c4d543adf8f98bc41f33d9cd503d2493 \ + --hash=sha256:c60fa1038814c3c682a8e3b68aeca598e7b90eb4a65d4a48b7e0e6fbae825507 \ + --hash=sha256:c7a8faa852e40cda4d23dff1243b3fb0e6eb5b1943c646559c2f8488fcfb7b98 \ + --hash=sha256:c9b7b91d072e239d4d0c6d868ae4a2f75dee55856227638488db156a4b462588 \ + --hash=sha256:c9d48e67fc623850ac65b350742b758085ea3aeada2f79aacd0200676abc29b3 \ + --hash=sha256:c9f3129e086297db6b24b83db59a0dbfdd8f325ffc7dcc184fc573473b13293f \ + --hash=sha256:c9f9c25738cbab7ca51763f724db95bc7f50f3174793389fee7f97e154bd9c83 \ + --hash=sha256:ca848e5edfe3aa31ea7355df790b25e55f910dbb540d587e738e1b0c644e1263 \ + --hash=sha256:cb66d346593cb6851498b9ed554582aede153e32c9835edb76995746e563ac6b \ + --hash=sha256:ce0fd7c57fa4fe9e0a9c390a04e1668c09fc09253e8fd76f1785069508686059 \ + --hash=sha256:cf390edab741574cc770e0b587cc2d6f2573152ba09bf85fdae10e556333ac77 \ + --hash=sha256:d42a7b02f204d5133d38d173766ff47a5da46fcf2c3876718b819f1f5500a55f \ + --hash=sha256:d621fdcb127f9cdc87e35388e064f43db519bffef74a3fd34b6cd1f19364e74f \ + --hash=sha256:d63ab508ecb528c9ede793a362411c064e8c55973e7968415bb0159144ba0583 \ + --hash=sha256:d695979588da837a36d5195d10812c4118515d33e4f41a9b4d0cd92bf7c2f716 \ + --hash=sha256:d8dc7e4071794c0405dc6a40559dbe77d7eec1e4bbf6eed5dbebc4e6577d12f3 \ + --hash=sha256:d9cd501f5d6c737e26b8d45f0facff43d3b7f2f5a8c2ef1c88800f6f310da418 \ + --hash=sha256:da1a626793a5fde9c97466d443ae7c62f3d9bdc8658fd8363f435852952996d3 \ + --hash=sha256:dc290f7366f2a31093d31aae3ebad13299d7e4de9daf4ef3e544868bb7755364 \ + --hash=sha256:dd86600a7083aecd5e0fee0680e5a328cbd1bf18046faa217e2a76cf2e01faf4 \ + --hash=sha256:dee8f3352ac0fce896bb0d2493221dad6d6af5b909d2c38d76827d7a6769766d \ + --hash=sha256:e24e9093771e8a27089176f202991199ebedad8f4791a6e2abf88acd5c69bd9d \ + --hash=sha256:e2b98f3de3e6ec78333c6d4db6b8d40634ce6dc05140815a195916385f912176 \ + --hash=sha256:e359ff2f38be3c856a129c1c5c4cbd70e219188456c1bf266fd4f94e2baae278 \ + --hash=sha256:e4111b5d46d1488399562897f7b7b827bdc065bba0355449348a34cc7545fef3 \ + --hash=sha256:e553d545d05094f3950ef36b328f75f5ef8568a0438c26966d9a0f1fc187a42b \ + --hash=sha256:e79bc2bef5c7ca6cf4c2561aab00340d369b80ea56976a4a4c6ce5e3ed8fb59d \ + --hash=sha256:e83c54a9b8477079d6dbd3814ab444f43819ae1aa06b5098c1020d77747897c0 \ + --hash=sha256:e956b1a4e8fd8ea6fdb823670bf6ee3eade44c5ba3bbca06aa67805e4815a9e6 \ + --hash=sha256:ea7b157d98bd8dd2cae312422657a122c059cbbfa67fd7b53c49d8aa2b5f4202 \ + --hash=sha256:eb90b2efd8d76777995ffdf3a134f079a79d59727a3301ca98f818fc142e841a \ + --hash=sha256:ec851cf4a64d98e09101a18190fd20de2928b2ae1541093236664a47fb101271 \ + --hash=sha256:eff2109cbac3a8b04000d1b687331ff538d9d72d266c5aede545fd6bf90d303d \ + --hash=sha256:eff48c792788e33ace8e8e3ef6a0db45741d6acdc35c9b89dc914469fa040536 \ + --hash=sha256:f064c47d2a281378c3d173d47ddd5b9faa4273dbcc40bffe51ff5d6817aad7ec \ + --hash=sha256:f1a31825aa626279c25d797c105641d398865a93e96a042cb89c7faaa8905e90 \ + --hash=sha256:f26362daeb8e71de918899c3f590129858ede3559bc1d251ac26441bd0221ab6 \ + --hash=sha256:f2a9839e13bdb0b611cbe55f6ef865dfcbeff1bb2d48de313fe94b9dc33de1ba \ + --hash=sha256:f2b2839a83b9a9d03ddf89f440bb43627832e77b9a4276988d8aa0bb49aed0df \ + --hash=sha256:f2b936ead8be8e1771b3a4508f5b8ba4087a57c59e3f6e91bbbb6849000ee302 \ + --hash=sha256:f53479d8e13f93e1e3ef0e933a1901cbb4d8745c61c3876ff5b7c544e40f13ed \ + --hash=sha256:f5a509325aea8802d2ab710a5f2fb7c4d7b1a6c90f9a148dbc9f84311a1aa15e \ + --hash=sha256:f64380470531ca49aceb15179ea47dd7be2fc38e41207eca4f7d05e4d6c9f795 \ + --hash=sha256:f6831526d8bacbaa6a0a54673cc63855facfe12424d0e4cd68ee8e089ee60a22 \ + --hash=sha256:f6efd3d8a3ee0b86dd7ce3e6ffe47dcc115f71c38cd9f7392e6387fa541a243e \ + --hash=sha256:f8e86a0b5e28418162dfa68d6dc4bee873ea2efbd6ee67789903a0ee37a81c14 \ + --hash=sha256:f99686036e0575b4bd1947f9efcf12c5bdf310b2348fb72db84a4861287df6fb \ + --hash=sha256:f9f528029f32c081622f36a0149898929c0a89f601f1cdc6620c57cd836762db \ + --hash=sha256:f9ffee37361effc04fc1ae120f00e5dfe2491b05d637cd45e62b8a8b35bc66fd \ + --hash=sha256:fa1febd6e8ad9eedfbbeac0cb5d86346ed0b78a846d822c3af36fddde503516b \ + --hash=sha256:fb3dce09d52103f29a4648649df1ab217e28ccc2e9a9c08e8daab472e5cda9c9 \ + --hash=sha256:fb83c398f50f1c030cce9dd8accfab5fb02ab5f695c157e051ac2a21840d7574 \ + --hash=sha256:fbf323d9c22c6ac25cd8c4633566393282fa22db583ef0320f82bded10d7cc2e \ + --hash=sha256:fdc8799f1a4835b40c615cbc9ce73c8a42f51c3218726d285367dcf853f36128 \ + --hash=sha256:fe3ae82b1c7d031da7bb300188fe31018fbc05e1e496e0edb7c930d6ecf88d1a \ + --hash=sha256:fe5a0d96cb339d383d3f720ffe94d8a7a51ccc8abf4d348a3ee058b5625f948e \ + --hash=sha256:fe765b3c7e7fcd14ef9b77836295847cb73229cdbe7eced958ed013d2f1c4025 \ + --hash=sha256:fe86ae86241d0aee7561b19bf892c9250a976e4a56fc052405cd537180e7af15 + # via datasets +yarl==1.25.1 \ + --hash=sha256:0136d640dfa9b0523853e411430a99f8a91eca85774c6420285a33b755bc6de3 \ + --hash=sha256:03dd38de09bc213e9a8b29761eec33ee1d5318dac0e49d8af36e4d27830e23a7 \ + --hash=sha256:0a191bfdb30a79b98e5d175d75285f9fcb78bf0e46ba5efda042e1c72071a0de \ + --hash=sha256:0a66db89ea473abeac4b70523cafd94db3772380e565f9d28af7a179b7af71fa \ + --hash=sha256:0ae12ff2b805fa02c4dab838005caef735e39986322698c48588d3beacb65c62 \ + --hash=sha256:0f12afda4eea8c8994a76d4df1875c765194f5fbe8a9d197929ea303caee29ec \ + --hash=sha256:10b2fd95332f0d716d5eee3c9fb2ce8eada19082de7fee83d32e37992fd75c26 \ + --hash=sha256:126a2533570c554719ca40a1288fdee1700b6bc82e7131aa69fa85252d92e651 \ + --hash=sha256:12b6bc4906e11f5e1a1cdcb12296e7afbd366c783cc8073403cd2fb74334e453 \ + --hash=sha256:142c06c4d6a35ee3ec5da08499805e879cb3ca7c1fbfbecb0140fe72403818d6 \ + --hash=sha256:14b79a30a93a3ce2e8832603fd0ab780ada281b0ba5110b519a634f2d7d7d1fc \ + --hash=sha256:17c9877a89fb6e2bca6f9087eb24cd7fb434653946ef5075e470d23d49b52287 \ + --hash=sha256:192a866877a49993949ef1975864ad8728bea28ee810f6abe1a0729c2b500426 \ + --hash=sha256:1ab7618921a93767387a4b83776f751588f5b5ae9bb5bc96620e2e2e00bca868 \ + --hash=sha256:1e80dcf1446e1b080b1932b0d103c464a04112f5bc31f0f983ad418172063cde \ + --hash=sha256:1f51020b2eb8a003c84925638ec63c21a750a4bddd3a22ec8eac6a742dadf1b9 \ + --hash=sha256:1fb2a01ba8cd9c5d2c5dc1ec35e0fc951d04b4f037541d4ac090c993ce58b3d7 \ + --hash=sha256:2239a02249d9326655419e0168a28ca9008938eaab31dc29fc875c217927a6c0 \ + --hash=sha256:23bf5b403c879a54964e0feac7285688e04bb220074878d737d331522da0a5bf \ + --hash=sha256:24ce942011a61953e7d313438038f4d32ff21387b775f58a957f7a07dd55ef95 \ + --hash=sha256:25868beca8b6765f8f7d0e11fe6dd7c66dd4b0793b9500286d20cc92352126a5 \ + --hash=sha256:287e99ff5aa4dc1c7630bfc683ded6f106d756c99dec432a2d7f197a784f51c6 \ + --hash=sha256:29273edf1530e397bd07cb784db1fbe0d2590b77569f2e24679a9c0a2d763b94 \ + --hash=sha256:2b49375d22299b0a834c2bca72f39aaecc270d96fb24c30424899676f487b22a \ + --hash=sha256:30eec96e8a91bd588ce897c9543f6d5d8d34b28fbcba28a4dedf20ebeae9fe57 \ + --hash=sha256:319e070a01db9920fb63761843f96a104c8e2b9427266731810dc1e22595b17c \ + --hash=sha256:35dcbea443fafb3eece757ad4e514560ddeb6c34cfae1582c620d7b293d7feee \ + --hash=sha256:3f4d48a6112712973e676bd792121fee470e432d749177162d9949d5c9460a1b \ + --hash=sha256:3feb99222553a8cbedfa52c2f59dd84c3f50d5b582c728d522caf8d72769a54b \ + --hash=sha256:419f392a1da624877975709e3864dfe833af6cc7671b39318086d456e288380c \ + --hash=sha256:42a66563d8cc056ee32e6191e05097a7b2b3bc302e0bc3133daf8710eb18bd26 \ + --hash=sha256:48796ea00a303961507dc6c8437c4b325a6fc3f95f7c36c71b91ea9a8150963c \ + --hash=sha256:4bd6340d20ae2c7ca719b87b426e808e90743b676d05d4c26c4fb5ca71f41184 \ + --hash=sha256:4ca89e4e21854ed27ec753297dde84b16c9f8e53b14a4866fb44457d643c19f8 \ + --hash=sha256:4d781294bb815ecb5ea57ff6bbf8038e0a31a95fdf3e1788f66e0dc100d64b58 \ + --hash=sha256:4f1c91f5a5980a937ff8e238e98e6897e1ad74a4b1e2c0d68c73b5ffbb3f5c0b \ + --hash=sha256:564fdc7085d2245ab84f88882fdb1d6ac0723124bff6ded35bfb1c00f812630d \ + --hash=sha256:59ba3a6e1aa8cfe5adf4bd270fd965db21955401b7ca6f1696010c55ed4daec2 \ + --hash=sha256:5df89f769cc8ff94c3d7e7603386fba309d25ce5240132d26c15baa8d0e96c4c \ + --hash=sha256:632da579b2d879f6bad20f2cfa35ded1efe2f4f77f8abb26a6234a5b236acd2f \ + --hash=sha256:65b5b2066651b7432d389e9799d979c703bcc6ef44266bb8153ef54e91e4aab3 \ + --hash=sha256:664ec6a520b74a1df2810666eb67695fcb77fa663e6ea0a25aaf2e529cb24dfa \ + --hash=sha256:681c758b0490f9e96b78e5fa8e8dc6e648e9185bb6eaebe73183c33ea0c445f3 \ + --hash=sha256:683e362b8ba453080f7489c66f4ea794e751c35b72e7eab3575ef784c2fbc7fb \ + --hash=sha256:68782fdb4027b8d1eee25ec35e9a6db05e863b899eb0310b3a33b6c3fef55707 \ + --hash=sha256:6efaf45df6a849cef613a03a94c845647456662f85438c886bb67a9c027c8c2c \ + --hash=sha256:71f42c5b9a948c113bbdebfa544598321431d064ff959d32e99b1feb61d68345 \ + --hash=sha256:72849d892954be4d09e569b8b831ac39ce58417fedc767d4308a0fe542018a40 \ + --hash=sha256:72c34ac7ad4314c19362d5ce27626dcc8429bd30bbf8c179f4234078851f9492 \ + --hash=sha256:734f6e5400352ac4254456003d462866c684703570929cff7a7bde015d0cb371 \ + --hash=sha256:75baa6cf9b6d1c52f3e111a130e202fd8cf0a5b3a066c3f73d615e885092e4ec \ + --hash=sha256:77716e245c90f058466a05e6a465bb8600f767a8f4b18b4d40f3aff958e5f73c \ + --hash=sha256:77e5099b99b37f3cf79c246998ca9f7313a78054cd1809ec46bc1afad47e1c4c \ + --hash=sha256:783dd1467083f4d3f7722ad6a313f24c173e7571372738fcb7a6e6d1ba48df25 \ + --hash=sha256:7a5c3115595995779ee21f2567035793911c3802a43c74f3fbb0314929ec67ac \ + --hash=sha256:7c88edaec8c349ad4c5ad4c486a3defcc4b80ceb2f074436ffa0a87caf5e76a6 \ + --hash=sha256:7cb414a73e21a7ab58254926073f2930cb22f5b4314ea4260a687e2b3fd4dce3 \ + --hash=sha256:7d42e7e3ca399555578b4d617e3a6ecf13371b3743a115995fa010c7bf341459 \ + --hash=sha256:7d575b54cb3863ef9bc290ea4b009999d55dc237326131e4853cf33e888fee03 \ + --hash=sha256:7e4de3ac4adbad3d0bc7c6f4360a7dbff5de2f15e3b723be3198074e17fd9c40 \ + --hash=sha256:7efc9f082dfed77c316edffa9deb52888e1bc6789171887cc1f68e06d65465c8 \ + --hash=sha256:80a063f8297fc796296f00f100be520f209b23dc98f93ce8eba6ee7122598209 \ + --hash=sha256:80e47012e730da131c9f059c80936783f9659aae22dc31c03c0595590d11ed54 \ + --hash=sha256:83d4a37e4b95da4d8bda930d6d35b75b4cdadbacbb4980cae290ea3100b5d51d \ + --hash=sha256:83e9f4a25085bd4b7214701a0794ff1f50fc633ffb8bdfebf07abdd81c2db126 \ + --hash=sha256:85a18376073f8a39aa07be34f9fc77e2869aa72c55c441efdd2cf79a0407504d \ + --hash=sha256:87796fedc3ba97ec14fab55acb48584276e6c1e4c1e89c422bda62c838e754a9 \ + --hash=sha256:882569ff613758cac762a457a5d72d6e211b28d4bcfea89d1d71ea942b02eac0 \ + --hash=sha256:8ce4d6ccafb33d39bd78444612d14938ead674c25702ded2ee9c54a47735d225 \ + --hash=sha256:8e7d98cdbb6d71e726f7d525952867096053d1f290dd4e3c50d7d313a136f414 \ + --hash=sha256:8ee202350cf57abf0e9502a41601841019c25d3db7ff52d980aaf31446254059 \ + --hash=sha256:8fb0eb4955adf0579001581f2f71a126e8781ba61bcd120f127b0401163c6c2d \ + --hash=sha256:90c30ed53546da833c700115c0064c22120d1b1560f474699fd31f22dd668233 \ + --hash=sha256:9489e6abf47ba37f332075a91444c7cfedb03e6ce99fbb2f116bfe1ce810da3b \ + --hash=sha256:94d7aa6debf92a1dd14cb5280b083a764169a13cfb23a452111160274ed989f4 \ + --hash=sha256:98d370568f393215d605304cdb77b3d5539bd192c75b623c7304c42c8d6d8273 \ + --hash=sha256:9b1bdaae98bc016825dd3c9d8ee1832f829b3341f9cc6ebd1a1b0a7fef7367cc \ + --hash=sha256:9d693bf4bf534e9ba3ae2780cfd577f5135629f7b5ac653490859d0b77864865 \ + --hash=sha256:9d6ed3d17bccce4c05343e1ca8da13bc5c02c812a4e7282ddd05e8769322d3fc \ + --hash=sha256:9e23c82b63cd7652fc24d33ed6cc17099d607aa3b4fc4ddc75e95062f3d82df4 \ + --hash=sha256:a1daf47cd95a7c3a63456336bc5aaa8c86dd3a47d07ed3d0e76132ae4666a5a1 \ + --hash=sha256:a1e32763e641a1566507d90a8d3b19bfc3cc04a9d4e5ae3e32189874ed4b58a3 \ + --hash=sha256:a2059a2d891bd156bc5184e7ab7a56e78a84dfcfdeac8c501b552533ad1c36ee \ + --hash=sha256:a2ed0ba415ccdf08f14bf544cb78346d0f76086707ffee24921a2c84dbf1305a \ + --hash=sha256:a3faadac7d812ddac258feb57b9846b60c1b437c4f4b9ad42595c6f6fe4390df \ + --hash=sha256:a5877f2255aab518ebe528289037699201d5dc5f045f2396cb30aa02db22f57f \ + --hash=sha256:a78b50b4f7918a3de71105d5c0b93bbc57bb8339a4d03a9dfd449f9068e76f3d \ + --hash=sha256:a8c2b841478068440d8b733005d13a5ef535b9928cbc05f17182d410f32ba449 \ + --hash=sha256:a9ca696eb02e5c02a8afd872ada510eba9b7fe6e68b9572c2e9a9b1941e31e2e \ + --hash=sha256:aa4ed3dd308548f9e707d9caaf005d2d7f8c1e7868f858dfeb47fe76e16b391d \ + --hash=sha256:ab2054c5531af2a9ba7b69b8ec91e4f884420e83a8c5e579b013084cb57e5e5d \ + --hash=sha256:abb1384477f5901d436b5d2e5465954de46ea6098f59163d243660b5c4461d35 \ + --hash=sha256:acae6b45d1ace09b6ba3876da43b88366ef368f73b988c7f57e14231753d4420 \ + --hash=sha256:acfa7e22aa6c6e7a5996a41d275bfa01efa7ea56ab890590280e9063e2cf5c1b \ + --hash=sha256:af4ea5b37403ef4e30f3927eaed540db942bde01d8d3ff083527c0704d1c9c68 \ + --hash=sha256:b10dd0557ba422715b5206b3743192135a6022acca8baec51aa127d0a75db8fe \ + --hash=sha256:b13b88747769537f3d32e89e3a735da10c0a9e35d7322928c701b5f93d3afffd \ + --hash=sha256:b51c159a9794633f5e0db7ecec7b2b6e3734eca1f5d17dc989ff3552a43ff78b \ + --hash=sha256:b5402a340723fa7da00b5cff987ddab61276be6d11251ea71ae02bcac54890d8 \ + --hash=sha256:b7abffdf37af1cec6a2ad69b827aa84320db5894791bc8ed932dc93fb274b7e9 \ + --hash=sha256:b8075fe90bc08e40b8b8a1874fab42ee4c7b56af05c5886e9cc841397f916908 \ + --hash=sha256:bc3ac7bf569f6b64dad04dd7808c7872dae8a97df657856eac05e9b7e3614a85 \ + --hash=sha256:bd0912757081f89b107d6c00b2ff8a194401b0b87eadcf4481de2b865a8fd44f \ + --hash=sha256:bdc8d8b8c22e9e43ac68316b5e6cf083dec537f4ec213cb4aa967b583bc3fa64 \ + --hash=sha256:be80550d9bfe83d9b62398a37081a90434e6df2d978ec345c3d2820de6beddab \ + --hash=sha256:c6f117789d22dce188e5754e8bc65b7e6ebf8cb73963b9fa761f672a5883769d \ + --hash=sha256:ca32926d7d77bcc8838425c4c95e040a3ace1cb7dfdae599013458dcda2607ca \ + --hash=sha256:cce0727fd5ac04d372fa9bbfde9febc2bcf209aadfcf0468e45dec72719895d1 \ + --hash=sha256:d0f1489233a254bb3643d2f05de7d59019254d81daeca6b9162fe9edef57e0c7 \ + --hash=sha256:d1c557dfd5e3db046053a0bdc72261ade790ebe8e2c7a41b36b0ca1f14cb95f3 \ + --hash=sha256:d21f0fa80a02d05299207eeaafef345d812ace96d5306e4ef265e1d419a615fa \ + --hash=sha256:d45673badd08456d0340e9364eddafe1c53a9d2896424294de4d7dd71ad3ee57 \ + --hash=sha256:d5add7b4ca7afeea91d52e4d4e4db3b1fe9885b71f07054560d8c4296b7441a2 \ + --hash=sha256:d5f90e44653c4e0f78501ed9bb7d3fce835a8d62b7c6ed0cb16557534087e743 \ + --hash=sha256:d7306dee25b8a0e737363f347362b875094b4dc4e367311470656ae420fdbf8e \ + --hash=sha256:dbcef5a9119ef653653132cccaf999b30a0af6f33bb0a4ba80bec30056868487 \ + --hash=sha256:def538065f9e4d4cf1ae164bd59aba00dfa84f03923e0de4c3788f252d6bcd17 \ + --hash=sha256:df23df54b5114a17c2d0ef192433e2e5a9f0c5178c32375e90b7cfc965f349d0 \ + --hash=sha256:dfbf531053a0935f2e871bcd4753f90313688772ff8c017f5ea402e315a78c1f \ + --hash=sha256:e029648f9c951db30e98a7d7ec90835db88ec4b32820efe2a9bdc2287e032eb6 \ + --hash=sha256:e07595c7d6f4db270ceede356a1bd1c07a34f1c26f958d1ed0cd7b48e0d2bba3 \ + --hash=sha256:e12c538e00e7c1b286a07061046b90e8124e6a9793efae2c70db6a4aad07faad \ + --hash=sha256:e546fe1d4a93ebc2910f0d768baff19faa09843ab3f2036a67ed6e69fae4419d \ + --hash=sha256:e5637ca8d0bd7fb72648a6c7934af4baaccb697657f7438c9d264fc2abb8b0b1 \ + --hash=sha256:e636b64d24fd9c38053c5e389a1174c66361fa49dcfd220f4dd35b4abde7cb89 \ + --hash=sha256:e7011b8fb8c4054bf0c12e5edc6cd83778b0028e99ce59b18586ed036f92cfdc \ + --hash=sha256:e80f557716fd765439577131e526b8942ffc2c07bdbc5e39fa62f660ba1e963f \ + --hash=sha256:e92b6bcc741b86d67606c40d3cb9c7cc8e6c737f81e31f4a94efc204456c92e3 \ + --hash=sha256:eb96ed1ae6c7d072d60840c0434aef07a2df611812810807fbc54263a6053e9a \ + --hash=sha256:eda19ea5ee88742f47a2340816e6f2d40b53bed3ab5b69794769f36af9f35bb4 \ + --hash=sha256:ef74070ac553c59eb4f04258722066d6c6135b7baa03b2e9f2da65c096e96d98 \ + --hash=sha256:efb01a106f971cb3752856bca2318bbdf7f01bd8823779c461586cbe5ffd5258 \ + --hash=sha256:f074e8d4aa0a5798920ddb6de3d08b228c614ff3724c3e8bd7577f4bafea867b \ + --hash=sha256:f38a70074041d3b7e138e452799f5174198bae5bd5ab2000917badf403908c5f \ + --hash=sha256:f41753a76f4f63927d03a0d8ba8f5ce0f2083bec29a8cfaccc55371b1564b96b \ + --hash=sha256:f53dcd26694f148f738edc052b5a69234833e739f10f4c3287bdfd8ec0f7b326 \ + --hash=sha256:f61964f235a43738bfac50da46fc4254943a7eea3051aeb0b6fc7c992c29fadc \ + --hash=sha256:fe01645169a2112aa1d4ebc3e4c5f029c5c8f97adfc32e5d37c993b39a994d75 + # via aiohttp +jsonschema==4.26.0 --hash=sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce +jsonschema-specifications==2025.9.1 --hash=sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe +referencing==0.37.0 --hash=sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231 +rpds-py==2026.6.3 --hash=sha256:db08f45aecde626498fb3df07bcf6d2ec040af42e859a4f5040d79c200342911 diff --git a/Trainers/mlx_sft_mac/runtime-candidate/sft-qualification-proposal.json b/Trainers/mlx_sft_mac/runtime-candidate/sft-qualification-proposal.json index 0955eb02..3bf1150f 100644 --- a/Trainers/mlx_sft_mac/runtime-candidate/sft-qualification-proposal.json +++ b/Trainers/mlx_sft_mac/runtime-candidate/sft-qualification-proposal.json @@ -8,10 +8,10 @@ "python_version": "3.11.14", "mlx_version": "0.32.2", "mlx_lm_version": "0.32.2", - "dependency_lock_sha256": "f9a6298260cd17f2c3d567ae3674108a71efa68e99369906487a4d8c0feb7117", + "dependency_lock_sha256": "7c167018ee6bd61a3609971e3f834082c798e7bd2a3fa0c5c2564887da4fde4f", "hardware": "Apple M4", "physical_memory_bytes": 25769803776, - "capture_requirement": "Exact source, interpreter/framework, stdlib, 50-package candidate inventory, static native dependency graph, startup files, OS build and selected-wheel provenance; no self-issued promotion." + "capture_requirement": "Exact source including shared namespace, private interpreter/framework, stdlib, 54-package combined inventory, static native dependency graph, startup files, OS build and selected-wheel archive provenance; original 50 lock remains unchanged; no self-issued promotion." }, "model": { "repository": "mlx-community/Qwen3-0.6B-4bit", diff --git a/examples/training_qualification/native-host.md b/examples/training_qualification/native-host.md index 2011e131..f639e122 100644 --- a/examples/training_qualification/native-host.md +++ b/examples/training_qualification/native-host.md @@ -75,6 +75,41 @@ composition through `tuner.project`; the existing venv was not modified. Resolve these exact closure/composition blockers and recapture on the final pushed source before any signed execution admission. +The r7 proposal uses a single new owner-private interpreter and one 54-package +environment for host and child. The original 50 lock remains unchanged. The +four-wheel additive lock and exact combined lock are under +`Trainers/mlx_sft_mac/runtime-candidate/`; the combined SHA-256 is +`7c167018ee6bd61a3609971e3f834082c798e7bd2a3fa0c5c2564887da4fde4f`. +The checked-in `native-private-runtime.example.json` binds 54 selected wheel +URLs, filenames, sizes and hashes (146,218,037 bytes), the 20,051,753-byte +Astral interpreter archive, and a new unused private root. These are upstream +metadata selections awaiting actual archive and install readback. + +From the final clean pushed source, the reusable preparation CLI provides: + +```sh +# Future effects, held until separate root review. Copy the exact reviewed +# template to a NEW 0600 config file in this existing 0700 parent first. +private_config=/Users/jrosenbaum/.local/share/synaptic-qualification/native-runtime-r7-reviewed.json +python3 -B examples/training_qualification/native_runtime_preparation.py plan "$private_config" +python3 -B examples/training_qualification/native_runtime_preparation.py acquire "$private_config" +python3 -B examples/training_qualification/native_runtime_preparation.py install "$private_config" +python3 -B examples/training_qualification/native_runtime_preparation.py verify "$private_config" +``` + +Select the CPU host Python used for `python3` explicitly before those commands. +`plan` is local-only. `acquire` hard-bounds archive bytes before each write and +retains them. `install` uses only the exact retained archives, a hashed private +copy of uv, an offline local Python mirror and wheelhouse; both effectful phases +create consumed private claims. Every owned process has a 300-second deadline +and 4 KiB combined output cap. Temporary/cache paths remain under the private +root. Archive expansion is checked before install, but uv's temporary disk use +is not represented as a hard aggregate disk quota; the retained private tree +has a 2 GiB postphase guard. No phase signs admission. +Afterward review observed interpreter/framework/stdlib/packages, dylibs and +source trees. The final capture must include narrow `source/shared` alongside +`source/tuner` and `source/synaptic_tuner`. + Two synthetic publications were made through the existing `ingest` and `prepare-dataset` CLIs, then copied into private Mac staging. Raw/full-sequence publication digest: `0e4951bb2e54ed0985e6ca0d4c9940f0f5d75ccb78821c14544ef44897580cbe`. @@ -92,14 +127,17 @@ The model workspace is empty. Its adjacent `model-acquisition-draft.json` is incomplete by design: `members` and `tokenizer_members` are empty. No model metadata request, weights download or ML import has run. The reusable `native_model_preparation.py` action supports a later separately reviewed -acquisition using exact complete upstream metadata, finite bounds and private -create-only publication. Its observations never sign admission. The next +acquisition using exact complete upstream metadata, a 2 MiB streaming response +cap before Hub `ModelInfo` parsing, and exact prewrite bounded HTTPS transfers +to explicitly precreated 0700 nested destinations. The SDK's unbounded +`snapshot_download` cache/transfer path is not used. An unexpected redirect +host rejects. Its observations never sign admission. The next reviewed sequence is: ```sh -candidate_python=/Users/jrosenbaum/.local/share/synaptic-qualification/native-mlx-candidate-20261020/venv/bin/python +candidate_python=/Users/jrosenbaum/.local/share/synaptic-qualification/native-mlx-private-runtime-r6-20261020/venv/bin/python stage=/Users/jrosenbaum/.local/share/synaptic-qualification/native-mlx-candidate-20261020/prep-r5-aba3c576-20261020 -# After explicit approval for an SDK metadata request, from the final clean source: +# After separate approval for one bounded Hub metadata request, from final clean source: "$candidate_python" -B examples/training_qualification/native_model_preparation.py \ metadata-candidate "$stage/model-acquisition-draft.json" # Independently review every candidate member and complete tokenizer selection; @@ -123,20 +161,21 @@ The following capture/admission/start commands are for a **later reviewed stage**. The current source and runtime closure cannot pass capture yet. ```sh -# From the clean pushed engine checkout, with the existing private 3.11.14 venv. +# From final clean pushed source with the reviewed new private 3.11.14 venv. umask 077 set -C # Every shell redirection below creates a new file or fails. -candidate_python=/Users/jrosenbaum/.local/share/synaptic-qualification/native-mlx-candidate-20261020/venv/bin/python +candidate_python=/Users/jrosenbaum/.local/share/synaptic-qualification/native-mlx-private-runtime-r6-20261020/venv/bin/python private_root=/Users/jrosenbaum/.local/share/synaptic-qualification/native-mlx-candidate-20261020/reviewed-attempt-1 # Create and inspect this fresh owner-only directory before writing any files. mkdir -m 700 "$private_root" -# Prepare capture-config.json with exact stdlib, private package tree, narrow -# engine/tuner source tree, executable/framework and system-library roots. +# Prepare capture-config.json with observed exact stdlib, private package tree, +# narrow source/tuner + source/synaptic_tuner + source/shared trees, and +# executable/framework/system-library roots. # The capture command creates capture.json once and reports OBSERVED_NOT_ADMITTED. "$candidate_python" -B scripts/capture_native_mlx_runtime.py \ --config "$private_root/capture-config.json" \ - --dependency-lock Trainers/mlx_sft_mac/runtime-candidate/requirements.lock \ + --dependency-lock Trainers/mlx_sft_mac/runtime-candidate/requirements.runtime.lock \ --output "$private_root/capture.json" # Run separately for complete, already reviewed private model and tokenizer diff --git a/examples/training_qualification/native_model_preparation.py b/examples/training_qualification/native_model_preparation.py index 0c7ac8e1..3a183082 100644 --- a/examples/training_qualification/native_model_preparation.py +++ b/examples/training_qualification/native_model_preparation.py @@ -9,7 +9,7 @@ from __future__ import annotations import argparse -from contextlib import ExitStack, redirect_stderr, redirect_stdout +from contextlib import ExitStack, contextmanager, redirect_stderr, redirect_stdout import hashlib import json import os @@ -18,6 +18,8 @@ import stat import sys import unicodedata +from urllib.parse import quote, urlsplit +from urllib.request import HTTPRedirectHandler, ProxyHandler, Request, build_opener ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT)) @@ -31,6 +33,10 @@ _ENDPOINT = "https://huggingface.co" _MAX_BYTES = 512 * 1024 * 1024 _MAX_FILES = 128 +_MAX_METADATA_RESPONSE = 2 * 1024 * 1024 +_CHUNK = 64 * 1024 +_REDIRECT_HOSTS = frozenset({"huggingface.co", "cdn-lfs.huggingface.co", + "cas-bridge.xethub.hf.co"}) _MEMBER_FIELDS = {"path", "size", "kind", "upstream_digest"} @@ -141,11 +147,74 @@ def _upstream(member, raw): return hashlib.sha256(raw).hexdigest() +def _url_host(url, allowed): + parsed = urlsplit(url) + if (parsed.scheme != "https" or parsed.hostname not in allowed + or parsed.port not in (None, 443) or parsed.username or parsed.password + or parsed.fragment): + raise ValueError("model HTTPS destination differs") + + +class _RestrictedRedirect(HTTPRedirectHandler): + def __init__(self, hosts, maximum): + self.hosts, self.maximum, self.count = hosts, maximum, 0 + + def redirect_request(self, request, fp, code, msg, headers, newurl): + self.count += 1 + if self.count > self.maximum: + raise ValueError("model redirect count differs") + _url_host(newurl, self.hosts) + return super().redirect_request(request, fp, code, msg, headers, newurl) + + +@contextmanager +def _response(url, *, hosts, redirects, opener=None): + _url_host(url, hosts) + request = Request(url, headers={"Accept-Encoding": "identity", + "User-Agent": "synaptic-native-model-preparation/1"}) + if opener is None: + opener = build_opener(ProxyHandler({}), _RestrictedRedirect(hosts, redirects)).open + with opener(request, timeout=20) as response: + _url_host(response.geturl(), hosts) + if (response.status != 200 or response.headers.get("Content-Encoding", "identity") != "identity"): + raise ValueError("model HTTP response differs") + yield response + + +def _bounded_metadata(config, *, opener=None): + url = (f"{_ENDPOINT}/api/models/{config['repository']}/revision/" + f"{config['revision']}?blobs=true") + with _response(url, hosts=frozenset({"huggingface.co"}), redirects=0, + opener=opener) as response: + declared = response.headers.get("Content-Length") + if declared is not None and (not declared.isdecimal() or int(declared) > _MAX_METADATA_RESPONSE): + raise ValueError("model metadata response exceeds bound") + chunks, count = [], 0 + while True: + chunk = response.read(min(_CHUNK, _MAX_METADATA_RESPONSE + 1 - count)) + if not chunk: + break + count += len(chunk) + if count > _MAX_METADATA_RESPONSE: + raise ValueError("model metadata response exceeds bound") + chunks.append(chunk) + raw = b"".join(chunks) + value = json.loads(raw, object_pairs_hook=_pairs, + parse_constant=lambda _: (_ for _ in ()).throw(ValueError())) + if type(value) is not dict: + raise ValueError("model metadata object differs") + from huggingface_hub.hf_api import ModelInfo + return ModelInfo(**value) + + def _observe_metadata(config, *, api_type): - info = api_type(endpoint=_ENDPOINT, token=False).model_info( - config["repository"], revision=config["revision"], files_metadata=True, - token=False, timeout=30, - ) + if api_type is None: + info = _bounded_metadata(config) + else: + info = api_type(endpoint=_ENDPOINT, token=False).model_info( + config["repository"], revision=config["revision"], files_metadata=True, + token=False, timeout=30, + ) if info.sha != config["revision"] or type(info.siblings) not in (list, tuple): raise ValueError("model metadata revision differs") observed = [] @@ -180,9 +249,6 @@ def _metadata_candidate(config, *, api_type=None): pass else: raise ValueError("metadata candidate already exists") - if api_type is None: - from huggingface_hub import HfApi - api_type = HfApi with open(os.devnull, "w") as sink, redirect_stdout(sink), redirect_stderr(sink): observed = _observe_metadata(config, api_type=api_type) names = tuple(item["path"] for item in observed) @@ -225,6 +291,66 @@ def _publish_member(root, name, raw): root.check() +@contextmanager +def _member_parent(root, name): + parts = relative_member(name) + with ExitStack() as stack: + parent = root.fd + for part in parts[:-1]: + try: + os.mkdir(part, mode=0o700, dir_fd=parent) + except FileExistsError: + pass + fd = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent) + stack.callback(os.close, fd) + info = os.fstat(fd) + if info.st_uid != os.getuid() or not stat.S_ISDIR(info.st_mode) or info.st_mode & 0o077: + raise ValueError("model SDK destination parent is not private") + parent = fd + root.check() + yield parent, parts[-1] + root.check() + + +def _fetch_member(config, member, root, *, remaining, opener=None): + """One no-retry, no-cache HTTPS transfer; stop before any excess byte is written.""" + if remaining < member["size"]: + raise ValueError("model aggregate transfer exceeds bound") + name = member["path"] + expected_url = (f"{_ENDPOINT}/{config['repository']}/resolve/{config['revision']}/" + f"{quote(name, safe='/')}") + from huggingface_hub import hf_hub_url + url = hf_hub_url(config["repository"], name, revision=config["revision"], endpoint=_ENDPOINT) + if url != expected_url: + raise ValueError("model Hub resolve URL differs") + with _member_parent(root, name) as (parent, leaf): + fd = os.open(leaf, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, dir_fd=parent) + size = 0 + digest = hashlib.sha256() if member["kind"] == "lfs" else hashlib.sha1() + if member["kind"] == "git": + digest.update(f"blob {member['size']}\0".encode()) + with os.fdopen(fd, "wb") as output: + with _response(url, hosts=_REDIRECT_HOSTS, redirects=2, opener=opener) as response: + declared = response.headers.get("Content-Length") + if declared is not None and (not declared.isdecimal() or int(declared) != member["size"]): + raise ValueError("model transfer declared size differs") + while True: + chunk = response.read(min(_CHUNK, member["size"] + 1 - size)) + if not chunk: + break + size += len(chunk) + if size > member["size"] or size > remaining: + raise ValueError("model transfer exceeds admitted bytes") + output.write(chunk) + digest.update(chunk) + output.flush() + os.fsync(output.fileno()) + if size != member["size"] or digest.hexdigest() != member["upstream_digest"]: + raise ValueError("model transfer identity differs") + return size + + def _acquire(config, *, api_type=None, download=None): previous = os.umask(0o077) try: @@ -239,26 +365,40 @@ def _acquire_private(config, *, api_type=None, download=None): with NativeTree(workspace, private=True) as root: os.mkdir("scratch", mode=0o700, dir_fd=root.fd) root.check() - if api_type is None or download is None: - from huggingface_hub import HfApi, snapshot_download - api_type = api_type or HfApi - download = download or snapshot_download with open(os.devnull, "w") as sink, redirect_stdout(sink), redirect_stderr(sink): _metadata(members, config, api_type=api_type) scratch = workspace / "scratch" with NativeTree(scratch, private=True) as held: os.mkdir("repository", mode=0o700, dir_fd=held.fd) + os.mkdir("sdk-cache", mode=0o700, dir_fd=held.fd) held.check() repository = scratch / "repository" - with open(os.devnull, "w") as sink, redirect_stdout(sink), redirect_stderr(sink): - result = download( - repo_id=config["repository"], revision=config["revision"], token=False, - endpoint=_ENDPOINT, local_dir=repository, cache_dir=scratch / "sdk-cache", - allow_patterns=[item["path"] for item in members], max_workers=1, - etag_timeout=10, - ) - if Path(result) != repository: - raise ValueError("model SDK destination differs") + with NativeTree(repository, private=True) as source, NativeTree(scratch / "sdk-cache", private=True) as cache: + # Every selected nested destination exists at 0700 before a transport can write. + for member in members: + with _member_parent(source, member["path"]): + pass + cache.check() + if download is None: + transferred = 0 + with open(os.devnull, "w") as sink, redirect_stdout(sink), redirect_stderr(sink): + for member in members: + transferred += _fetch_member(config, member, source, + remaining=config["maximum_bytes"] - transferred) + else: + # Injection is used by CPU publication fixtures only. The public CLI + # always uses the bounded transfer above. + with open(os.devnull, "w") as sink, redirect_stdout(sink), redirect_stderr(sink): + result = download( + repo_id=config["repository"], revision=config["revision"], token=False, + endpoint=_ENDPOINT, local_dir=repository, cache_dir=scratch / "sdk-cache", + allow_patterns=[item["path"] for item in members], max_workers=1, + etag_timeout=10, + ) + if Path(result) != repository: + raise ValueError("model SDK destination differs") + source.check() + cache.check() bytes_by_name = {} content_hashes = {} with NativeTree(repository, private=True) as source: @@ -323,7 +463,9 @@ def main(argv=None): else: environment = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "LC_ALL": "C", "HF_HOME": config["workspace_root"], - "HF_HUB_DISABLE_TELEMETRY": "1", "TMPDIR": config["workspace_root"]} + "HF_HUB_DISABLE_TELEMETRY": "1", "HF_HUB_DISABLE_IMPLICIT_TOKEN": "1", + "XDG_CACHE_HOME": config["workspace_root"], + "TMPDIR": config["workspace_root"]} status, output = run_bounded_command( [sys.executable, "-I", "-B", str(Path(__file__).resolve()), "_metadata_worker" if args.action == "metadata-candidate" else "_worker", @@ -332,8 +474,10 @@ def main(argv=None): ) if status != 0: raise ValueError("model acquisition worker uncertain") - worker_result = json.loads(output, object_pairs_hook=_pairs) - if _canonical(worker_result) != output: + if not output.endswith(b"\n") or output.endswith(b"\n\n"): + raise ValueError("model acquisition worker framing differs") + worker_result = json.loads(output[:-1], object_pairs_hook=_pairs) + if _canonical(worker_result) + b"\n" != output: raise ValueError("model acquisition worker result differs") if args.action == "metadata-candidate": with NativeTree(Path(config["workspace_root"]), private=True) as root: diff --git a/examples/training_qualification/native_runtime_preparation.py b/examples/training_qualification/native_runtime_preparation.py new file mode 100644 index 00000000..8a79ecd9 --- /dev/null +++ b/examples/training_qualification/native_runtime_preparation.py @@ -0,0 +1,491 @@ +#!/usr/bin/env python3 +"""Prepare an observed private native MLX runtime from exact retained archives. + +This workflow never grants native execution authority. `plan` is read-only; +`acquire` and `install` consume distinct create-only claims. Network acquisition +has a hard byte limit before every write. Installation reads only exact local +archives under an owned, private root and has a finite owned-process deadline. +""" + +from __future__ import annotations + +import argparse +from contextlib import contextmanager +import hashlib +import io +import json +import os +from pathlib import Path, PurePosixPath +import re +import stat +import sys +import tarfile +import zipfile + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from examples.training_qualification.bounded_command import run_bounded_command +from native_model_preparation import _response +from tuner.runtime.native_files import NativeTree, file_identity + + +BASE = ROOT / "Trainers/mlx_sft_mac/runtime-candidate" +_SCHEMA = "synaptic-native-private-runtime-preparation/v1" +_ORIGINAL_SHA = "f9a6298260cd17f2c3d567ae3674108a71efa68e99369906487a4d8c0feb7117" +_ARCHIVE_SHA = "d5f10f6fdc6f4afb5d251578edb89d8c0ece8f9f56e38b0019e2a94e8f0f4852" +_ARCHIVE_SIZE = 20051753 +_UV_SHA = "c87143aea2480b09f107a9f97fa93899d116f0307b4d210649a735e141bdd889" +_UV_BANNER = b"uv 0.12.23 (Homebrew 2026-10-03 aarch64-apple-darwin)" +_EXTRA = frozenset({"jsonschema", "jsonschema-specifications", "referencing", "rpds-py"}) +_CHUNK = 64 * 1024 +_MAX_CONFIG = 64 * 1024 +_MAX_WHEELS = 54 + + +def _canonical(value): + return json.dumps(value, sort_keys=True, separators=(",", ":"), + ensure_ascii=False, allow_nan=False).encode() + + +def _pairs(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("native runtime duplicate config field") + result[key] = value + return result + + +def _load(path): + path = Path(path) + if not path.is_absolute() or str(path) != os.path.normpath(str(path)): + raise ValueError("native runtime config path differs") + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + before = os.fstat(fd) + if (not stat.S_ISREG(before.st_mode) or before.st_uid != os.getuid() + or before.st_mode & 0o022 or not 0 < before.st_size <= _MAX_CONFIG): + raise ValueError("native runtime config file differs") + raw = os.read(fd, _MAX_CONFIG + 1) + if (len(raw) != before.st_size or file_identity(before) != file_identity(os.fstat(fd)) + or file_identity(before) != file_identity(path.lstat())): + raise ValueError("native runtime config changed") + finally: + os.close(fd) + return json.loads(raw, object_pairs_hook=_pairs, + parse_constant=lambda _: (_ for _ in ()).throw(ValueError())) + + +def _lock_bytes(): + original = (BASE / "requirements.lock").read_bytes() + extra = (BASE / "requirements.host-additions.lock").read_bytes() + combined = (BASE / "requirements.runtime.lock").read_bytes() + if (hashlib.sha256(original).hexdigest() != _ORIGINAL_SHA + or combined != original + extra): + raise ValueError("native runtime original or combined lock differs") + return original, extra, combined + + +def _retained_locks(root): + expected = _lock_bytes() + with NativeTree(root, private=True) as tree: + actual = tuple(tree.read(name, maximum=1 << 20) for name in + ("original.lock", "host-additions.lock", "runtime.lock")) + if actual != expected: + raise ValueError("native runtime retained lock bytes differ") + + +def _lock_packages(raw): + text = raw.decode("ascii") + matches = list(re.finditer(r"(?m)^([A-Za-z0-9][A-Za-z0-9_.-]*)==([^\s\\]+)", text)) + result = {} + for index, match in enumerate(matches): + name = re.sub(r"[-_.]+", "-", match.group(1)).lower() + block = text[match.end():matches[index + 1].start() if index + 1 < len(matches) else len(text)] + hashes = frozenset(re.findall(r"--hash=sha256:([0-9a-f]{64})", block)) + if name in result or not hashes: + raise ValueError("native runtime package lock differs") + result[name] = (match.group(2), hashes) + return result + + +def _config(value): + fields = {"schema", "status", "private_root", "uv_executable", "uv_sha256", "uv_version", + "python_archive", "python_install_key", "original_lock_sha256", + "additions_lock_sha256", "runtime_lock_sha256", "wheel_archive_bytes", + "maximum_worker_seconds", "maximum_command_output_bytes", + "maximum_python_unpacked_bytes", "maximum_wheel_unpacked_bytes", + "maximum_private_tree_bytes", "wheels"} + if (type(value) is not dict or set(value) != fields or value["schema"] != _SCHEMA + or value["status"] != "REVIEW_REQUIRED_NO_EXECUTION_AUTHORITY"): + raise ValueError("native runtime configuration differs") + private = value["private_root"] + if (type(private) is not str or not Path(private).is_absolute() + or private != os.path.normpath(private) or Path(private).name.startswith(".")): + raise ValueError("native runtime private path differs") + if (value["uv_executable"] != "/opt/homebrew/bin/uv" + or value["uv_sha256"] != _UV_SHA + or value["uv_version"] != "0.12.23" + or value["python_install_key"] != "cpython-3.11.14-macos-aarch64-none"): + raise ValueError("native runtime tool selection differs") + archive = value["python_archive"] + if (type(archive) is not dict or set(archive) != {"url", "filename", "size", "sha256"} + or archive["url"] != "https://github.com/astral-sh/python-build-standalone/releases/download/20260211/cpython-3.11.14%2B20260211-aarch64-apple-darwin-install_only_stripped.tar.gz" + or archive["filename"] != "cpython-3.11.14+20260211-aarch64-apple-darwin-install_only_stripped.tar.gz" + or archive["size"] != _ARCHIVE_SIZE or archive["sha256"] != _ARCHIVE_SHA): + raise ValueError("native runtime interpreter archive differs") + original, extra, combined = _lock_bytes() + if (value["original_lock_sha256"] != _ORIGINAL_SHA + or value["additions_lock_sha256"] != hashlib.sha256(extra).hexdigest() + or value["runtime_lock_sha256"] != hashlib.sha256(combined).hexdigest()): + raise ValueError("native runtime lock digest differs") + original_packages, extra_packages = _lock_packages(original), _lock_packages(extra) + if (len(original_packages) != 50 or set(extra_packages) != _EXTRA + or set(original_packages) & set(extra_packages)): + raise ValueError("native runtime locked inventory differs") + packages = {**original_packages, **extra_packages} + wheels = value["wheels"] + if type(wheels) is not list or len(wheels) != _MAX_WHEELS: + raise ValueError("native runtime wheel count differs") + seen = set() + total = 0 + for wheel in wheels: + if type(wheel) is not dict or set(wheel) != {"package", "version", "filename", "sha256", "size", "url"}: + raise ValueError("native runtime wheel fields differ") + name = wheel["package"] + if (type(name) is not str or name in seen or name not in packages + or type(wheel["version"]) is not str + or (wheel["version"], wheel["sha256"]) not in { + (packages[name][0], digest) for digest in packages[name][1] + }): + raise ValueError("native runtime wheel lock selection differs") + filename = wheel["filename"] + if (type(filename) is not str or "/" in filename or not filename.endswith(".whl") + or not filename.lower().startswith(name.replace("-", "_") + "-" + wheel["version"] + "-") + or type(wheel["size"]) is not int or not 1 <= wheel["size"] <= 128 * 1024 * 1024 + or type(wheel["url"]) is not str + or not wheel["url"].startswith("https://files.pythonhosted.org/packages/") + or not wheel["url"].endswith("/" + filename)): + raise ValueError("native runtime wheel archive selection differs") + seen.add(name) + total += wheel["size"] + if (seen != set(packages) or total != value["wheel_archive_bytes"] + or total != 146218037 or type(value["maximum_worker_seconds"]) is not int + or not 1 <= value["maximum_worker_seconds"] <= 300 + or value["maximum_command_output_bytes"] != 4096 + or value["maximum_python_unpacked_bytes"] != 536870912 + or value["maximum_wheel_unpacked_bytes"] != 1073741824 + or value["maximum_private_tree_bytes"] != 2147483648): + raise ValueError("native runtime complete archive budget differs") + return Path(private), packages + + +def _root_layout(root): + with NativeTree(root.parent, private=True) as parent: + os.mkdir(root.name, mode=0o700, dir_fd=parent.fd) + parent.check() + with NativeTree(root, private=True) as tree: + for name in ("python-archive", "wheelhouse", "uv-cache", "tmp", "cache", "tools"): + os.mkdir(name, mode=0o700, dir_fd=tree.fd) + tree.check() + with NativeTree(root / "python-archive", private=True) as tree: + os.mkdir("20260211", mode=0o700, dir_fd=tree.fd) + tree.check() + + +def _stream_archive(url, root, filename, size, digest, *, hosts, redirects): + hasher = hashlib.sha256() + with NativeTree(root, private=True) as tree: + fd = os.open(filename, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, dir_fd=tree.fd) + count = 0 + with os.fdopen(fd, "wb") as output: + with _response(url, hosts=hosts, redirects=redirects) as response: + declared = response.headers.get("Content-Length") + if declared is not None and (not declared.isdecimal() or int(declared) != size): + raise ValueError("native runtime archive declared size differs") + while True: + chunk = response.read(min(_CHUNK, size + 1 - count)) + if not chunk: + break + count += len(chunk) + if count > size: + raise ValueError("native runtime archive transfer exceeds bound") + output.write(chunk) + hasher.update(chunk) + output.flush() + os.fsync(output.fileno()) + tree.check() + if count != size or hasher.hexdigest() != digest: + raise ValueError("native runtime archive bytes differ") + + +def _archive_read(root, name, size, digest): + with NativeTree(root, private=True) as tree: + raw = tree.read(name, maximum=size) + if len(raw) != size or hashlib.sha256(raw).hexdigest() != digest: + raise ValueError("native runtime retained archive differs") + return raw + + +def _private_tree_bytes(root, maximum): + """Reject retained excess after a phase; this is not a transient quota.""" + total, count = 0, 0 + for directory, folders, files in os.walk(root, followlinks=False): + for name in (*folders, *files): + info = os.lstat(Path(directory) / name) + count += 1 + total += info.st_size + if count > 100000 or total > maximum: + raise ValueError("native runtime private tree budget exceeded") + return total + + +def _archives(value, root): + archive = value["python_archive"] + python = _archive_read(root / "python-archive/20260211", archive["filename"], + archive["size"], archive["sha256"]) + with tarfile.open(fileobj=io.BytesIO(python), mode="r:gz") as tar: + members = tar.getmembers() + if (len(members) > 16384 + or sum(member.size for member in members) > value["maximum_python_unpacked_bytes"] + or any(member.name.startswith("/") or ".." in PurePosixPath(member.name).parts + for member in members)): + raise ValueError("native runtime Python archive expansion differs") + expanded = 0 + for wheel in value["wheels"]: + raw = _archive_read(root / "wheelhouse", wheel["filename"], + wheel["size"], wheel["sha256"]) + with zipfile.ZipFile(io.BytesIO(raw)) as zipped: + infos = zipped.infolist() + if len(infos) > 100000 or any( + item.filename.startswith("/") or ".." in PurePosixPath(item.filename).parts + for item in infos + ): + raise ValueError("native runtime wheel members differ") + expanded += sum(item.file_size for item in infos) + if expanded > value["maximum_wheel_unpacked_bytes"]: + raise ValueError("native runtime wheel expansion exceeds bound") + with NativeTree(root / "wheelhouse", private=True) as tree: + observed = tree.inventory(maximum_total=value["wheel_archive_bytes"]) + if {item.path for item in observed} != {item["filename"] for item in value["wheels"]}: + raise ValueError("native runtime wheelhouse has extra or missing archives") + return expanded + + +def _copy_uv(value, root): + source = Path(value["uv_executable"]) + with source.open("rb") as input_file: + raw = input_file.read(64 * 1024 * 1024 + 1) + if len(raw) > 64 * 1024 * 1024 or hashlib.sha256(raw).hexdigest() != value["uv_sha256"]: + raise ValueError("native runtime uv executable bytes differ") + with NativeTree(root / "tools", private=True) as tree: + fd = os.open("uv", os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o500, dir_fd=tree.fd) + with os.fdopen(fd, "wb") as output: + output.write(raw) + output.flush() + os.fsync(output.fileno()) + tree.check() + + +def _environment(root): + return {"PATH": "/usr/bin:/bin", "LC_ALL": "C", "TMPDIR": str(root / "tmp"), + "UV_CACHE_DIR": str(root / "uv-cache"), "XDG_CACHE_HOME": str(root / "cache"), + "UV_PYTHON_INSTALL_REGISTRY": "0", "UV_NO_PROGRESS": "1", "UV_NO_CACHE": "1"} + + +@contextmanager +def _private_environment(root): + selected = _environment(root) + prior = {key: os.environ.get(key) for key in selected} + os.environ.update(selected) + try: + yield + finally: + for key, value in prior.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +def _command(argv, root, value): + status, output = run_bounded_command( + argv, cwd=str(root), environment=_environment(root), + maximum=value["maximum_command_output_bytes"], timeout=value["maximum_worker_seconds"], + ) + if status: + raise ValueError("native runtime owned command failed") + _private_tree_bytes(root, value["maximum_private_tree_bytes"]) + return output + + +def _acquire(value, root): + _root_layout(root) + with NativeTree(root, private=True) as tree: + for name, raw in zip(("original.lock", "host-additions.lock", "runtime.lock"), + _lock_bytes(), strict=True): + tree.write_exclusive(name, raw) + with _private_environment(root): + archive = value["python_archive"] + _stream_archive(archive["url"], root / "python-archive/20260211", + archive["filename"], archive["size"], archive["sha256"], + hosts=frozenset({"github.com", "release-assets.githubusercontent.com", + "objects.githubusercontent.com"}), redirects=2) + transferred = archive["size"] + for wheel in value["wheels"]: + if transferred + wheel["size"] > archive["size"] + value["wheel_archive_bytes"]: + raise ValueError("native runtime aggregate transfer exceeds bound") + _stream_archive(wheel["url"], root / "wheelhouse", wheel["filename"], + wheel["size"], wheel["sha256"], + hosts=frozenset({"files.pythonhosted.org"}), redirects=0) + transferred += wheel["size"] + expanded = _archives(value, root) + _copy_uv(value, root) + _private_tree_bytes(root, value["maximum_private_tree_bytes"]) + receipt = {"schema": "synaptic-native-private-runtime-archives/v1", + "status": "OBSERVED_NOT_ADMITTED", "selection_sha256": hashlib.sha256(_canonical(value)).hexdigest(), + "archive_bytes": transferred, "wheel_count": len(value["wheels"]), + "wheel_unpacked_bytes": expanded, + "runtime_lock_sha256": value["runtime_lock_sha256"]} + with NativeTree(root, private=True) as tree: + tree.write_exclusive("archive-result.json", _canonical(receipt)) + return receipt + + +def _installed(value, root, packages): + python = root / "venv/bin/python" + code = ("import importlib.metadata as m,json,platform,sys,sysconfig;from pathlib import Path;" + "items=[(x.metadata['Name'].lower().replace('_','-'),x.version) for x in m.distributions()];" + "print(json.dumps({'version':list(sys.version_info[:3]),'machine':platform.machine()," + "'executable':str(Path(sys.executable).resolve()),'stdlib':str(Path(sysconfig.get_path('stdlib')).resolve())," + "'distributions':items},sort_keys=True,separators=(',',':')))" ) + raw = _command([str(python), "-I", "-B", "-c", code], root, value) + observed = json.loads(raw, object_pairs_hook=_pairs) + items = observed["distributions"] + if (observed["version"] != [3, 11, 14] or observed["machine"] != "arm64" + or len(items) != 54 or len({name for name, _ in items}) != 54 + or dict(items) != {name: version for name, (version, _) in packages.items()}): + raise ValueError("native runtime installed distribution inventory differs") + executable, stdlib = Path(observed["executable"]), Path(observed["stdlib"]) + managed = root / "uv-python" / value["python_install_key"] + if (not executable.is_relative_to(managed) or not stdlib.is_relative_to(managed) + or not executable.is_file() or not stdlib.is_dir()): + raise ValueError("native runtime private interpreter path differs") + with NativeTree(executable.parent) as tree: + executable_sha256 = hashlib.sha256(tree.read(executable.name)).hexdigest() + return {"python_version": "3.11.14", "distribution_count": len(items), + "inventory_sha256": hashlib.sha256(_canonical(sorted(items))).hexdigest(), + "executable": str(executable), "executable_sha256": executable_sha256, + "stdlib": str(stdlib)} + + +def _install(value, root, packages): + _retained_locks(root) + with NativeTree(root, private=True) as tree: + archive_receipt = json.loads(tree.read("archive-result.json", maximum=4096)) + if (archive_receipt.get("status") != "OBSERVED_NOT_ADMITTED" + or archive_receipt.get("selection_sha256") != hashlib.sha256(_canonical(value)).hexdigest() + or archive_receipt.get("runtime_lock_sha256") != value["runtime_lock_sha256"]): + raise ValueError("native runtime archive receipt differs") + tree.write_exclusive("install-claim.json", _canonical({ + "schema": "synaptic-native-runtime-install-claim/v1", "status": "CONSUMED_UNCERTAIN", + "selection_sha256": archive_receipt["selection_sha256"]})) + expanded = _archives(value, root) + uv = root / "tools/uv" + if hashlib.sha256(uv.read_bytes()).hexdigest() != value["uv_sha256"]: + raise ValueError("native runtime private uv differs") + if _command([str(uv), "--version"], root, value).strip() != _UV_BANNER: + raise ValueError("native runtime uv version differs") + _command([str(uv), "--no-config", "--offline", "python", "install", "--install-dir", + str(root / "uv-python"), "--no-bin", "--mirror", + (root / "python-archive").as_uri(), "cpython@3.11.14"], root, value) + managed = root / "uv-python" / value["python_install_key"] / "bin/python3.11" + if not managed.is_file(): + raise ValueError("native runtime private interpreter absent") + _command([str(uv), "--no-config", "venv", "--python", str(managed), + "--no-python-downloads", str(root / "venv")], root, value) + _command([str(uv), "--no-config", "--offline", "pip", "sync", "--python", + str(root / "venv/bin/python"), "--no-index", "--find-links", str(root / "wheelhouse"), + "--require-hashes", "--only-binary", ":all:", "--link-mode", "copy", + str(root / "runtime.lock")], root, value) + _command([str(uv), "--no-config", "--offline", "pip", "check", "--python", + str(root / "venv/bin/python")], root, value) + result = _installed(value, root, packages) + result.update({"schema": "synaptic-native-private-runtime-install/v1", + "status": "OBSERVED_NOT_ADMITTED", "selection_sha256": hashlib.sha256(_canonical(value)).hexdigest(), + "wheel_unpacked_bytes": expanded}) + with NativeTree(root, private=True) as tree: + tree.write_exclusive("install-result.json", _canonical(result)) + return result + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("plan", "acquire", "install", "verify", "_acquire_worker")) + parser.add_argument("config", type=Path) + args = parser.parse_args(argv) + try: + value = _load(args.config) + root, packages = _config(value) + if args.action in {"acquire", "install", "_acquire_worker"}: + with NativeTree(args.config.parent, private=True): + pass + if args.action == "plan": + result = {"status": "REVIEW_REQUIRED_NO_EXECUTION_AUTHORITY", + "selection_sha256": hashlib.sha256(_canonical(value)).hexdigest(), + "wheel_count": 54, "wheel_archive_bytes": value["wheel_archive_bytes"], + "total_archive_bytes": value["wheel_archive_bytes"] + _ARCHIVE_SIZE, + "runtime_lock_sha256": value["runtime_lock_sha256"]} + elif args.action == "_acquire_worker": + prior_umask = os.umask(0o077) + try: + result = _acquire(value, root) + finally: + os.umask(prior_umask) + elif args.action == "install": + # uv commands own separate process groups. Run this phase in the + # caller so a timed-out outer worker cannot orphan an inner uv. + prior_umask = os.umask(0o077) + try: + result = _install(value, root, packages) + finally: + os.umask(prior_umask) + with NativeTree(root, private=True) as tree: + retained = json.loads(tree.read("install-result.json", maximum=4096), + object_pairs_hook=_pairs) + if retained != result: + raise ValueError("native runtime retained install result differs") + elif args.action == "verify": + _retained_locks(root) + _archives(value, root) + result = _installed(value, root, packages) + result["status"] = "OBSERVED_NOT_ADMITTED" + else: + status, raw = run_bounded_command( + [sys.executable, "-I", "-B", str(Path(__file__).resolve()), "_acquire_worker", str(args.config)], + environment={"PATH": "/usr/bin:/bin", "LC_ALL": "C"}, + maximum=value["maximum_command_output_bytes"], timeout=value["maximum_worker_seconds"], + ) + if status or not raw.endswith(b"\n") or raw.endswith(b"\n\n"): + raise ValueError("native runtime worker uncertain") + result = json.loads(raw[:-1], object_pairs_hook=_pairs) + if _canonical(result) + b"\n" != raw: + raise ValueError("native runtime worker framing differs") + with NativeTree(root, private=True) as tree: + retained = json.loads(tree.read("archive-result.json", maximum=4096), + object_pairs_hook=_pairs) + if retained != result: + raise ValueError("native runtime retained result differs") + print(_canonical(result).decode()) + return 0 + except Exception: + print('{"diagnostic_code":"native_runtime_preparation_refused"}') + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/cli/test_native_model_preparation.py b/tests/cli/test_native_model_preparation.py index 2c1a29d4..8c36c689 100644 --- a/tests/cli/test_native_model_preparation.py +++ b/tests/cli/test_native_model_preparation.py @@ -1,15 +1,18 @@ """Private native model preparation contracts; no SDK network or ML imports.""" from hashlib import sha1 +import io import json import os from pathlib import Path +import sys from types import SimpleNamespace import tempfile import pytest from examples.training_qualification import native_model_preparation as preparation +from examples.training_qualification.bounded_command import run_bounded_command def _git(raw): @@ -64,9 +67,12 @@ def download(**options): assert options["max_workers"] == 1 and options["etag_timeout"] == 10 assert options["allow_patterns"] == sorted(files) root = Path(options["local_dir"]) + cache = Path(options["cache_dir"]) + assert cache.is_dir() and cache.stat().st_mode & 0o077 == 0 for name, raw in files.items(): target = root / name - target.parent.mkdir(parents=True, exist_ok=True) + assert target.parent.is_dir() + assert target.parent.stat().st_mode & 0o077 == 0 if redirect == name: target.symlink_to(root / "config.json") else: @@ -95,7 +101,8 @@ def test_model_plan_is_local_and_fake_sdk_publishes_only_exact_bytes(candidate, assert calls == ["metadata", "download"] -def test_nested_model_publication_stays_private_under_ambient_group_umask(candidate): +@pytest.mark.parametrize("ambient", [0o022, 0o002]) +def test_nested_model_publication_stays_private_under_ambient_group_umask(candidate, ambient): config, files, root = candidate files["weights/model.safetensors"] = files.pop("model.safetensors") config["members"] = [ @@ -103,7 +110,7 @@ def test_nested_model_publication_stays_private_under_ambient_group_umask(candid for name, raw in sorted(files.items()) ] api, download, calls = _sdk(config, files) - previous = os.umask(0o002) + previous = os.umask(ambient) try: preparation._acquire(config, api_type=api, download=download) finally: @@ -113,6 +120,107 @@ def test_nested_model_publication_stays_private_under_ambient_group_umask(candid assert (root / "model-snapshot/weights").stat().st_mode & 0o077 == 0 +class _Response(io.BytesIO): + status = 200 + + def __init__(self, body, url, *, headers=None): + super().__init__(body) + self.url = url + self.headers = headers or {} + + def geturl(self): + return self.url + + +def test_metadata_body_limit_rejects_before_json_parse(candidate): + config, _, _ = candidate + seen = [] + oversized = preparation._canonical({ + "id": config["repository"], "sha": config["revision"], "siblings": [], + "cardData": "x" * preparation._MAX_METADATA_RESPONSE, + }) + assert len(oversized) > preparation._MAX_METADATA_RESPONSE + + def open_url(request, timeout): + seen.append((request.full_url, timeout)) + return _Response(oversized, request.full_url) + + with pytest.raises(ValueError, match="response exceeds bound"): + preparation._bounded_metadata(config, opener=open_url) + assert len(seen) == 1 and seen[0][0].endswith("?blobs=true") + + +def test_bounded_hub_metadata_is_parsed_after_complete_stream(candidate): + config, files, _ = candidate + body = preparation._canonical({ + "id": config["repository"], "sha": config["revision"], + "siblings": [{"rfilename": name, "size": len(raw), "blobId": _git(raw)} + for name, raw in sorted(files.items())], + }) + + def open_url(request, timeout): + assert request.get_header("Accept-encoding") == "identity" + return _Response(body, request.full_url, headers={"Content-Length": str(len(body))}) + + info = preparation._bounded_metadata(config, opener=open_url) + assert info.sha == config["revision"] + assert [sibling.rfilename for sibling in info.siblings] == sorted(files) + + +def test_model_transfer_stops_before_overflow_write_and_retains_claim(candidate): + config, files, workspace = candidate + member = next(item for item in config["members"] if item["path"] == "model.safetensors") + repository = workspace / "repository" + repository.mkdir(mode=0o700) + + def open_url(request, timeout): + return _Response(files["model.safetensors"] + b"excess", request.full_url) + + with preparation.NativeTree(repository, private=True) as root: + with pytest.raises(ValueError, match="exceeds admitted bytes"): + preparation._fetch_member(config, member, root, remaining=512, opener=open_url) + assert (repository / "model.safetensors").read_bytes() == b"" + assert not (workspace / "model-snapshot").exists() + + +def test_interrupted_transfer_retains_scratch_and_rejects_replay(candidate): + config, files, root = candidate + api, _, calls = _sdk(config, files) + + def interrupted(**options): + calls.append("download") + (Path(options["local_dir"]) / "config.json").write_bytes(b"{") + raise OSError("untrusted network detail") + + with pytest.raises(OSError): + preparation._acquire(config, api_type=api, download=interrupted) + assert calls == ["metadata", "download"] + assert (root / "scratch/repository/config.json").read_bytes() == b"{" + assert not (root / "model-snapshot").exists() + with pytest.raises(ValueError, match="prior acquisition claim"): + preparation._acquire(config, api_type=api, download=interrupted) + assert calls == ["metadata", "download"] + + +def test_real_child_newline_is_accepted_by_parent_readback(candidate, monkeypatch, capsys): + config, files, root = candidate + config["members"] = [] + config["tokenizer_members"] = [] + (root / "config.json").write_bytes(preparation._canonical(config)) + api, _, _ = _sdk(config, files) + result = preparation._metadata_candidate(config, api_type=api) + + def real_child(_argv, **_kwargs): + body = preparation._canonical(result).decode() + return run_bounded_command([sys.executable, "-c", "print(" + repr(body) + ")"], + environment={"PATH": os.environ.get("PATH", "/usr/bin:/bin")}, + maximum=4096, timeout=5) + + monkeypatch.setattr(preparation, "run_bounded_command", real_child) + assert preparation.main(["metadata-candidate", str(root / "config.json")]) == 0 + assert json.loads(capsys.readouterr().out) == result + + def test_model_metadata_substitution_rejects_before_download(candidate): config, files, root = candidate config["maximum_files"] = 4 diff --git a/tests/cli/test_native_runtime_preparation.py b/tests/cli/test_native_runtime_preparation.py new file mode 100644 index 00000000..5c50fd59 --- /dev/null +++ b/tests/cli/test_native_runtime_preparation.py @@ -0,0 +1,189 @@ +"""Private runtime proposal and archive transport CPU contracts.""" + +from contextlib import contextmanager +import hashlib +import io +import json +import os +from pathlib import Path +import sys +import tempfile +import tarfile +import zipfile + +import pytest + +from examples.training_qualification import native_runtime_preparation as preparation +from examples.training_qualification.bounded_command import BoundedCommandError + + +@pytest.fixture +def proposal(): + config = json.loads((preparation.BASE / "native-private-runtime.example.json").read_bytes()) + previous = os.umask(0o077) + try: + with tempfile.TemporaryDirectory(prefix="native-runtime-r7-", dir=Path.home()) as name: + parent = Path(name) + config["private_root"] = str(parent / "new-runtime") + path = parent / "proposal.json" + path.write_bytes(preparation._canonical(config)) + yield config, path, parent + finally: + os.umask(previous) + + +def test_complete_wheel_selection_and_lock_are_exact(proposal, capsys): + config, path, _ = proposal + root, packages = preparation._config(config) + assert root == Path(config["private_root"]) + assert len(packages) == 54 + assert len(config["wheels"]) == 54 + assert sum(item["size"] for item in config["wheels"]) == 146218037 + assert config["original_lock_sha256"] == preparation._ORIGINAL_SHA + assert preparation.main(["plan", str(path)]) == 0 + result = json.loads(capsys.readouterr().out) + assert result["total_archive_bytes"] == 166269790 + assert result["runtime_lock_sha256"] == config["runtime_lock_sha256"] + + +@pytest.mark.parametrize("change", [ + lambda x: x["wheels"].pop(), + lambda x: x["wheels"][0].update(size=x["wheels"][0]["size"] + 1), + lambda x: x["wheels"][0].update(sha256="0" * 64), + lambda x: x["wheels"][0].update(url="https://unreviewed.invalid/wheel.whl"), + lambda x: x.update(uv_sha256="0" * 64), +]) +def test_unreviewed_runtime_selection_rejects_before_claim(proposal, change): + config, _, parent = proposal + change(config) + with pytest.raises(ValueError): + preparation._config(config) + assert not (parent / "new-runtime").exists() + + +class _Response(io.BytesIO): + status = 200 + + def __init__(self, raw): + super().__init__(raw) + self.headers = {} + + +def test_archive_stream_excess_byte_is_never_written(proposal, monkeypatch): + _, _, parent = proposal + target = parent / "wheelhouse" + target.mkdir(mode=0o700) + + @contextmanager + def response(_url, **_kwargs): + with _Response(b"123456") as opened: + yield opened + + monkeypatch.setattr(preparation, "_response", response) + with pytest.raises(ValueError, match="transfer exceeds bound"): + preparation._stream_archive("https://files.pythonhosted.org/packages/a.whl", target, + "a.whl", 5, "0" * 64, + hosts=frozenset({"files.pythonhosted.org"}), redirects=0) + assert (target / "a.whl").read_bytes() == b"" + + +def test_retained_archive_inventory_rejects_extra_wheel(proposal): + config, _, parent = proposal + root = parent / "new-runtime" + preparation._root_layout(root) + tar_buffer = io.BytesIO() + with tarfile.open(fileobj=tar_buffer, mode="w:gz") as tar: + item = tarfile.TarInfo("python/bin/python3.11") + item.size = 1 + tar.addfile(item, io.BytesIO(b"p")) + python = tar_buffer.getvalue() + (root / "python-archive/20260211/python.tar.gz").write_bytes(python) + zip_buffer = io.BytesIO() + with zipfile.ZipFile(zip_buffer, "w") as zipped: + zipped.writestr("package/module.py", "x=1") + wheel = zip_buffer.getvalue() + (root / "wheelhouse/package-1-py3-none-any.whl").write_bytes(wheel) + config["python_archive"] = {"filename": "python.tar.gz", "size": len(python), + "sha256": hashlib.sha256(python).hexdigest()} + config["wheels"] = [{"filename": "package-1-py3-none-any.whl", "size": len(wheel), + "sha256": hashlib.sha256(wheel).hexdigest()}] + config["wheel_archive_bytes"] = len(wheel) + assert preparation._archives(config, root) == len(b"x=1") + (root / "wheelhouse/extra.whl").write_bytes(b"extra") + with pytest.raises(ValueError): + preparation._archives(config, root) + + +def test_failed_acquisition_keeps_claimed_private_root(proposal, monkeypatch): + config, _, parent = proposal + calls = [] + + def fail(*args, **kwargs): + calls.append("transfer") + raise ValueError("bounded transfer failed") + + monkeypatch.setattr(preparation, "_stream_archive", fail) + with pytest.raises(ValueError, match="bounded transfer failed"): + preparation._acquire(config, parent / "new-runtime") + root = parent / "new-runtime" + assert calls == ["transfer"] and root.is_dir() and root.stat().st_mode & 0o077 == 0 + assert (root / "python-archive/20260211").is_dir() + with pytest.raises(FileExistsError): + preparation._acquire(config, root) + assert calls == ["transfer"] + + +def test_retained_private_tree_budget_is_explicit_postphase_guard(proposal): + _, _, parent = proposal + (parent / "large").write_bytes(b"x" * 64) + with pytest.raises(ValueError, match="private tree budget"): + preparation._private_tree_bytes(parent, 16) + + +def test_install_claim_precedes_archive_validation(proposal, monkeypatch): + config, _, parent = proposal + root = parent / "new-runtime" + preparation._root_layout(root) + with preparation.NativeTree(root, private=True) as tree: + for name, raw in zip(("original.lock", "host-additions.lock", "runtime.lock"), + preparation._lock_bytes(), strict=True): + tree.write_exclusive(name, raw) + tree.write_exclusive("archive-result.json", preparation._canonical({ + "status": "OBSERVED_NOT_ADMITTED", + "selection_sha256": hashlib.sha256(preparation._canonical(config)).hexdigest(), + "runtime_lock_sha256": config["runtime_lock_sha256"], + })) + + def fail(*_args): + raise ValueError("archive bytes missing") + + monkeypatch.setattr(preparation, "_archives", fail) + with pytest.raises(ValueError, match="archive bytes missing"): + preparation._install(config, root, preparation._config(config)[1]) + assert json.loads((root / "install-claim.json").read_bytes())["status"] == "CONSUMED_UNCERTAIN" + with pytest.raises(FileExistsError): + preparation._install(config, root, preparation._config(config)[1]) + + +def test_mutated_retained_lock_rejects_before_install_claim(proposal): + config, _, parent = proposal + root = parent / "new-runtime" + preparation._root_layout(root) + with preparation.NativeTree(root, private=True) as tree: + for name, raw in zip(("original.lock", "host-additions.lock", "runtime.lock"), + preparation._lock_bytes(), strict=True): + tree.write_exclusive(name, raw) + (root / "runtime.lock").write_bytes(b"different") + with pytest.raises(ValueError, match="retained lock bytes"): + preparation._install(config, root, preparation._config(config)[1]) + assert not (root / "install-claim.json").exists() + + +def test_owned_command_flood_and_deadline_are_bounded(proposal): + config, _, parent = proposal + with pytest.raises(BoundedCommandError): + preparation._command([sys.executable, "-c", "import sys;sys.stdout.write('x'*1048576)"], + parent, config) + short = {**config, "maximum_worker_seconds": 0.05} + with pytest.raises(BoundedCommandError): + preparation._command([sys.executable, "-c", "import time;time.sleep(2)"], parent, short) From 1c0c6b25d4a014da511748bd8ac532e8572aebed Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:19:48 -0400 Subject: [PATCH 06/12] Reject incompatible native wheels before private preparation --- .../mlx_sft_mac/runtime-candidate/README.md | 3 +- .../native-private-runtime.example.json | 7 ++++- .../native_runtime_preparation.py | 31 ++++++++++++++++++- tests/cli/test_native_model_preparation.py | 21 +++++++++++++ tests/cli/test_native_runtime_preparation.py | 12 +++++++ 5 files changed, 71 insertions(+), 3 deletions(-) diff --git a/Trainers/mlx_sft_mac/runtime-candidate/README.md b/Trainers/mlx_sft_mac/runtime-candidate/README.md index d4fd87cd..60d5fad6 100644 --- a/Trainers/mlx_sft_mac/runtime-candidate/README.md +++ b/Trainers/mlx_sft_mac/runtime-candidate/README.md @@ -106,7 +106,8 @@ The checked-in [`native-private-runtime.example.json`](native-private-runtime.example.json) declares the exact Astral CPython 3.11.14+20260211 arm64 archive (20,051,753 bytes, SHA-256 `d5f10f6fdc6f4afb5d251578edb89d8c0ece8f9f56e38b0019e2a94e8f0f4852`), -uv 0.12.23 executable hash, every selected wheel filename/URL/size/hash and an +the observed Darwin arm64/macOS 27.0.1 target, uv 0.12.23 executable hash, +every selected compatible wheel filename/URL/size/hash and an owner-private **unused** output root. The 54 wheel archives total 146,218,037 bytes; all retained download archives total 166,269,790 bytes. These are official upstream metadata selections awaiting actual archive readback. diff --git a/Trainers/mlx_sft_mac/runtime-candidate/native-private-runtime.example.json b/Trainers/mlx_sft_mac/runtime-candidate/native-private-runtime.example.json index 43996057..cfa9c049 100644 --- a/Trainers/mlx_sft_mac/runtime-candidate/native-private-runtime.example.json +++ b/Trainers/mlx_sft_mac/runtime-candidate/native-private-runtime.example.json @@ -454,5 +454,10 @@ "url": "https://files.pythonhosted.org/packages/ce/7c/fbb40fe2d53747c40aa36a9e2bd2178a202f942bda0b670f3306d4aefbce/yarl-1.25.1-cp311-cp311-macosx_11_0_arm64.whl" } ], - "maximum_private_tree_bytes": 2147483648 + "maximum_private_tree_bytes": 2147483648, + "target": { + "system": "Darwin", + "machine": "arm64", + "macos": "27.0.1" + } } diff --git a/examples/training_qualification/native_runtime_preparation.py b/examples/training_qualification/native_runtime_preparation.py index 8a79ecd9..6214b2a8 100644 --- a/examples/training_qualification/native_runtime_preparation.py +++ b/examples/training_qualification/native_runtime_preparation.py @@ -16,6 +16,7 @@ import json import os from pathlib import Path, PurePosixPath +import platform import re import stat import sys @@ -111,8 +112,32 @@ def _lock_packages(raw): return result +def _compatible_wheel(filename, version): + stem = filename[:-4] + pieces = stem.rsplit("-", 3) + if len(pieces) != 4: + return False + _, python_tag, abi_tag, platform_tag = pieces + if platform_tag == "any": + return abi_tag == "none" and python_tag in {"py3", "py2.py3", "py311"} + match = re.fullmatch(r"macosx_(\d+)_(\d+)_(arm64|universal2)", platform_tag) + if not match or (int(match.group(1)), int(match.group(2))) > version[:2]: + return False + if python_tag == "cp311": + return abi_tag in {"cp311", "abi3"} + if python_tag in {"cp38", "cp39", "cp310"}: + return abi_tag == "abi3" + return python_tag == "py3" and abi_tag == "none" + + +def _host_target(): + if (platform.system() != "Darwin" or platform.machine() != "arm64" + or platform.mac_ver()[0] != "27.0.1"): + raise ValueError("native runtime Mac target differs") + + def _config(value): - fields = {"schema", "status", "private_root", "uv_executable", "uv_sha256", "uv_version", + fields = {"schema", "status", "private_root", "target", "uv_executable", "uv_sha256", "uv_version", "python_archive", "python_install_key", "original_lock_sha256", "additions_lock_sha256", "runtime_lock_sha256", "wheel_archive_bytes", "maximum_worker_seconds", "maximum_command_output_bytes", @@ -121,6 +146,8 @@ def _config(value): if (type(value) is not dict or set(value) != fields or value["schema"] != _SCHEMA or value["status"] != "REVIEW_REQUIRED_NO_EXECUTION_AUTHORITY"): raise ValueError("native runtime configuration differs") + if value["target"] != {"system": "Darwin", "machine": "arm64", "macos": "27.0.1"}: + raise ValueError("native runtime target selection differs") private = value["private_root"] if (type(private) is not str or not Path(private).is_absolute() or private != os.path.normpath(private) or Path(private).name.startswith(".")): @@ -164,6 +191,7 @@ def _config(value): filename = wheel["filename"] if (type(filename) is not str or "/" in filename or not filename.endswith(".whl") or not filename.lower().startswith(name.replace("-", "_") + "-" + wheel["version"] + "-") + or not _compatible_wheel(filename, (27, 0, 1)) or type(wheel["size"]) is not int or not 1 <= wheel["size"] <= 128 * 1024 * 1024 or type(wheel["url"]) is not str or not wheel["url"].startswith("https://files.pythonhosted.org/packages/") @@ -432,6 +460,7 @@ def main(argv=None): value = _load(args.config) root, packages = _config(value) if args.action in {"acquire", "install", "_acquire_worker"}: + _host_target() with NativeTree(args.config.parent, private=True): pass if args.action == "plan": diff --git a/tests/cli/test_native_model_preparation.py b/tests/cli/test_native_model_preparation.py index 8c36c689..34fc8049 100644 --- a/tests/cli/test_native_model_preparation.py +++ b/tests/cli/test_native_model_preparation.py @@ -1,5 +1,6 @@ """Private native model preparation contracts; no SDK network or ML imports.""" +from contextlib import contextmanager from hashlib import sha1 import io import json @@ -202,6 +203,26 @@ def interrupted(**options): assert calls == ["metadata", "download"] +def test_real_bounded_transfer_overflow_retains_claim_without_publication(candidate, monkeypatch): + config, files, root = candidate + api, _, calls = _sdk(config, files) + + @contextmanager + def overflow(request_url, **_options): + with _Response(b"{}x", request_url) as response: + yield response + + monkeypatch.setattr(preparation, "_response", overflow) + with pytest.raises(ValueError, match="exceeds admitted bytes"): + preparation._acquire(config, api_type=api) + assert calls == ["metadata"] + assert (root / "scratch/repository/config.json").read_bytes() == b"" + assert not (root / "model-snapshot").exists() + with pytest.raises(ValueError, match="prior acquisition claim"): + preparation._acquire(config, api_type=api) + assert calls == ["metadata"] + + def test_real_child_newline_is_accepted_by_parent_readback(candidate, monkeypatch, capsys): config, files, root = candidate config["members"] = [] diff --git a/tests/cli/test_native_runtime_preparation.py b/tests/cli/test_native_runtime_preparation.py index 5c50fd59..cf4fdad2 100644 --- a/tests/cli/test_native_runtime_preparation.py +++ b/tests/cli/test_native_runtime_preparation.py @@ -38,6 +38,10 @@ def test_complete_wheel_selection_and_lock_are_exact(proposal, capsys): assert root == Path(config["private_root"]) assert len(packages) == 54 assert len(config["wheels"]) == 54 + assert all(preparation._compatible_wheel(item["filename"], (27, 0, 1)) + for item in config["wheels"]) + assert not preparation._compatible_wheel("mlx-0.32.2-cp311-cp311-manylinux_x86_64.whl", + (27, 0, 1)) assert sum(item["size"] for item in config["wheels"]) == 146218037 assert config["original_lock_sha256"] == preparation._ORIGINAL_SHA assert preparation.main(["plan", str(path)]) == 0 @@ -61,6 +65,14 @@ def test_unreviewed_runtime_selection_rejects_before_claim(proposal, change): assert not (parent / "new-runtime").exists() +def test_effectful_cli_rejects_wrong_host_before_private_claim(proposal, capsys, monkeypatch): + _, path, parent = proposal + monkeypatch.setattr(preparation.platform, "system", lambda: "Linux") + assert preparation.main(["acquire", str(path)]) == 2 + assert capsys.readouterr().out == '{"diagnostic_code":"native_runtime_preparation_refused"}\n' + assert not (parent / "new-runtime").exists() + + class _Response(io.BytesIO): status = 200 From ecc1b18723ad969d4ae253bcbad866dfc519b049 Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:50:20 -0400 Subject: [PATCH 07/12] Seal exact private uv lock files after native runtime install --- .agents/skills/fine-tuning/SKILL.md | 2 +- .../reference/native-local-training.md | 14 ++ .claude/skills/fine-tuning/SKILL.md | 2 +- .../reference/native-local-training.md | 14 ++ .skills/fine-tuning/SKILL.md | 2 +- .../reference/native-local-training.md | 14 ++ AGENTS.md | 6 + .../mlx_sft_mac/runtime-candidate/README.md | 13 + .../training_qualification/native-host.md | 13 + .../native_runtime_preparation.py | 179 +++++++++++++- tests/cli/test_native_runtime_preparation.py | 223 ++++++++++++++++++ 11 files changed, 473 insertions(+), 9 deletions(-) diff --git a/.agents/skills/fine-tuning/SKILL.md b/.agents/skills/fine-tuning/SKILL.md index 9c54c10b..4ce10f76 100644 --- a/.agents/skills/fine-tuning/SKILL.md +++ b/.agents/skills/fine-tuning/SKILL.md @@ -59,7 +59,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | | Native MLX restricted host consumer | `python -B examples/training_qualification/native_host.py inspect ` is read-only; exact-run `start` requires an externally signed root admission and separately supplied public-key pin. No live Mac qualification yet; see `reference/native-local-training.md` | | Capture reviewed native runtime bytes | `python -B scripts/capture_native_mlx_runtime.py --config --dependency-lock --output ` — observation only; explicit roots, dependencies and original pins required; see `reference/native-local-training.md` | -| Plan private native Mac runtime | `python -B examples/training_qualification/native_runtime_preparation.py plan ` — local 54-wheel archive plan only; later `acquire`/`install` consume separate private claims and need independent review; see `reference/native-local-training.md` | +| Plan private native Mac runtime | `python -B examples/training_qualification/native_runtime_preparation.py plan ` — local 54-wheel archive plan only; later `acquire`/`install` consume separate private claims. `seal-uv-locks` is a separately reviewed one-use custody repair for an exact completed install; see `reference/native-local-training.md` | | Prepare a reviewed private native model snapshot | `python -B examples/training_qualification/native_model_preparation.py inspect ` is local-only; `metadata-candidate` performs one bounded SDK metadata request and `acquire` downloads only after separate exact inventory review. Neither grants training admission; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.agents/skills/fine-tuning/reference/native-local-training.md b/.agents/skills/fine-tuning/reference/native-local-training.md index e1de1f0f..943bb800 100644 --- a/.agents/skills/fine-tuning/reference/native-local-training.md +++ b/.agents/skills/fine-tuning/reference/native-local-training.md @@ -248,6 +248,20 @@ private-tree guard runs after each phase. Include narrow `source/tuner` and `source/synaptic_tuner`. Do not chmod Homebrew or reuse the old venv/capture attempt. +Pinned uv 0.12.23 was observed creating empty owner-owned `uv-python/.lock` +and `venv/.lock` at mode 0666 despite umask 077 inside the private 0700 +runtime. The checked-in fresh `install` seals only those exact regular, +zero-byte, single-link files to 0600 after the last uv call and checks the +installed tree before its result. For the already completed r9 install, +`seal-uv-locks ` is a separate, one-use repair action +requiring independent review and exact retained source/config/archive/uv/ +install-record admission. It writes a consumed repair claim before either +descriptor mode change; interruption or identity drift retains the claim and +partial evidence without automatic replay. It preserves the original install +claim/result. `verify` rejects incomplete repair records and writable members. +Do not run static capture until custody passes; the capture must bind the +final pushed source, including `shared`, and the combined 54-package lock. + The checked-in `examples/training_qualification/native_model_preparation.py` is the later reviewed model-acquisition action: `inspect` is local, `metadata-candidate` reads at most 2 MiB of the exact Hub metadata response diff --git a/.claude/skills/fine-tuning/SKILL.md b/.claude/skills/fine-tuning/SKILL.md index 9c54c10b..4ce10f76 100644 --- a/.claude/skills/fine-tuning/SKILL.md +++ b/.claude/skills/fine-tuning/SKILL.md @@ -59,7 +59,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | | Native MLX restricted host consumer | `python -B examples/training_qualification/native_host.py inspect ` is read-only; exact-run `start` requires an externally signed root admission and separately supplied public-key pin. No live Mac qualification yet; see `reference/native-local-training.md` | | Capture reviewed native runtime bytes | `python -B scripts/capture_native_mlx_runtime.py --config --dependency-lock --output ` — observation only; explicit roots, dependencies and original pins required; see `reference/native-local-training.md` | -| Plan private native Mac runtime | `python -B examples/training_qualification/native_runtime_preparation.py plan ` — local 54-wheel archive plan only; later `acquire`/`install` consume separate private claims and need independent review; see `reference/native-local-training.md` | +| Plan private native Mac runtime | `python -B examples/training_qualification/native_runtime_preparation.py plan ` — local 54-wheel archive plan only; later `acquire`/`install` consume separate private claims. `seal-uv-locks` is a separately reviewed one-use custody repair for an exact completed install; see `reference/native-local-training.md` | | Prepare a reviewed private native model snapshot | `python -B examples/training_qualification/native_model_preparation.py inspect ` is local-only; `metadata-candidate` performs one bounded SDK metadata request and `acquire` downloads only after separate exact inventory review. Neither grants training admission; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.claude/skills/fine-tuning/reference/native-local-training.md b/.claude/skills/fine-tuning/reference/native-local-training.md index e1de1f0f..943bb800 100644 --- a/.claude/skills/fine-tuning/reference/native-local-training.md +++ b/.claude/skills/fine-tuning/reference/native-local-training.md @@ -248,6 +248,20 @@ private-tree guard runs after each phase. Include narrow `source/tuner` and `source/synaptic_tuner`. Do not chmod Homebrew or reuse the old venv/capture attempt. +Pinned uv 0.12.23 was observed creating empty owner-owned `uv-python/.lock` +and `venv/.lock` at mode 0666 despite umask 077 inside the private 0700 +runtime. The checked-in fresh `install` seals only those exact regular, +zero-byte, single-link files to 0600 after the last uv call and checks the +installed tree before its result. For the already completed r9 install, +`seal-uv-locks ` is a separate, one-use repair action +requiring independent review and exact retained source/config/archive/uv/ +install-record admission. It writes a consumed repair claim before either +descriptor mode change; interruption or identity drift retains the claim and +partial evidence without automatic replay. It preserves the original install +claim/result. `verify` rejects incomplete repair records and writable members. +Do not run static capture until custody passes; the capture must bind the +final pushed source, including `shared`, and the combined 54-package lock. + The checked-in `examples/training_qualification/native_model_preparation.py` is the later reviewed model-acquisition action: `inspect` is local, `metadata-candidate` reads at most 2 MiB of the exact Hub metadata response diff --git a/.skills/fine-tuning/SKILL.md b/.skills/fine-tuning/SKILL.md index 9c54c10b..4ce10f76 100644 --- a/.skills/fine-tuning/SKILL.md +++ b/.skills/fine-tuning/SKILL.md @@ -59,7 +59,7 @@ Train language models with SFT, KTO, and GRPO locally or on supported cloud prov | Diagnose an offline SFT worker rejection | Read the exact retained `trainer.stderr.log`; closed phase/category/milestone labels locate a boundary, not a cause or retry authority; see `reference/sft-training.md` | | Native MLX restricted host consumer | `python -B examples/training_qualification/native_host.py inspect ` is read-only; exact-run `start` requires an externally signed root admission and separately supplied public-key pin. No live Mac qualification yet; see `reference/native-local-training.md` | | Capture reviewed native runtime bytes | `python -B scripts/capture_native_mlx_runtime.py --config --dependency-lock --output ` — observation only; explicit roots, dependencies and original pins required; see `reference/native-local-training.md` | -| Plan private native Mac runtime | `python -B examples/training_qualification/native_runtime_preparation.py plan ` — local 54-wheel archive plan only; later `acquire`/`install` consume separate private claims and need independent review; see `reference/native-local-training.md` | +| Plan private native Mac runtime | `python -B examples/training_qualification/native_runtime_preparation.py plan ` — local 54-wheel archive plan only; later `acquire`/`install` consume separate private claims. `seal-uv-locks` is a separately reviewed one-use custody repair for an exact completed install; see `reference/native-local-training.md` | | Prepare a reviewed private native model snapshot | `python -B examples/training_qualification/native_model_preparation.py inspect ` is local-only; `metadata-candidate` performs one bounded SDK metadata request and `acquire` downloads only after separate exact inventory review. Neither grants training admission; see `reference/native-local-training.md` | | Plan a derived training image | `python scripts/qualify_derived_training_image.py plan --config Trainers/image_profiles/.yaml` | | Warm Space scaffold | `python3 Trainers/cloud/scripts/manage_space.py render --template vllm_warm --output-dir /tmp/my-space --base-image ghcr.io//:` | diff --git a/.skills/fine-tuning/reference/native-local-training.md b/.skills/fine-tuning/reference/native-local-training.md index e1de1f0f..943bb800 100644 --- a/.skills/fine-tuning/reference/native-local-training.md +++ b/.skills/fine-tuning/reference/native-local-training.md @@ -248,6 +248,20 @@ private-tree guard runs after each phase. Include narrow `source/tuner` and `source/synaptic_tuner`. Do not chmod Homebrew or reuse the old venv/capture attempt. +Pinned uv 0.12.23 was observed creating empty owner-owned `uv-python/.lock` +and `venv/.lock` at mode 0666 despite umask 077 inside the private 0700 +runtime. The checked-in fresh `install` seals only those exact regular, +zero-byte, single-link files to 0600 after the last uv call and checks the +installed tree before its result. For the already completed r9 install, +`seal-uv-locks ` is a separate, one-use repair action +requiring independent review and exact retained source/config/archive/uv/ +install-record admission. It writes a consumed repair claim before either +descriptor mode change; interruption or identity drift retains the claim and +partial evidence without automatic replay. It preserves the original install +claim/result. `verify` rejects incomplete repair records and writable members. +Do not run static capture until custody passes; the capture must bind the +final pushed source, including `shared`, and the combined 54-package lock. + The checked-in `examples/training_qualification/native_model_preparation.py` is the later reviewed model-acquisition action: `inspect` is local, `metadata-candidate` reads at most 2 MiB of the exact Hub metadata response diff --git a/AGENTS.md b/AGENTS.md index 39441996..d636bac3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -51,6 +51,12 @@ This repository has a few cloud-training constraints that are easy to relearn th durable claims, child lifecycle and native artifact verification host-owned. Do not enable native training in Syntunia before those joins and real Mac qualification are complete. See `docs/plans/native-desktop-training-api-v1.md`. +- The private Mac runtime install may leave pinned uv's two exact empty + `uv-python/.lock` and `venv/.lock` files mode 0666 despite umask 077. Fresh + installs must seal those files before a successful result. An existing + consumed install requires the separately reviewed `seal-uv-locks` action; + preserve its claim and partial modes on interruption. Never chmod the tree + broadly, reset install claims, or capture before custody passes. - For Mac qualification preparation, preserve the original 50-package MLX lock and use the reviewed four-wheel additive/flat 54-package lock for the current host and child import graph. Prepare a new owner-private interpreter/runtime diff --git a/Trainers/mlx_sft_mac/runtime-candidate/README.md b/Trainers/mlx_sft_mac/runtime-candidate/README.md index 60d5fad6..305de66d 100644 --- a/Trainers/mlx_sft_mac/runtime-candidate/README.md +++ b/Trainers/mlx_sft_mac/runtime-candidate/README.md @@ -132,6 +132,19 @@ bounded before installation and retained private-tree observations reject over quota on uv's transient filesystem use. A failed acquisition or install keeps its consumed private state; choose a new reviewed root for another attempt. +Pinned uv 0.12.23 was observed creating empty owner-owned `uv-python/.lock` +and `venv/.lock` files with mode 0666 despite umask 077. Fresh `install` now +checks those exact regular, zero-byte, single-link files after the last uv +call and seals only them to 0600 before publishing `install-result.json`. +The separate `seal-uv-locks` action is for an already completed exact install: +it rechecks retained archives, uv bytes, prior claim/result and installed +inventory, then consumes a create-only repair claim before touching either +held file descriptor. An interruption retains the claim and any partial modes; +do not rerun or delete it. `verify` rejects an incomplete repair record or a +writable installed member. The action changes no archive, package, prior +claim/result or Homebrew file. The r9 install still needs a separately +reviewed repair before static capture. + The future static capture must use this new private interpreter and the `requirements.runtime.lock` bytes. Its narrow source list must include `source/shared` as well as `source/tuner` and `source/synaptic_tuner`; the r5 diff --git a/examples/training_qualification/native-host.md b/examples/training_qualification/native-host.md index f639e122..4f7b11c7 100644 --- a/examples/training_qualification/native-host.md +++ b/examples/training_qualification/native-host.md @@ -160,6 +160,19 @@ tokenizer loadability before wiring them into the host config. The following capture/admission/start commands are for a **later reviewed stage**. The current source and runtime closure cannot pass capture yet. +The r9 offline install and metadata-only `verify` succeeded with a private +Python 3.11.14 executable and 54 locked distributions. Full custody review +found only two empty uv lock files, `uv-python/.lock` and `venv/.lock`, at mode +0666. The prior install claim/result remain consumed and unchanged. After +independent review of the final pushed source and the exact retained r9 +records, a single `native_runtime_preparation.py seal-uv-locks +` may seal those two held files to 0600 and retain a +create-only repair claim/result. A failed or interrupted repair cannot be +replayed automatically. Run checked-in `verify` and inspect exact custody +afterward. Do not run capture from the stale r5 config: it selects Homebrew, +omits `shared`, and binds the original 50-package lock. Prepare a fresh +capture config against the final pushed source and combined 54-package lock. + ```sh # From final clean pushed source with the reviewed new private 3.11.14 venv. umask 077 diff --git a/examples/training_qualification/native_runtime_preparation.py b/examples/training_qualification/native_runtime_preparation.py index 6214b2a8..b5746bc8 100644 --- a/examples/training_qualification/native_runtime_preparation.py +++ b/examples/training_qualification/native_runtime_preparation.py @@ -2,7 +2,8 @@ """Prepare an observed private native MLX runtime from exact retained archives. This workflow never grants native execution authority. `plan` is read-only; -`acquire` and `install` consume distinct create-only claims. Network acquisition +`acquire`, `install` and a completed-install uv-lock repair consume distinct +create-only claims. Network acquisition has a hard byte limit before every write. Installation reads only exact local archives under an owned, private root and has a finite owned-process deadline. """ @@ -10,7 +11,7 @@ from __future__ import annotations import argparse -from contextlib import contextmanager +from contextlib import ExitStack, contextmanager import hashlib import io import json @@ -43,6 +44,7 @@ _CHUNK = 64 * 1024 _MAX_CONFIG = 64 * 1024 _MAX_WHEELS = 54 +_UV_LOCK_PARENTS = ("uv-python", "venv") def _canonical(value): @@ -351,6 +353,154 @@ def _command(argv, root, value): return output +def _installed_tree_custody(root, maximum): + """Check installed entries after uv exits; uv may create writable lock files.""" + with NativeTree(root, private=True) as tree: + count = total = 0 + for directory, folders, files in os.walk(root, followlinks=False): + for name in (*folders, *files): + path = Path(directory) / name + info = path.lstat() + count += 1 + total += info.st_size + if count > 100000 or total > maximum or info.st_uid != os.getuid(): + raise ValueError("native runtime installed tree custody differs") + if stat.S_ISLNK(info.st_mode): + target = os.readlink(path) + lexical = Path(os.path.normpath(os.path.join(path.parent, target))) + if (not lexical.is_relative_to(root) + or not path.resolve(strict=True).is_relative_to(root)): + raise ValueError("native runtime installed link escapes") + elif (not (stat.S_ISDIR(info.st_mode) or stat.S_ISREG(info.st_mode)) + or info.st_mode & 0o022): + raise ValueError("native runtime installed member mode differs") + tree.check() + + +def _seal_uv_locks(root, *, repair_claim=None): + """Hold both exact uv locks before changing either; interrupted repair stays claimed.""" + with ExitStack() as stack: + root_tree = stack.enter_context(NativeTree(root, private=True)) + held = [] + for parent_name in _UV_LOCK_PARENTS: + parent = stack.enter_context(NativeTree(root / parent_name, private=True)) + fd = os.open(".lock", os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, + dir_fd=parent.fd) + stack.callback(os.close, fd) + info = os.fstat(fd) + current = os.stat(".lock", dir_fd=parent.fd, follow_symlinks=False) + if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() + or info.st_nlink != 1 or info.st_size != 0 + or stat.S_IMODE(info.st_mode) not in ({0o666} if repair_claim else {0o600, 0o666}) + or file_identity(info) != file_identity(current)): + raise ValueError("native runtime uv lock custody differs") + held.append((parent, fd, info)) + if repair_claim is None and len({stat.S_IMODE(info.st_mode) for _, _, info in held}) != 1: + raise ValueError("native runtime uv lock modes differ") + root_tree.check() + if repair_claim is not None: + root_tree.write_exclusive("uv-lock-repair-claim.json", _canonical(repair_claim)) + for parent, fd, before in held: + current = os.stat(".lock", dir_fd=parent.fd, follow_symlinks=False) + if file_identity(before) != file_identity(os.fstat(fd)) or file_identity(before) != file_identity(current): + raise ValueError("native runtime uv lock changed") + if stat.S_IMODE(before.st_mode) != 0o600: + os.fchmod(fd, 0o600) + os.fsync(fd) + after = os.fstat(fd) + current = os.stat(".lock", dir_fd=parent.fd, follow_symlinks=False) + if (file_identity(after) != file_identity(current) + or (after.st_dev, after.st_ino) != (before.st_dev, before.st_ino) + or not stat.S_ISREG(after.st_mode) or after.st_uid != os.getuid() + or after.st_nlink != 1 or after.st_size != 0 + or stat.S_IMODE(after.st_mode) != 0o600): + raise ValueError("native runtime sealed uv lock differs") + parent.check() + root_tree.check() + + +def _repair_uv_locks(value, root, packages): + """One-use seal for an already completed exact install; no uv/install replay.""" + with NativeTree(root, private=True) as tree: + for name in ("uv-lock-repair-claim.json", "uv-lock-repair-result.json"): + try: + os.stat(name, dir_fd=tree.fd, follow_symlinks=False) + except FileNotFoundError: + continue + raise ValueError("native runtime uv lock repair slot is consumed") + _retained_locks(root) + expanded = _archives(value, root) + selection = hashlib.sha256(_canonical(value)).hexdigest() + archive_bytes = value["python_archive"]["size"] + value["wheel_archive_bytes"] + with NativeTree(root, private=True) as tree: + archive = json.loads(tree.read("archive-result.json", maximum=4096), object_pairs_hook=_pairs) + claim = json.loads(tree.read("install-claim.json", maximum=4096), object_pairs_hook=_pairs) + original = tree.read("install-result.json", maximum=4096) + if archive != {"schema": "synaptic-native-private-runtime-archives/v1", + "status": "OBSERVED_NOT_ADMITTED", "selection_sha256": selection, + "archive_bytes": archive_bytes, "wheel_count": len(value["wheels"]), + "wheel_unpacked_bytes": expanded, + "runtime_lock_sha256": value["runtime_lock_sha256"]}: + raise ValueError("native runtime archive receipt differs") + if claim != {"schema": "synaptic-native-runtime-install-claim/v1", + "status": "CONSUMED_UNCERTAIN", "selection_sha256": selection}: + raise ValueError("native runtime install claim differs") + with NativeTree(root / "tools", private=True) as tools: + if hashlib.sha256(tools.read("uv", maximum=64 * 1024 * 1024)).hexdigest() != value["uv_sha256"]: + raise ValueError("native runtime private uv differs") + installed = _installed(value, root, packages) + expected = {**installed, "schema": "synaptic-native-private-runtime-install/v1", + "status": "OBSERVED_NOT_ADMITTED", "selection_sha256": selection, + "wheel_unpacked_bytes": expanded} + if original != _canonical(expected): + raise ValueError("native runtime retained install result differs") + original_sha = hashlib.sha256(original).hexdigest() + repair_claim = {"schema": "synaptic-native-uv-lock-repair-claim/v1", + "status": "CONSUMED_UNCERTAIN", "selection_sha256": selection, + "install_result_sha256": original_sha} + _seal_uv_locks(root, repair_claim=repair_claim) + _installed_tree_custody(root, value["maximum_private_tree_bytes"]) + result = {"schema": "synaptic-native-uv-lock-repair/v1", + "status": "OBSERVED_NOT_ADMITTED", "selection_sha256": selection, + "install_result_sha256": original_sha, + "sealed_members": ["uv-python/.lock", "venv/.lock"]} + with NativeTree(root, private=True) as tree: + tree.write_exclusive("uv-lock-repair-result.json", _canonical(result)) + if tree.read("install-result.json", maximum=4096) != original: + raise ValueError("native runtime prior install result changed") + return result + + +def _check_uv_repair_record(root, selection): + with NativeTree(root, private=True) as tree: + def exists(name): + try: + os.stat(name, dir_fd=tree.fd, follow_symlinks=False) + except FileNotFoundError: + return False + return True + + claim_exists = exists("uv-lock-repair-claim.json") + result_exists = exists("uv-lock-repair-result.json") + if bool(claim_exists) != bool(result_exists): + raise ValueError("native runtime uv lock repair is uncertain") + if claim_exists: + claim = json.loads(tree.read("uv-lock-repair-claim.json", maximum=4096), object_pairs_hook=_pairs) + result = json.loads(tree.read("uv-lock-repair-result.json", maximum=4096), object_pairs_hook=_pairs) + original = tree.read("install-result.json", maximum=4096) + digest = hashlib.sha256(original).hexdigest() + if (claim != {"schema": "synaptic-native-uv-lock-repair-claim/v1", + "status": "CONSUMED_UNCERTAIN", + "selection_sha256": selection, + "install_result_sha256": digest} + or result != {"schema": "synaptic-native-uv-lock-repair/v1", + "status": "OBSERVED_NOT_ADMITTED", + "selection_sha256": selection, + "install_result_sha256": digest, + "sealed_members": ["uv-python/.lock", "venv/.lock"]}): + raise ValueError("native runtime uv lock repair record differs") + + def _acquire(value, root): _root_layout(root) with NativeTree(root, private=True) as tree: @@ -442,6 +592,8 @@ def _install(value, root, packages): str(root / "runtime.lock")], root, value) _command([str(uv), "--no-config", "--offline", "pip", "check", "--python", str(root / "venv/bin/python")], root, value) + _seal_uv_locks(root) + _installed_tree_custody(root, value["maximum_private_tree_bytes"]) result = _installed(value, root, packages) result.update({"schema": "synaptic-native-private-runtime-install/v1", "status": "OBSERVED_NOT_ADMITTED", "selection_sha256": hashlib.sha256(_canonical(value)).hexdigest(), @@ -453,13 +605,14 @@ def _install(value, root, packages): def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("action", choices=("plan", "acquire", "install", "verify", "_acquire_worker")) + parser.add_argument("action", choices=("plan", "acquire", "install", "verify", + "seal-uv-locks", "_acquire_worker")) parser.add_argument("config", type=Path) args = parser.parse_args(argv) try: value = _load(args.config) root, packages = _config(value) - if args.action in {"acquire", "install", "_acquire_worker"}: + if args.action in {"acquire", "install", "seal-uv-locks", "_acquire_worker"}: _host_target() with NativeTree(args.config.parent, private=True): pass @@ -488,10 +641,24 @@ def main(argv=None): object_pairs_hook=_pairs) if retained != result: raise ValueError("native runtime retained install result differs") + elif args.action == "seal-uv-locks": + result = _repair_uv_locks(value, root, packages) + with NativeTree(root, private=True) as tree: + if tree.read("uv-lock-repair-result.json", maximum=4096) != _canonical(result): + raise ValueError("native runtime uv lock repair readback differs") elif args.action == "verify": _retained_locks(root) - _archives(value, root) + expanded = _archives(value, root) + _check_uv_repair_record(root, hashlib.sha256(_canonical(value)).hexdigest()) + _installed_tree_custody(root, value["maximum_private_tree_bytes"]) result = _installed(value, root, packages) + expected = {**result, "schema": "synaptic-native-private-runtime-install/v1", + "status": "OBSERVED_NOT_ADMITTED", + "selection_sha256": hashlib.sha256(_canonical(value)).hexdigest(), + "wheel_unpacked_bytes": expanded} + with NativeTree(root, private=True) as tree: + if tree.read("install-result.json", maximum=4096) != _canonical(expected): + raise ValueError("native runtime retained install result differs") result["status"] = "OBSERVED_NOT_ADMITTED" else: status, raw = run_bounded_command( @@ -511,7 +678,7 @@ def main(argv=None): raise ValueError("native runtime retained result differs") print(_canonical(result).decode()) return 0 - except Exception: + except BaseException: print('{"diagnostic_code":"native_runtime_preparation_refused"}') return 2 diff --git a/tests/cli/test_native_runtime_preparation.py b/tests/cli/test_native_runtime_preparation.py index cf4fdad2..ed83cce0 100644 --- a/tests/cli/test_native_runtime_preparation.py +++ b/tests/cli/test_native_runtime_preparation.py @@ -199,3 +199,226 @@ def test_owned_command_flood_and_deadline_are_bounded(proposal): short = {**config, "maximum_worker_seconds": 0.05} with pytest.raises(BoundedCommandError): preparation._command([sys.executable, "-c", "import time;time.sleep(2)"], parent, short) + + +def _completed_install(proposal, monkeypatch): + config, _, parent = proposal + root = parent / "new-runtime" + preparation._root_layout(root) + (root / "uv-python").mkdir(mode=0o700) + (root / "venv").mkdir(mode=0o700) + for parent_name in preparation._UV_LOCK_PARENTS: + lock = root / parent_name / ".lock" + lock.write_bytes(b"") + lock.chmod(0o666) # Pinned uv's observed output, despite umask 077. + uv = root / "tools/uv" + uv.write_bytes(b"selected uv bytes") + config["uv_sha256"] = hashlib.sha256(uv.read_bytes()).hexdigest() + selection = hashlib.sha256(preparation._canonical(config)).hexdigest() + expanded = 500687472 + installed = {"python_version": "3.11.14", "distribution_count": 54, + "inventory_sha256": "a" * 64, "executable": str(root / "uv-python/bin/python3.11"), + "executable_sha256": "b" * 64, "stdlib": str(root / "uv-python/lib/python3.11")} + result = {**installed, "schema": "synaptic-native-private-runtime-install/v1", + "status": "OBSERVED_NOT_ADMITTED", "selection_sha256": selection, + "wheel_unpacked_bytes": expanded} + with preparation.NativeTree(root, private=True) as tree: + for name, raw in zip(("original.lock", "host-additions.lock", "runtime.lock"), + preparation._lock_bytes(), strict=True): + tree.write_exclusive(name, raw) + tree.write_exclusive("archive-result.json", preparation._canonical({ + "schema": "synaptic-native-private-runtime-archives/v1", + "status": "OBSERVED_NOT_ADMITTED", "selection_sha256": selection, + "archive_bytes": 166269790, "wheel_count": 54, + "wheel_unpacked_bytes": expanded, + "runtime_lock_sha256": config["runtime_lock_sha256"], + })) + tree.write_exclusive("install-claim.json", preparation._canonical({ + "schema": "synaptic-native-runtime-install-claim/v1", + "status": "CONSUMED_UNCERTAIN", "selection_sha256": selection, + })) + tree.write_exclusive("install-result.json", preparation._canonical(result)) + monkeypatch.setattr(preparation, "_archives", lambda *_: expanded) + monkeypatch.setattr(preparation, "_installed", lambda *_: installed) + return config, root, preparation._canonical(result) + + +def test_completed_install_seals_only_uv_locks_and_preserves_prior_result(proposal, monkeypatch): + config, root, original = _completed_install(proposal, monkeypatch) + result = preparation._repair_uv_locks(config, root, preparation._config({ + **config, "uv_sha256": preparation._UV_SHA})[1]) + assert result["sealed_members"] == ["uv-python/.lock", "venv/.lock"] + assert (root / "install-result.json").read_bytes() == original + for parent in preparation._UV_LOCK_PARENTS: + lock = root / parent / ".lock" + assert lock.stat().st_mode & 0o777 == 0o600 and lock.stat().st_size == 0 + assert json.loads((root / "uv-lock-repair-claim.json").read_bytes())["status"] == "CONSUMED_UNCERTAIN" + preparation._check_uv_repair_record(root, hashlib.sha256(preparation._canonical(config)).hexdigest()) + with pytest.raises(ValueError, match="slot is consumed"): + preparation._repair_uv_locks(config, root, {}) + + +@pytest.mark.parametrize("damage", ["symlink", "content", "hardlink", "mode"]) +def test_repair_rejects_unexpected_uv_lock_without_mutation(proposal, monkeypatch, damage): + config, root, original = _completed_install(proposal, monkeypatch) + lock = root / "uv-python/.lock" + if damage == "symlink": + lock.unlink() + lock.symlink_to(root / "venv/.lock") + elif damage == "content": + lock.write_bytes(b"changed") + elif damage == "hardlink": + os.link(lock, root / "uv-python/other") + else: + lock.chmod(0o644) + with pytest.raises((ValueError, OSError)): + preparation._repair_uv_locks(config, root, {}) + assert not (root / "uv-lock-repair-claim.json").exists() + assert (root / "venv/.lock").stat().st_mode & 0o777 == 0o666 + assert (root / "install-result.json").read_bytes() == original + + +def test_repair_claim_collision_prevents_chmod(proposal, monkeypatch): + config, root, _ = _completed_install(proposal, monkeypatch) + (root / "uv-lock-repair-claim.json").write_bytes(b"prior claim") + with pytest.raises(ValueError, match="slot is consumed"): + preparation._repair_uv_locks(config, root, {}) + assert all((root / parent / ".lock").stat().st_mode & 0o777 == 0o666 + for parent in preparation._UV_LOCK_PARENTS) + assert not (root / "uv-lock-repair-result.json").exists() + + +@pytest.mark.parametrize("drift", ["uv", "archive", "install_result", "selection", "inventory"]) +def test_repair_rejects_prior_material_drift_before_claim(proposal, monkeypatch, drift): + config, root, _ = _completed_install(proposal, monkeypatch) + if drift == "uv": + (root / "tools/uv").write_bytes(b"different uv") + elif drift == "archive": + (root / "archive-result.json").write_bytes(b"{}") + elif drift == "install_result": + (root / "install-result.json").write_bytes(b"{}") + elif drift == "selection": + config["maximum_worker_seconds"] -= 1 + else: + monkeypatch.setattr(preparation, "_installed", lambda *_: { + "python_version": "3.11.14", "distribution_count": 54, + "inventory_sha256": "c" * 64, "executable": str(root / "uv-python/bin/python3.11"), + "executable_sha256": "b" * 64, "stdlib": str(root / "uv-python/lib/python3.11"), + }) + with pytest.raises(ValueError): + preparation._repair_uv_locks(config, root, {}) + assert not (root / "uv-lock-repair-claim.json").exists() + assert all((root / parent / ".lock").stat().st_mode & 0o777 == 0o666 + for parent in preparation._UV_LOCK_PARENTS) + + +def test_interrupted_repair_preserves_claim_and_partial_modes(proposal, monkeypatch): + config, root, original = _completed_install(proposal, monkeypatch) + real_fchmod = os.fchmod + calls = 0 + + def interrupt(fd, mode): + nonlocal calls + calls += 1 + if calls == 2: + raise KeyboardInterrupt + real_fchmod(fd, mode) + + monkeypatch.setattr(preparation.os, "fchmod", interrupt) + with pytest.raises(KeyboardInterrupt): + preparation._repair_uv_locks(config, root, {}) + assert (root / "uv-lock-repair-claim.json").exists() + assert not (root / "uv-lock-repair-result.json").exists() + assert (root / "uv-python/.lock").stat().st_mode & 0o777 == 0o600 + assert (root / "venv/.lock").stat().st_mode & 0o777 == 0o666 + assert (root / "install-result.json").read_bytes() == original + with pytest.raises(ValueError, match="repair is uncertain"): + preparation._check_uv_repair_record(root, hashlib.sha256(preparation._canonical(config)).hexdigest()) + with pytest.raises(ValueError, match="slot is consumed"): + preparation._repair_uv_locks(config, root, {}) + + +def test_fresh_install_seals_uv_lock_modes_before_result(proposal, monkeypatch): + config, _, parent = proposal + root = parent / "new-runtime" + preparation._root_layout(root) + uv = root / "tools/uv" + uv.write_bytes(b"selected uv bytes") + config["uv_sha256"] = hashlib.sha256(uv.read_bytes()).hexdigest() + with preparation.NativeTree(root, private=True) as tree: + for name, raw in zip(("original.lock", "host-additions.lock", "runtime.lock"), + preparation._lock_bytes(), strict=True): + tree.write_exclusive(name, raw) + tree.write_exclusive("archive-result.json", preparation._canonical({ + "status": "OBSERVED_NOT_ADMITTED", + "selection_sha256": hashlib.sha256(preparation._canonical(config)).hexdigest(), + "runtime_lock_sha256": config["runtime_lock_sha256"], + })) + monkeypatch.setattr(preparation, "_archives", lambda *_: 500687472) + monkeypatch.setattr(preparation, "_UV_BANNER", b"selected uv banner") + + def command(argv, *_): + if argv[-1] == "--version": + return b"selected uv banner\n" + if "install" in argv: + managed = root / "uv-python" / config["python_install_key"] / "bin/python3.11" + managed.parent.mkdir(parents=True, mode=0o700) + managed.write_bytes(b"python") + (root / "uv-python/.lock").write_bytes(b"") + (root / "uv-python/.lock").chmod(0o666) + elif "venv" in argv: + (root / "venv").mkdir(mode=0o700) + (root / "venv/.lock").write_bytes(b"") + (root / "venv/.lock").chmod(0o666) + return b"" + + monkeypatch.setattr(preparation, "_command", command) + monkeypatch.setattr(preparation, "_installed", lambda *_: { + "python_version": "3.11.14", "distribution_count": 54, + "inventory_sha256": "a" * 64, "executable": str(root / "uv-python/bin/python3.11"), + "executable_sha256": "b" * 64, "stdlib": str(root / "uv-python/lib/python3.11"), + }) + result = preparation._install(config, root, {}) + assert result["status"] == "OBSERVED_NOT_ADMITTED" + assert all((root / parent_name / ".lock").stat().st_mode & 0o777 == 0o600 + for parent_name in preparation._UV_LOCK_PARENTS) + assert (root / "install-result.json").exists() + + +def test_repair_rejects_lock_path_replacement_after_fchmod(proposal, monkeypatch): + config, root, original = _completed_install(proposal, monkeypatch) + real_fchmod = os.fchmod + + def replace_path(fd, mode): + real_fchmod(fd, mode) + lock = root / "uv-python/.lock" + lock.rename(root / "uv-python/.lock.replaced") + lock.write_bytes(b"") + + monkeypatch.setattr(preparation.os, "fchmod", replace_path) + with pytest.raises(ValueError, match="sealed uv lock differs"): + preparation._repair_uv_locks(config, root, {}) + assert (root / "uv-lock-repair-claim.json").exists() + assert not (root / "uv-lock-repair-result.json").exists() + assert (root / "install-result.json").read_bytes() == original + + +def test_repair_cli_interrupt_has_closed_diagnostic(proposal, monkeypatch, capsys): + _, path, _ = proposal + monkeypatch.setattr(preparation, "_host_target", lambda: None) + + def interrupt(*_args): + raise KeyboardInterrupt + + monkeypatch.setattr(preparation, "_repair_uv_locks", interrupt) + assert preparation.main(["seal-uv-locks", str(path)]) == 2 + assert capsys.readouterr().out == '{"diagnostic_code":"native_runtime_preparation_refused"}\n' + + +def test_installed_custody_rejects_escaping_link(proposal): + _, _, parent = proposal + root = parent / "new-runtime" + root.mkdir(mode=0o700) + (root / "outside").symlink_to(parent) + with pytest.raises(ValueError, match="link escapes"): + preparation._installed_tree_custody(root, 1024) From 6b962acd24e19f22b475f7562ae165545d0b849f Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 13:10:36 -0400 Subject: [PATCH 08/12] Resolve inherited native Mach-O runpaths within captured load chains --- .../reference/native-local-training.md | 10 ++ .../reference/native-local-training.md | 10 ++ .../reference/native-local-training.md | 10 ++ docs/plans/native-desktop-training-api-v1.md | 4 + .../training_qualification/native-host.md | 16 +++ tests/runtime/test_native_mlx_material.py | 86 ++++++++++++++++ tuner/runtime/native_macho.py | 98 +++++++++++++++---- 7 files changed, 215 insertions(+), 19 deletions(-) diff --git a/.agents/skills/fine-tuning/reference/native-local-training.md b/.agents/skills/fine-tuning/reference/native-local-training.md index 943bb800..8ec3b23e 100644 --- a/.agents/skills/fine-tuning/reference/native-local-training.md +++ b/.agents/skills/fine-tuning/reference/native-local-training.md @@ -92,6 +92,16 @@ Mach-O dependencies, uncaptured external files and extra declared OS libraries reject. OS shared-cache dependencies are explicitly trusted through the recorded OS/kernel build, not falsely described as hashed ordinary files. +For `@rpath` dependencies, inspect the complete captured load chain. Apple's +dyld runpath list includes `LC_RPATH` entries from the chain leading to the +dependent image; expand `@loader_path` relative to the image declaring each +entry. The static checker requires one captured Mach-O target across every +proven context and rejects ambiguous targets, paths outside the capture, +unreachable cycles and excessive contexts. Do not insert an ambient search +directory or a package-specific fallback to make a candidate pass. A static +capture remains an observation; repeat it on the final independently reviewed +pushed source after any checker change. + This captures installed bytes and static native dependencies. It does **not** prove selected wheel archive provenance, attest dynamic `dlopen` behavior, freeze the filesystem against a malicious same-uid process, or independently diff --git a/.claude/skills/fine-tuning/reference/native-local-training.md b/.claude/skills/fine-tuning/reference/native-local-training.md index 943bb800..8ec3b23e 100644 --- a/.claude/skills/fine-tuning/reference/native-local-training.md +++ b/.claude/skills/fine-tuning/reference/native-local-training.md @@ -92,6 +92,16 @@ Mach-O dependencies, uncaptured external files and extra declared OS libraries reject. OS shared-cache dependencies are explicitly trusted through the recorded OS/kernel build, not falsely described as hashed ordinary files. +For `@rpath` dependencies, inspect the complete captured load chain. Apple's +dyld runpath list includes `LC_RPATH` entries from the chain leading to the +dependent image; expand `@loader_path` relative to the image declaring each +entry. The static checker requires one captured Mach-O target across every +proven context and rejects ambiguous targets, paths outside the capture, +unreachable cycles and excessive contexts. Do not insert an ambient search +directory or a package-specific fallback to make a candidate pass. A static +capture remains an observation; repeat it on the final independently reviewed +pushed source after any checker change. + This captures installed bytes and static native dependencies. It does **not** prove selected wheel archive provenance, attest dynamic `dlopen` behavior, freeze the filesystem against a malicious same-uid process, or independently diff --git a/.skills/fine-tuning/reference/native-local-training.md b/.skills/fine-tuning/reference/native-local-training.md index 943bb800..8ec3b23e 100644 --- a/.skills/fine-tuning/reference/native-local-training.md +++ b/.skills/fine-tuning/reference/native-local-training.md @@ -92,6 +92,16 @@ Mach-O dependencies, uncaptured external files and extra declared OS libraries reject. OS shared-cache dependencies are explicitly trusted through the recorded OS/kernel build, not falsely described as hashed ordinary files. +For `@rpath` dependencies, inspect the complete captured load chain. Apple's +dyld runpath list includes `LC_RPATH` entries from the chain leading to the +dependent image; expand `@loader_path` relative to the image declaring each +entry. The static checker requires one captured Mach-O target across every +proven context and rejects ambiguous targets, paths outside the capture, +unreachable cycles and excessive contexts. Do not insert an ambient search +directory or a package-specific fallback to make a candidate pass. A static +capture remains an observation; repeat it on the final independently reviewed +pushed source after any checker change. + This captures installed bytes and static native dependencies. It does **not** prove selected wheel archive provenance, attest dynamic `dlopen` behavior, freeze the filesystem against a malicious same-uid process, or independently diff --git a/docs/plans/native-desktop-training-api-v1.md b/docs/plans/native-desktop-training-api-v1.md index 79a4ac37..35e08537 100644 --- a/docs/plans/native-desktop-training-api-v1.md +++ b/docs/plans/native-desktop-training-api-v1.md @@ -207,6 +207,10 @@ The next reviewable source increment adds: dependency parsing, explicit Homebrew alias/OS-shared-cache treatment and comparison against a host-reviewed capture digest. This is observed installed material, not independent wheel-origin or dynamic-loader attestation. + For nested `@rpath` links, carry only declared `LC_RPATH` paths along each + captured dependency chain, expanding `@loader_path` at the declaring image. + Reject ambiguous, outside-capture and unproven load contexts; never infer a + global search path from the presence of a library file. - `native_mlx_input.py`: retained prepared-lease hash/size/identity verification, shared tokenization over the exact declared rows, and private local tokenizer snapshot binding. Semantic publication provenance remains host-owned. diff --git a/examples/training_qualification/native-host.md b/examples/training_qualification/native-host.md index 4f7b11c7..9cf9f8aa 100644 --- a/examples/training_qualification/native-host.md +++ b/examples/training_qualification/native-host.md @@ -160,6 +160,22 @@ tokenizer loadability before wiring them into the host config. The following capture/admission/start commands are for a **later reviewed stage**. The current source and runtime closure cannot pass capture yet. +The r11 private 54-package runtime repair passed checked-in `verify`, but +static capture remained blocked by the pinned MLX wheel's nested runpath: +`mlx/core…so` declares `@loader_path/lib` and loads `libmlx.dylib`, which +requests `@rpath/libjaccl.dylib` without declaring its own rpath. The observed +file is inside the private package tree. The r12 resolver change models +declaring-loader-relative inherited paths and still requires a unique captured +target in every load context. Recheck the exact final pushed source before +running capture; the r11 unresolved selection is review material only. + +The separately admitted r12 model acquisition consumed its one attempt and +stopped at a zero-byte `scratch/repository/model.safetensors` after several +small members had been verified. The retained workspace has no published +model/tokenizer snapshots or acquisition result. This locates the transfer +boundary, not its cause or retry authority. Preserve scratch and inspect the +exact transport behavior before any new independently reviewed attempt. + The r9 offline install and metadata-only `verify` succeeded with a private Python 3.11.14 executable and 54 locked distributions. Full custody review found only two empty uv lock files, `uv-python/.lock` and `venv/.lock`, at mode diff --git a/tests/runtime/test_native_mlx_material.py b/tests/runtime/test_native_mlx_material.py index 3c2d9e54..88493b7b 100644 --- a/tests/runtime/test_native_mlx_material.py +++ b/tests/runtime/test_native_mlx_material.py @@ -124,6 +124,92 @@ def test_native_dependency_resolution_requires_captured_target(private_tree): assert (str(executable), "@loader_path/Python", str(library)) in edges +def test_native_runpath_follows_declaring_loader_through_dependencies(private_tree): + package = private_tree / "package" + library_dir = package / "lib" + library_dir.mkdir(parents=True) + core = package / "core.so" + engine = library_dir / "libmlx.dylib" + helper = library_dir / "libjaccl.dylib" + core.write_bytes(macho("@rpath/libmlx.dylib", rpaths=("@loader_path/lib",))) + engine.write_bytes(macho("@rpath/libjaccl.dylib")) + helper.write_bytes(macho("/usr/lib/libSystem.B.dylib")) + links = {str(p): macho_links(p.read_bytes()) for p in (core, engine, helper)} + edges = resolve_links(links, executable=str(core), admitted_files=set(links), + system_libraries=("/usr/lib/libSystem.B.dylib",)) + assert (str(core), "@rpath/libmlx.dylib", str(engine)) in edges + assert (str(engine), "@rpath/libjaccl.dylib", str(helper)) in edges + + +def test_native_runpath_expands_relative_to_its_declarer(private_tree): + entry = private_tree / "entry" + plugin = private_tree / "plugin" + libs = private_tree / "libs" + for path in (entry, plugin, libs): + path.mkdir() + executable = entry / "python" + core = plugin / "core.so" + engine = libs / "libengine.dylib" + helper = libs / "libhelper.dylib" + executable.write_bytes(macho("@loader_path/../plugin/core.so")) + core.write_bytes(macho("@rpath/libengine.dylib", rpaths=("@loader_path/../libs",))) + engine.write_bytes(macho("@rpath/libhelper.dylib")) + helper.write_bytes(macho()) + links = {str(p): macho_links(p.read_bytes()) for p in (executable, core, engine, helper)} + edges = resolve_links(links, executable=str(executable), admitted_files=set(links), + system_libraries=()) + assert (str(engine), "@rpath/libhelper.dylib", str(helper)) in edges + + +def test_native_runpath_rejects_different_ancestor_bindings(private_tree): + first = private_tree / "first" + second = private_tree / "second" + for directory in (first, second): + directory.mkdir() + roots = (private_tree / "first.so", private_tree / "second.so") + shared = private_tree / "shared.dylib" + leaves = (first / "leaf.dylib", second / "leaf.dylib") + for root, directory in zip(roots, (first, second), strict=True): + root.write_bytes(macho("@loader_path/shared.dylib", + rpaths=(f"@loader_path/{directory.name}",))) + shared.write_bytes(macho("@rpath/leaf.dylib")) + for leaf in leaves: + leaf.write_bytes(macho()) + links = {str(p): macho_links(p.read_bytes()) for p in (*roots, shared, *leaves)} + with pytest.raises(ValueError, match="differs across load chains"): + resolve_links(links, executable=str(roots[0]), admitted_files=set(links), + system_libraries=()) + + +def test_native_runpath_rejects_outside_captured_target(private_tree): + executable = private_tree / "python" + library = private_tree / "libengine.dylib" + executable.write_bytes(macho("@rpath/libengine.dylib", rpaths=("@loader_path",))) + library.write_bytes(macho()) + links = {str(p): macho_links(p.read_bytes()) for p in (executable, library)} + with pytest.raises(ValueError, match="outside captured"): + resolve_links(links, executable=str(executable), admitted_files={str(executable)}, + system_libraries=()) + + +def test_native_runpath_cycle_is_bounded_and_orphan_cycle_rejects(private_tree): + executable = private_tree / "python" + first = private_tree / "first.dylib" + second = private_tree / "second.dylib" + executable.write_bytes(macho("@loader_path/first.dylib")) + first.write_bytes(macho("@loader_path/second.dylib")) + second.write_bytes(macho("@loader_path/first.dylib")) + links = {str(p): macho_links(p.read_bytes()) for p in (executable, first, second)} + edges = resolve_links(links, executable=str(executable), admitted_files=set(links), + system_libraries=()) + assert len(edges) == 3 + executable.write_bytes(macho()) + links[str(executable)] = macho_links(executable.read_bytes()) + with pytest.raises(ValueError, match="no proven load chain"): + resolve_links(links, executable=str(executable), admitted_files=set(links), + system_libraries=()) + + @pytest.fixture def capture(private_tree, monkeypatch): trees = [private_tree / name for name in ("stdlib", "packages", "source")] diff --git a/tuner/runtime/native_macho.py b/tuner/runtime/native_macho.py index 22b47153..48f09612 100644 --- a/tuner/runtime/native_macho.py +++ b/tuner/runtime/native_macho.py @@ -86,12 +86,25 @@ def resolve_links( inventories: dict[str, MachOLinks], *, executable: str, admitted_files: set[str], system_libraries: tuple[str, ...], ) -> tuple[tuple[str, str, str], ...]: - """Resolve every observed native dependency into captured bytes or OS trust.""" + """Resolve captured load chains, including their inherited dyld runpaths. + + dyld builds the run path list from LC_RPATH commands in the dependency + chain. Expand each entry relative to the image that *declared* it, then + carry that directory into its dependent images. We reject ambiguous + targets even where dyld's search order might choose one: this observation + cannot approve an environmental or order-dependent library choice. + """ if (len(system_libraries) != len(set(system_libraries)) or any(not (p.startswith("/usr/lib/") or p.startswith("/System/Library/")) for p in system_libraries)): raise ValueError("invalid native OS library trust set") - edges = [] + if executable not in inventories or not set(inventories) <= admitted_files: + raise ValueError("native executable or inventory is outside captured files") + if (len(inventories) > 4096 + or sum(len(links.dependencies) for links in inventories.values()) > 16384 + or sum(len(links.rpaths) for links in inventories.values()) > 4096): + raise ValueError("native dependency inventory exceeds bound") + edges = set() used_system = set() def expand(value, loader): @@ -103,34 +116,81 @@ def expand(value, loader): raise ValueError("unsupported native dependency resolution") return value - for loader, links in sorted(inventories.items()): + declared = { + loader: tuple(expand(path, loader) for path in links.rpaths) + for loader, links in inventories.items() + } + if any(len(paths) > 128 for paths in declared.values()): + raise ValueError("native runpath inventory exceeds bound") + + def targets(candidates): + resolved = set() + for candidate in candidates: + path = Path(candidate) + if path.exists(): + # Capture records the load spelling and canonical alias target. + # Revalidation repeats this resolution against the exact files. + canonical = str(path.resolve(strict=True)) + if canonical not in admitted_files: + raise ValueError("native dependency is outside captured files") + resolved.add(canonical) + return resolved + + # An extension module can be a load-chain root even though the executable + # has no LC_LOAD_DYLIB edge to it. Determine possible incoming images only + # from declared paths and captured bytes; unreachable cycles then reject. + all_paths = tuple(sorted({path for paths in declared.values() for path in paths})) + possible_incoming = set() + for loader, links in inventories.items(): for dependency in links.dependencies: + if dependency in system_libraries: + continue + candidates = ( + (path + dependency[len("@rpath"):] for path in all_paths) + if dependency.startswith("@rpath/") + else (expand(dependency, loader),) + ) + possible_incoming.update(targets(candidates) & inventories.keys()) + + roots = sorted((set(inventories) - possible_incoming) | {executable}) + seen = set() + bindings = {} + pending = [(root, declared[root]) for root in roots] + while pending: + loader, inherited = pending.pop() + context = (loader, inherited) + if context in seen: + continue + seen.add(context) + if len(seen) > 8192 or len(inherited) > 4096: + raise ValueError("native load-chain context exceeds bound") + for dependency in inventories[loader].dependencies: if dependency in system_libraries: used_system.add(dependency) - edges.append((loader, dependency, dependency)) + edges.add((loader, dependency, dependency)) continue candidates = ( - [expand(rpath, loader) + dependency[len("@rpath"):] for rpath in links.rpaths] + (path + dependency[len("@rpath"):] for path in inherited) if dependency.startswith("@rpath/") - else [expand(dependency, loader)] + else (expand(dependency, loader),) ) - resolved = set() - for candidate in candidates: - path = Path(candidate) - # Explicitly resolve Homebrew's stable aliases to reviewed bytes. - # The capture retains both the load spelling and canonical target; - # revalidation repeats resolution, detecting alias changes. - if path.exists(): - canonical = str(path.resolve(strict=True)) - if canonical not in admitted_files: - raise ValueError("native dependency is outside captured files") - resolved.add(canonical) + resolved = targets(candidates) if len(resolved) != 1: raise ValueError("native dependency is unresolved or ambiguous") target = resolved.pop() if target not in inventories: raise ValueError("native dependency target is not arm64 Mach-O") - edges.append((loader, dependency, target)) + binding = (loader, dependency) + if binding in bindings and bindings[binding] != target: + raise ValueError("native dependency differs across load chains") + bindings[binding] = target + edges.add((loader, dependency, target)) + child_paths = tuple(dict.fromkeys((*declared[target], *inherited))) + pending.append((target, child_paths)) + if len(pending) > 8192: + raise ValueError("native load-chain context exceeds bound") + if {name for name, _ in seen} != set(inventories): + raise ValueError("native dependency has no proven load chain") if used_system != set(system_libraries): raise ValueError("native OS library trust set contains unused members") - return tuple(edges) + return tuple(sorted(edges)) From 0382cd292411e75a687701d7fd8be0ad3b22f0ad Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 13:12:07 -0400 Subject: [PATCH 09/12] Cite Apple runpath guidance for native capture --- .agents/skills/fine-tuning/reference/native-local-training.md | 3 ++- .claude/skills/fine-tuning/reference/native-local-training.md | 3 ++- .skills/fine-tuning/reference/native-local-training.md | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.agents/skills/fine-tuning/reference/native-local-training.md b/.agents/skills/fine-tuning/reference/native-local-training.md index 8ec3b23e..ded03ac5 100644 --- a/.agents/skills/fine-tuning/reference/native-local-training.md +++ b/.agents/skills/fine-tuning/reference/native-local-training.md @@ -93,7 +93,8 @@ reject. OS shared-cache dependencies are explicitly trusted through the recorded OS/kernel build, not falsely described as hashed ordinary files. For `@rpath` dependencies, inspect the complete captured load chain. Apple's -dyld runpath list includes `LC_RPATH` entries from the chain leading to the +[run-path guidance](https://developer.apple.com/library/archive/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/RunpathDependentLibraries.html) +and `dyld(1)` contract describe `LC_RPATH` entries from the chain leading to the dependent image; expand `@loader_path` relative to the image declaring each entry. The static checker requires one captured Mach-O target across every proven context and rejects ambiguous targets, paths outside the capture, diff --git a/.claude/skills/fine-tuning/reference/native-local-training.md b/.claude/skills/fine-tuning/reference/native-local-training.md index 8ec3b23e..ded03ac5 100644 --- a/.claude/skills/fine-tuning/reference/native-local-training.md +++ b/.claude/skills/fine-tuning/reference/native-local-training.md @@ -93,7 +93,8 @@ reject. OS shared-cache dependencies are explicitly trusted through the recorded OS/kernel build, not falsely described as hashed ordinary files. For `@rpath` dependencies, inspect the complete captured load chain. Apple's -dyld runpath list includes `LC_RPATH` entries from the chain leading to the +[run-path guidance](https://developer.apple.com/library/archive/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/RunpathDependentLibraries.html) +and `dyld(1)` contract describe `LC_RPATH` entries from the chain leading to the dependent image; expand `@loader_path` relative to the image declaring each entry. The static checker requires one captured Mach-O target across every proven context and rejects ambiguous targets, paths outside the capture, diff --git a/.skills/fine-tuning/reference/native-local-training.md b/.skills/fine-tuning/reference/native-local-training.md index 8ec3b23e..ded03ac5 100644 --- a/.skills/fine-tuning/reference/native-local-training.md +++ b/.skills/fine-tuning/reference/native-local-training.md @@ -93,7 +93,8 @@ reject. OS shared-cache dependencies are explicitly trusted through the recorded OS/kernel build, not falsely described as hashed ordinary files. For `@rpath` dependencies, inspect the complete captured load chain. Apple's -dyld runpath list includes `LC_RPATH` entries from the chain leading to the +[run-path guidance](https://developer.apple.com/library/archive/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/RunpathDependentLibraries.html) +and `dyld(1)` contract describe `LC_RPATH` entries from the chain leading to the dependent image; expand `@loader_path` relative to the image declaring each entry. The static checker requires one captured Mach-O target across every proven context and rejects ambiguous targets, paths outside the capture, From 8b299278b6c279c339fe73df90e3cb07e61100d2 Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 13:21:31 -0400 Subject: [PATCH 10/12] Add bounded one-use native model transfer probe --- .../reference/native-local-training.md | 16 +- .../reference/native-local-training.md | 16 +- .../reference/native-local-training.md | 16 +- .../training_qualification/native-host.md | 20 +- .../native_model_preparation.py | 181 +++++++++++++++++- tests/cli/test_native_model_preparation.py | 88 ++++++++- 6 files changed, 320 insertions(+), 17 deletions(-) diff --git a/.agents/skills/fine-tuning/reference/native-local-training.md b/.agents/skills/fine-tuning/reference/native-local-training.md index ded03ac5..ace2f328 100644 --- a/.agents/skills/fine-tuning/reference/native-local-training.md +++ b/.agents/skills/fine-tuning/reference/native-local-training.md @@ -283,10 +283,22 @@ per-file and aggregate caps reject before excess writes. The SDK's unbounded snapshot/cache download path is not used. Unexpected redirect hosts fail closed. It verifies upstream hashes and publishes private create-only snapshots. Its receipt is -`OBSERVED_NOT_ADMITTED`. No Mac model bytes or tokenizer labels have yet been -acquired/reviewed. See `examples/training_qualification/native-host.md` for +`OBSERVED_NOT_ADMITTED`. No complete Mac model snapshot or tokenizer labels +have yet been acquired/reviewed. See `examples/training_qualification/native-host.md` for exact staging paths, publication digests and commands. +The first bounded model acquisition left a consumed partial workspace with +five verified small members and a zero-byte `model.safetensors`. Preserve it; +the byte count alone does not diagnose the transport. A separately admitted +`probe-transfer` uses the existing helper and a fresh private record root, +binding the exact reviewed acquisition config hash and selected member. It +performs at most one HEAD chain and one GET chain, up to three redirects each, +20 seconds per request, 120 seconds for the owned child, and reads no more +than one GET body byte without a Range header. Unknown redirect origins stop +before follow. Closed observations contain no URL, query, header value, +credential or exception text. Its consumed claim and result cannot repair or +authorize replay of the old acquisition. See the host example for the config. + See `docs/plans/native-desktop-training-api-v1.md` for remaining review and real-Mac acceptance. The CPU facade tests substitute the ML child; they are not a hardware result and must never unlock the candidate. diff --git a/.claude/skills/fine-tuning/reference/native-local-training.md b/.claude/skills/fine-tuning/reference/native-local-training.md index ded03ac5..ace2f328 100644 --- a/.claude/skills/fine-tuning/reference/native-local-training.md +++ b/.claude/skills/fine-tuning/reference/native-local-training.md @@ -283,10 +283,22 @@ per-file and aggregate caps reject before excess writes. The SDK's unbounded snapshot/cache download path is not used. Unexpected redirect hosts fail closed. It verifies upstream hashes and publishes private create-only snapshots. Its receipt is -`OBSERVED_NOT_ADMITTED`. No Mac model bytes or tokenizer labels have yet been -acquired/reviewed. See `examples/training_qualification/native-host.md` for +`OBSERVED_NOT_ADMITTED`. No complete Mac model snapshot or tokenizer labels +have yet been acquired/reviewed. See `examples/training_qualification/native-host.md` for exact staging paths, publication digests and commands. +The first bounded model acquisition left a consumed partial workspace with +five verified small members and a zero-byte `model.safetensors`. Preserve it; +the byte count alone does not diagnose the transport. A separately admitted +`probe-transfer` uses the existing helper and a fresh private record root, +binding the exact reviewed acquisition config hash and selected member. It +performs at most one HEAD chain and one GET chain, up to three redirects each, +20 seconds per request, 120 seconds for the owned child, and reads no more +than one GET body byte without a Range header. Unknown redirect origins stop +before follow. Closed observations contain no URL, query, header value, +credential or exception text. Its consumed claim and result cannot repair or +authorize replay of the old acquisition. See the host example for the config. + See `docs/plans/native-desktop-training-api-v1.md` for remaining review and real-Mac acceptance. The CPU facade tests substitute the ML child; they are not a hardware result and must never unlock the candidate. diff --git a/.skills/fine-tuning/reference/native-local-training.md b/.skills/fine-tuning/reference/native-local-training.md index ded03ac5..ace2f328 100644 --- a/.skills/fine-tuning/reference/native-local-training.md +++ b/.skills/fine-tuning/reference/native-local-training.md @@ -283,10 +283,22 @@ per-file and aggregate caps reject before excess writes. The SDK's unbounded snapshot/cache download path is not used. Unexpected redirect hosts fail closed. It verifies upstream hashes and publishes private create-only snapshots. Its receipt is -`OBSERVED_NOT_ADMITTED`. No Mac model bytes or tokenizer labels have yet been -acquired/reviewed. See `examples/training_qualification/native-host.md` for +`OBSERVED_NOT_ADMITTED`. No complete Mac model snapshot or tokenizer labels +have yet been acquired/reviewed. See `examples/training_qualification/native-host.md` for exact staging paths, publication digests and commands. +The first bounded model acquisition left a consumed partial workspace with +five verified small members and a zero-byte `model.safetensors`. Preserve it; +the byte count alone does not diagnose the transport. A separately admitted +`probe-transfer` uses the existing helper and a fresh private record root, +binding the exact reviewed acquisition config hash and selected member. It +performs at most one HEAD chain and one GET chain, up to three redirects each, +20 seconds per request, 120 seconds for the owned child, and reads no more +than one GET body byte without a Range header. Unknown redirect origins stop +before follow. Closed observations contain no URL, query, header value, +credential or exception text. Its consumed claim and result cannot repair or +authorize replay of the old acquisition. See the host example for the config. + See `docs/plans/native-desktop-training-api-v1.md` for remaining review and real-Mac acceptance. The CPU facade tests substitute the ML child; they are not a hardware result and must never unlock the candidate. diff --git a/examples/training_qualification/native-host.md b/examples/training_qualification/native-host.md index 9cf9f8aa..8b6c7e62 100644 --- a/examples/training_qualification/native-host.md +++ b/examples/training_qualification/native-host.md @@ -123,9 +123,11 @@ shape, but exact tokenizer IDs, assistant-only label masks, and `enable_thinking=false` still require the pinned tokenizer and shared preprocessor before training. These data hashes do not grant training authority. -The model workspace is empty. Its adjacent `model-acquisition-draft.json` is -incomplete by design: `members` and `tokenizer_members` are empty. No model -metadata request, weights download or ML import has run. The reusable +The r12 one-use model acquisition workspace is consumed and partial: five +small repository members were verified, while `model.safetensors` remains a +zero-byte exclusive file. No snapshot publication exists. Preserve that +workspace without retry or cleanup. A zero-byte file does not establish a +transport cause. The reusable `native_model_preparation.py` action supports a later separately reviewed acquisition using exact complete upstream metadata, a 2 MiB streaming response cap before Hub `ModelInfo` parsing, and exact prewrite bounded HTTPS transfers @@ -155,6 +157,18 @@ The template paths are placeholders, never execution authority. Review the complete metadata candidate, selected bytes, both snapshot manifests and tokenizer loadability before wiring them into the host config. +For a separately admitted transport diagnosis, `probe-transfer` takes a fresh +private config with schema `synaptic-native-model-transfer-probe/v1`: exact +`source_config` path and `source_sha256`, `member_path`, fresh `record_root`, +`timeout_seconds: 120`, `maximum_redirects: 3`, `get_body_bytes: 1`, and +`range_requested: false`. It creates a consumed claim before network I/O, +issues at most one HEAD chain and one GET chain through the approved HTTPS +origins with proxy disabled, and reads at most one GET body byte. It writes +only closed status, origin-class, encoding and length observations; unknown +redirect origins stop before request. HEAD and GET may differ, and this probe +cannot assign a cause to the earlier failed acquisition or authorize retry. +Use a fresh reviewed workspace for any later acquisition. + ## Reusable preparation and review sequence The following capture/admission/start commands are for a **later reviewed diff --git a/examples/training_qualification/native_model_preparation.py b/examples/training_qualification/native_model_preparation.py index 3a183082..ca77d00e 100644 --- a/examples/training_qualification/native_model_preparation.py +++ b/examples/training_qualification/native_model_preparation.py @@ -4,6 +4,8 @@ ``inspect`` validates an independently reviewed inventory without network access. ``acquire`` is a later, separately authorized effect. It retains a consumed workspace on any uncertain failure and never retries automatically. +``probe-transfer`` observes only a separately admitted HEAD and one-byte GET +chain in a fresh private record root; it never repairs an acquisition. """ from __future__ import annotations @@ -15,10 +17,13 @@ import os from pathlib import Path import re +import socket +import ssl import stat import sys import unicodedata -from urllib.parse import quote, urlsplit +from urllib.error import HTTPError, URLError +from urllib.parse import quote, urljoin, urlsplit from urllib.request import HTTPRedirectHandler, ProxyHandler, Request, build_opener ROOT = Path(__file__).resolve().parents[2] @@ -37,6 +42,19 @@ _CHUNK = 64 * 1024 _REDIRECT_HOSTS = frozenset({"huggingface.co", "cdn-lfs.huggingface.co", "cas-bridge.xethub.hf.co"}) +_OFFICIAL_ORIGIN_CLASSES = { + "huggingface.co": "hub", + "cdn-lfs.huggingface.co": "legacy_lfs", + "cas-bridge.xethub.hf.co": "xet_bridge", + "cas-server.xethub.hf.co": "official_xet_cas_us", + "cas-server.xethub-eu.hf.co": "official_xet_cas_eu", + "transfer.xethub.hf.co": "official_xet_transfer_us", + "transfer.xethub-eu.hf.co": "official_xet_transfer_eu", + "us.aws.cdn.hf.co": "official_cdn_aws_us", + "us.gcp.cdn.hf.co": "official_cdn_gcp_us", + "cdn-lfs-us-1.hf.co": "official_lfs_us", + "cdn-lfs-eu-1.hf.co": "official_lfs_eu", +} _MEMBER_FIELDS = {"path", "size", "kind", "upstream_digest"} @@ -135,6 +153,141 @@ def _validate(value): return members +def _probe_selection(value, *, require_fresh): + """Bind a diagnostic to reviewed acquisition bytes, not its consumed workspace.""" + if (type(value) is not dict or set(value) != { + "schema", "source_config", "source_sha256", "member_path", "record_root", + "timeout_seconds", "maximum_redirects", "get_body_bytes", "range_requested", + } or value["schema"] != "synaptic-native-model-transfer-probe/v1" + or type(value["source_sha256"]) is not str + or re.fullmatch(r"[0-9a-f]{64}", value["source_sha256"]) is None + or value["timeout_seconds"] != 120 or type(value["timeout_seconds"]) is not int + or value["maximum_redirects"] != 3 or type(value["maximum_redirects"]) is not int + or value["get_body_bytes"] != 1 or type(value["get_body_bytes"]) is not int + or value["range_requested"] is not False): + raise ValueError("model transfer probe selection differs") + source_path = value["source_config"] + if (type(source_path) is not str or not Path(source_path).is_absolute() + or str(Path(source_path)) != os.path.normpath(source_path)): + raise ValueError("model transfer source path differs") + with NativeTree(Path(source_path).parent, private=True) as parent: + source_raw = parent.read(Path(source_path).name, maximum=1 << 20) + if hashlib.sha256(source_raw).hexdigest() != value["source_sha256"]: + raise ValueError("model transfer source bytes differ") + source = json.loads(source_raw, object_pairs_hook=_pairs, + parse_constant=lambda _: (_ for _ in ()).throw(ValueError())) + _validate_common(source) + member_path = value["member_path"] + if type(member_path) is not str: + raise ValueError("model transfer member differs") + relative_member(member_path) + selected = [_member(item) for item in source["members"] if type(item) is dict + and item.get("path") == member_path] + if len(selected) != 1: + raise ValueError("model transfer member differs from source") + record_root = value["record_root"] + if (type(record_root) is not str or not Path(record_root).is_absolute() + or str(Path(record_root)) != os.path.normpath(record_root)): + raise ValueError("model transfer record root differs") + with NativeTree(Path(record_root), private=True) as root: + if require_fresh and root.members(): + raise ValueError("model transfer probe slot is consumed") + return source, selected[0] + + +class _NoRedirect(HTTPRedirectHandler): + def redirect_request(self, request, fp, code, msg, headers, newurl): + return None + + +def _origin_class(url): + try: + parsed = urlsplit(url) + if (parsed.scheme != "https" or parsed.username or parsed.password + or parsed.port not in (None, 443) or parsed.fragment): + return "invalid_origin" + return _OFFICIAL_ORIGIN_CLASSES.get(parsed.hostname, "unrecognized_origin") + except ValueError: + return "invalid_origin" + + +def _probe_chain(url, method, *, expected_size, opener=None): + """Observe one method chain; never follow a URL outside acquisition policy.""" + if opener is None: + opener = build_opener(ProxyHandler({}), _NoRedirect()).open + hops = [] + for redirect_count in range(4): + if _origin_class(url) == "invalid_origin" or urlsplit(url).hostname not in _REDIRECT_HOSTS: + return {"method": method, "outcome": "destination_rejected", "hops": hops} + request = Request(url, method=method, + headers={"Accept-Encoding": "identity", + "User-Agent": "synaptic-native-model-transfer-probe/1"}) + try: + response = opener(request, timeout=20) + except HTTPError as error: + response = error + except (URLError, TimeoutError, socket.timeout, ssl.SSLError, OSError): + return {"method": method, "outcome": "transport_error", "hops": hops} + with response: + status = response.status + entry = {"origin_class": _origin_class(url), + "status": status if type(status) is int and 100 <= status <= 599 else 0} + headers = response.headers + if status in (301, 302, 303, 307, 308): + location = headers.get("Location") + target = urljoin(url, location) if location else "" + entry["target_origin_class"] = _origin_class(target) + entry["target_allowed"] = (entry["target_origin_class"] != "invalid_origin" + and urlsplit(target).hostname in _REDIRECT_HOSTS) + hops.append(entry) + if not entry["target_allowed"]: + return {"method": method, "outcome": "redirect_destination_rejected", + "hops": hops} + if redirect_count == 3: + return {"method": method, "outcome": "redirect_limit", "hops": hops} + url = target + continue + declared = headers.get("Content-Length") + entry["content_length"] = (int(declared) if declared is not None + and declared.isdecimal() and len(declared) <= 12 else None) + entry["declared_size_matches"] = entry["content_length"] == expected_size + entry["encoding_identity"] = headers.get("Content-Encoding", "identity") == "identity" + if method == "GET" and status == 200: + try: + entry["body_bytes_read"] = len(response.read(1)) + except (URLError, TimeoutError, socket.timeout, ssl.SSLError, OSError): + hops.append(entry) + return {"method": method, "outcome": "body_read_error", "hops": hops} + hops.append(entry) + return {"method": method, "outcome": "response_observed", "hops": hops} + raise AssertionError("unreachable redirect bound") + + +def _probe_transfer(config, *, opener=None): + source, member = _probe_selection(config, require_fresh=True) + root_path = Path(config["record_root"]) + selection = hashlib.sha256(_canonical(config)).hexdigest() + claim = {"schema": "synaptic-native-model-transfer-probe-claim/v1", + "selection_sha256": selection, "status": "CONSUMED"} + with NativeTree(root_path, private=True) as root: + root.write_exclusive("probe-claim.json", _canonical(claim)) + url = (f"{_ENDPOINT}/{source['repository']}/resolve/{source['revision']}/" + f"{quote(member['path'], safe='/')}") + observations = [_probe_chain(url, method, expected_size=member["size"], opener=opener) + for method in ("HEAD", "GET")] + result = {"schema": "synaptic-native-model-transfer-probe-result/v1", + "status": "OBSERVED_NOT_ADMITTED", "selection_sha256": selection, + "source_sha256": config["source_sha256"], "member_path": member["path"], + "member_declared_bytes": member["size"], "range_requested": False, + "observations": observations} + raw = _canonical(result) + if len(raw) > 4096: + raise ValueError("model transfer probe result exceeds bound") + with NativeTree(root_path, private=True) as root: + root.write_exclusive("probe-result.json", raw) + return result + + def _upstream(member, raw): if len(raw) != member["size"]: raise ValueError("model member byte count differs") @@ -442,12 +595,15 @@ def _acquire_private(config, *, api_type=None, download=None): def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("action", choices=("inspect", "metadata-candidate", "acquire", + "probe-transfer", "_probe_worker", "_metadata_worker", "_worker")) parser.add_argument("config", type=Path) args = parser.parse_args(argv) try: config = _read_config(args.config) - if args.action in {"metadata-candidate", "_metadata_worker"}: + if args.action in {"probe-transfer", "_probe_worker"}: + _probe_selection(config, require_fresh=True) + elif args.action in {"metadata-candidate", "_metadata_worker"}: _validate_common(config) else: _validate(config) @@ -456,19 +612,23 @@ def main(argv=None): "selection_sha256": hashlib.sha256(_canonical(config)).hexdigest(), "member_count": len(config["members"]), "declared_bytes": sum(item["size"] for item in config["members"])} + elif args.action == "_probe_worker": + result = _probe_transfer(config) elif args.action == "_metadata_worker": result = _metadata_candidate(config) elif args.action == "_worker": result = _acquire(config) else: + workspace = (config["record_root"] if args.action == "probe-transfer" + else config["workspace_root"]) environment = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "LC_ALL": "C", - "HF_HOME": config["workspace_root"], + "HF_HOME": workspace, "HF_HUB_DISABLE_TELEMETRY": "1", "HF_HUB_DISABLE_IMPLICIT_TOKEN": "1", - "XDG_CACHE_HOME": config["workspace_root"], - "TMPDIR": config["workspace_root"]} + "XDG_CACHE_HOME": workspace, "TMPDIR": workspace} status, output = run_bounded_command( [sys.executable, "-I", "-B", str(Path(__file__).resolve()), - "_metadata_worker" if args.action == "metadata-candidate" else "_worker", + ("_probe_worker" if args.action == "probe-transfer" else + "_metadata_worker" if args.action == "metadata-candidate" else "_worker"), str(args.config)], environment=environment, maximum=4096, timeout=config["timeout_seconds"], ) @@ -479,7 +639,14 @@ def main(argv=None): worker_result = json.loads(output[:-1], object_pairs_hook=_pairs) if _canonical(worker_result) + b"\n" != output: raise ValueError("model acquisition worker result differs") - if args.action == "metadata-candidate": + if args.action == "probe-transfer": + with NativeTree(Path(config["record_root"]), private=True) as root: + result = json.loads(root.read("probe-result.json", maximum=4096), + object_pairs_hook=_pairs) + if (result != worker_result or result.get("status") != "OBSERVED_NOT_ADMITTED" + or result.get("selection_sha256") != hashlib.sha256(_canonical(config)).hexdigest()): + raise ValueError("model transfer probe result differs") + elif args.action == "metadata-candidate": with NativeTree(Path(config["workspace_root"]), private=True) as root: candidate = json.loads(root.read("metadata-candidate.json", maximum=1 << 20)) result = {"status": candidate["status"], "repository": config["repository"], diff --git a/tests/cli/test_native_model_preparation.py b/tests/cli/test_native_model_preparation.py index 34fc8049..27e71aec 100644 --- a/tests/cli/test_native_model_preparation.py +++ b/tests/cli/test_native_model_preparation.py @@ -1,7 +1,7 @@ """Private native model preparation contracts; no SDK network or ML imports.""" from contextlib import contextmanager -from hashlib import sha1 +from hashlib import sha1, sha256 import io import json import os @@ -184,6 +184,92 @@ def open_url(request, timeout): assert not (workspace / "model-snapshot").exists() +def _probe_config(candidate): + source, _, workspace = candidate + source_path = workspace / "config.json" + record = workspace / "transport-probe" + record.mkdir(mode=0o700) + return {"schema": "synaptic-native-model-transfer-probe/v1", + "source_config": str(source_path), + "source_sha256": sha256(source_path.read_bytes()).hexdigest(), + "member_path": "model.safetensors", "record_root": str(record), + "timeout_seconds": 120, "maximum_redirects": 3, + "get_body_bytes": 1, "range_requested": False} + + +def test_transfer_probe_caps_body_and_preserves_consumed_acquisition(candidate): + config = _probe_config(candidate) + root = Path(config["record_root"]) + old = Path(candidate[0]["workspace_root"]) + (old / "scratch").mkdir(mode=0o700) + (old / "scratch" / "original").write_bytes(b"retained") + calls = [] + + class Limited(_Response): + def read(self, size=-1): + assert size == 1 + return super().read(size) + + def open_url(request, timeout): + calls.append((request.get_method(), request.get_header("Range"), timeout)) + return Limited(b"large body", request.full_url, headers={"Content-Length": "335450584"}) + + result = preparation._probe_transfer(config, opener=open_url) + assert calls == [("HEAD", None, 20), ("GET", None, 20)] + assert result["range_requested"] is False + assert result["observations"][1]["hops"][0]["body_bytes_read"] == 1 + assert (old / "scratch" / "original").read_bytes() == b"retained" + assert (root / "probe-result.json").read_bytes() == preparation._canonical(result) + with pytest.raises(ValueError, match="consumed"): + preparation._probe_transfer(config, opener=open_url) + assert len(calls) == 2 + + +def test_transfer_probe_unknown_redirect_stops_before_target_and_never_leaks_url(candidate): + config = _probe_config(candidate) + calls = [] + + class Redirect(io.BytesIO): + status = 302 + + def __init__(self, url): + super().__init__(b"") + self.url = url + self.headers = {"Location": "https://unknown.example/private?token=SECRET"} + + def geturl(self): + return self.url + + def redirect(request, timeout): + calls.append(request.full_url) + return Redirect(request.full_url) + + result = preparation._probe_transfer(config, opener=redirect) + assert len(calls) == 2 + assert all(item["outcome"] == "redirect_destination_rejected" + for item in result["observations"]) + assert all(item["hops"][0]["target_origin_class"] == "unrecognized_origin" + for item in result["observations"]) + assert b"SECRET" not in (Path(config["record_root"]) / "probe-result.json").read_bytes() + + +def test_transfer_probe_binds_source_bytes_and_retains_claim_on_transport_error(candidate): + config = _probe_config(candidate) + source_path = Path(config["source_config"]) + source_path.write_bytes(source_path.read_bytes() + b"\n") + with pytest.raises(ValueError, match="source bytes"): + preparation._probe_transfer(config) + assert not (Path(config["record_root"]) / "probe-claim.json").exists() + config["source_sha256"] = sha256(source_path.read_bytes()).hexdigest() + + def fail(request, timeout): + raise OSError("PRIVATE_URL_TOKEN") + + result = preparation._probe_transfer(config, opener=fail) + assert [item["outcome"] for item in result["observations"]] == ["transport_error"] * 2 + assert b"PRIVATE_URL_TOKEN" not in (Path(config["record_root"]) / "probe-result.json").read_bytes() + + def test_interrupted_transfer_retains_scratch_and_rejects_replay(candidate): config, files, root = candidate api, _, calls = _sdk(config, files) From 955011e0e98d86e30adf9aa5e1d82c1dd8235e6d Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 13:24:56 -0400 Subject: [PATCH 11/12] Allow observed Hugging Face CDN origin for bounded model transfer --- .../reference/native-local-training.md | 5 ++++ .../reference/native-local-training.md | 5 ++++ .../reference/native-local-training.md | 5 ++++ .../training_qualification/native-host.md | 7 +++++ .../native_model_preparation.py | 4 ++- tests/cli/test_native_model_preparation.py | 29 +++++++++++++++++++ 6 files changed, 54 insertions(+), 1 deletion(-) diff --git a/.agents/skills/fine-tuning/reference/native-local-training.md b/.agents/skills/fine-tuning/reference/native-local-training.md index ace2f328..52c784dd 100644 --- a/.agents/skills/fine-tuning/reference/native-local-training.md +++ b/.agents/skills/fine-tuning/reference/native-local-training.md @@ -298,6 +298,11 @@ than one GET body byte without a Range header. Unknown redirect origins stop before follow. Closed observations contain no URL, query, header value, credential or exception text. Its consumed claim and result cannot repair or authorize replay of the old acquisition. See the host example for the config. +The r13 one-use probe observed HTTP 302 on both methods to the exact +`us.aws.cdn.hf.co` origin documented by Hugging Face; the preexisting +acquisition allowlist rejected it before follow. Only that exact observed +origin was added to the helper allowlist. Preserve the r12 consumed workspace; +the probe cannot retroactively establish its failure cause or admit a retry. See `docs/plans/native-desktop-training-api-v1.md` for remaining review and real-Mac acceptance. The CPU facade tests substitute the ML child; diff --git a/.claude/skills/fine-tuning/reference/native-local-training.md b/.claude/skills/fine-tuning/reference/native-local-training.md index ace2f328..52c784dd 100644 --- a/.claude/skills/fine-tuning/reference/native-local-training.md +++ b/.claude/skills/fine-tuning/reference/native-local-training.md @@ -298,6 +298,11 @@ than one GET body byte without a Range header. Unknown redirect origins stop before follow. Closed observations contain no URL, query, header value, credential or exception text. Its consumed claim and result cannot repair or authorize replay of the old acquisition. See the host example for the config. +The r13 one-use probe observed HTTP 302 on both methods to the exact +`us.aws.cdn.hf.co` origin documented by Hugging Face; the preexisting +acquisition allowlist rejected it before follow. Only that exact observed +origin was added to the helper allowlist. Preserve the r12 consumed workspace; +the probe cannot retroactively establish its failure cause or admit a retry. See `docs/plans/native-desktop-training-api-v1.md` for remaining review and real-Mac acceptance. The CPU facade tests substitute the ML child; diff --git a/.skills/fine-tuning/reference/native-local-training.md b/.skills/fine-tuning/reference/native-local-training.md index ace2f328..52c784dd 100644 --- a/.skills/fine-tuning/reference/native-local-training.md +++ b/.skills/fine-tuning/reference/native-local-training.md @@ -298,6 +298,11 @@ than one GET body byte without a Range header. Unknown redirect origins stop before follow. Closed observations contain no URL, query, header value, credential or exception text. Its consumed claim and result cannot repair or authorize replay of the old acquisition. See the host example for the config. +The r13 one-use probe observed HTTP 302 on both methods to the exact +`us.aws.cdn.hf.co` origin documented by Hugging Face; the preexisting +acquisition allowlist rejected it before follow. Only that exact observed +origin was added to the helper allowlist. Preserve the r12 consumed workspace; +the probe cannot retroactively establish its failure cause or admit a retry. See `docs/plans/native-desktop-training-api-v1.md` for remaining review and real-Mac acceptance. The CPU facade tests substitute the ML child; diff --git a/examples/training_qualification/native-host.md b/examples/training_qualification/native-host.md index 8b6c7e62..802bf0f0 100644 --- a/examples/training_qualification/native-host.md +++ b/examples/training_qualification/native-host.md @@ -169,6 +169,13 @@ redirect origins stop before request. HEAD and GET may differ, and this probe cannot assign a cause to the earlier failed acquisition or authorize retry. Use a fresh reviewed workspace for any later acquisition. +The r13 probe observed HTTP 302 for both HEAD and GET from the exact Hub +resolve URL to `us.aws.cdn.hf.co`, which the acquisition helper rejected before +following. Hugging Face lists that exact HTTPS origin as a model-download CDN. +The helper now allows that one observed origin while retaining the same +redirect count, size and hash bounds. This observation does not prove why the +r12 transfer stopped, and no model weight bytes were fetched by the probe. + ## Reusable preparation and review sequence The following capture/admission/start commands are for a **later reviewed diff --git a/examples/training_qualification/native_model_preparation.py b/examples/training_qualification/native_model_preparation.py index ca77d00e..0d6710ab 100644 --- a/examples/training_qualification/native_model_preparation.py +++ b/examples/training_qualification/native_model_preparation.py @@ -40,8 +40,10 @@ _MAX_FILES = 128 _MAX_METADATA_RESPONSE = 2 * 1024 * 1024 _CHUNK = 64 * 1024 +# The exact additional CDN origin was observed in a separately admitted +# one-byte probe and is listed in Hugging Face's model-download documentation. _REDIRECT_HOSTS = frozenset({"huggingface.co", "cdn-lfs.huggingface.co", - "cas-bridge.xethub.hf.co"}) + "cas-bridge.xethub.hf.co", "us.aws.cdn.hf.co"}) _OFFICIAL_ORIGIN_CLASSES = { "huggingface.co": "hub", "cdn-lfs.huggingface.co": "legacy_lfs", diff --git a/tests/cli/test_native_model_preparation.py b/tests/cli/test_native_model_preparation.py index 27e71aec..4115d227 100644 --- a/tests/cli/test_native_model_preparation.py +++ b/tests/cli/test_native_model_preparation.py @@ -9,6 +9,7 @@ import sys from types import SimpleNamespace import tempfile +from urllib.request import Request import pytest @@ -184,6 +185,34 @@ def open_url(request, timeout): assert not (workspace / "model-snapshot").exists() +def test_observed_exact_cdn_is_allowed_for_bounded_member_bytes(candidate): + config, files, workspace = candidate + member = next(item for item in config["members"] if item["path"] == "model.safetensors") + repository = workspace / "repository" + repository.mkdir(mode=0o700) + cdn = "https://us.aws.cdn.hf.co/opaque?signature=PRIVATE" + + def open_url(request, timeout): + assert request.full_url.startswith("https://huggingface.co/") + assert timeout == 20 + return _Response(files["model.safetensors"], cdn, + headers={"Content-Length": str(member["size"])}) + + with preparation.NativeTree(repository, private=True) as root: + assert preparation._fetch_member(config, member, root, remaining=512, + opener=open_url) == member["size"] + assert (repository / "model.safetensors").read_bytes() == files["model.safetensors"] + initial = Request("https://huggingface.co/reviewed/model/resolve/" + "a" * 40 + "/model.safetensors") + handler = preparation._RestrictedRedirect(preparation._REDIRECT_HOSTS, 2) + assert handler.redirect_request(initial, None, 302, "", {}, cdn).full_url == cdn + for rejected in ("https://us.aws.cdn.hf.co.evil.test/file", + "http://us.aws.cdn.hf.co/file"): + with pytest.raises(ValueError, match="HTTPS destination"): + preparation._url_host(rejected, preparation._REDIRECT_HOSTS) + with pytest.raises(ValueError, match="HTTPS destination"): + handler.redirect_request(initial, None, 302, "", {}, "https://unreviewed.example/file") + + def _probe_config(candidate): source, _, workspace = candidate source_path = workspace / "config.json" From 40ff2e757d862f63357e839c45acf65061d48dc8 Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Tue, 20 Oct 2026 13:28:37 -0400 Subject: [PATCH 12/12] Reject empty userinfo in native model HTTPS origins --- .../training_qualification/native_model_preparation.py | 6 ++++-- tests/cli/test_native_model_preparation.py | 9 ++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/examples/training_qualification/native_model_preparation.py b/examples/training_qualification/native_model_preparation.py index 0d6710ab..ccd6da29 100644 --- a/examples/training_qualification/native_model_preparation.py +++ b/examples/training_qualification/native_model_preparation.py @@ -205,7 +205,8 @@ def redirect_request(self, request, fp, code, msg, headers, newurl): def _origin_class(url): try: parsed = urlsplit(url) - if (parsed.scheme != "https" or parsed.username or parsed.password + if (parsed.scheme != "https" or parsed.username is not None + or parsed.password is not None or parsed.port not in (None, 443) or parsed.fragment): return "invalid_origin" return _OFFICIAL_ORIGIN_CLASSES.get(parsed.hostname, "unrecognized_origin") @@ -305,7 +306,8 @@ def _upstream(member, raw): def _url_host(url, allowed): parsed = urlsplit(url) if (parsed.scheme != "https" or parsed.hostname not in allowed - or parsed.port not in (None, 443) or parsed.username or parsed.password + or parsed.port not in (None, 443) or parsed.username is not None + or parsed.password is not None or parsed.fragment): raise ValueError("model HTTPS destination differs") diff --git a/tests/cli/test_native_model_preparation.py b/tests/cli/test_native_model_preparation.py index 4115d227..57b04a9e 100644 --- a/tests/cli/test_native_model_preparation.py +++ b/tests/cli/test_native_model_preparation.py @@ -205,10 +205,17 @@ def open_url(request, timeout): initial = Request("https://huggingface.co/reviewed/model/resolve/" + "a" * 40 + "/model.safetensors") handler = preparation._RestrictedRedirect(preparation._REDIRECT_HOSTS, 2) assert handler.redirect_request(initial, None, 302, "", {}, cdn).full_url == cdn + assert preparation._origin_class(cdn) == "official_cdn_aws_us" + preparation._url_host(cdn, preparation._REDIRECT_HOSTS) for rejected in ("https://us.aws.cdn.hf.co.evil.test/file", - "http://us.aws.cdn.hf.co/file"): + "http://us.aws.cdn.hf.co/file", + "https://@us.aws.cdn.hf.co/file", + "https://:@us.aws.cdn.hf.co/file", + "https://user@us.aws.cdn.hf.co/file", + "https://user:pass@us.aws.cdn.hf.co/file"): with pytest.raises(ValueError, match="HTTPS destination"): preparation._url_host(rejected, preparation._REDIRECT_HOSTS) + assert preparation._origin_class(rejected) != "official_cdn_aws_us" with pytest.raises(ValueError, match="HTTPS destination"): handler.redirect_request(initial, None, 302, "", {}, "https://unreviewed.example/file")