From b5d8299b768564c09c541e3dfc04b3d28eef2fd6 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Wed, 30 Sep 2026 00:38:06 -0400 Subject: [PATCH 1/2] feat: add explicit CPU-state streaming AdamW backend --- docs/CPU_STATE_OPTIMIZER.md | 50 +++++ schemas/train-config.schema.json | 12 ++ src/kev_laya/cpu_state_adamw.py | 184 +++++++++++++++++++ src/kev_laya/training.py | 33 +++- tests/test_cpu_state_adamw.py | 214 ++++++++++++++++++++++ tests/test_cpu_state_training_contract.py | 119 ++++++++++++ 6 files changed, 608 insertions(+), 4 deletions(-) create mode 100644 docs/CPU_STATE_OPTIMIZER.md create mode 100644 src/kev_laya/cpu_state_adamw.py create mode 100644 tests/test_cpu_state_adamw.py create mode 100644 tests/test_cpu_state_training_contract.py diff --git a/docs/CPU_STATE_OPTIMIZER.md b/docs/CPU_STATE_OPTIMIZER.md new file mode 100644 index 0000000..32cd5d5 --- /dev/null +++ b/docs/CPU_STATE_OPTIMIZER.md @@ -0,0 +1,50 @@ +# Explicit CPU optimizer state + +`cpu_state_streaming_v1` is an experimental, explicitly selected CUDA AdamW +backend. It requires PyTorch `2.10.0+cu128` and trainable CUDA FP32 parameters, +including FP32 parameters used under BF16 autocast. Default and fused optimizer +selection and their recorded configuration hashes are preserved. + +Select it in the training configuration with an explicit persistent state budget: + +```json +{ + "training": { + "optimizer_backend": "cpu_state_streaming_v1", + "optimizer_state_max_bytes": 4000000000 + } +} +``` + +The budget bounds two CPU moment tensors and one FP32 CPU scalar step per +trainable parameter. It does not bound total host RAM, checkpoint deserialization, +model parameters or gradients. The backend rejects an insufficient budget before +construction. Each update stages one parameter's moments on CUDA and calls the +pinned stock single-tensor AdamW functional. Global gradient clipping remains in +the trainer. There is no automatic fallback or allocator-cap increase. + +Algorithm version and ordered trainable parameter names, shapes and dtypes enter +the opt-in configuration identity. Resume across default, fused and CPU-state +backends is refused. State loading validates the complete payload before changing +the optimizer, keeps moments on CPU, and rejects malformed or aliased state. +Live parameter replacement or reordering is also refused. The trainer releases +temporary checkpoint/model objects after restoring optimizer, scheduler, sampler +and RNG state; the CLI independently releases its unused checkpoint payload. + +## Qualification scope + +The reviewed external primitive candidate completed eight CUDA unit tests with +no skips under an owned 120-second watchdog and a 70% allocator cap. This covers +stock AdamW primitive equivalence, CPU state residency, malformed-state refusal +and optimizer restore. Its unchanged optimizer math is distinct from the trainer +integration: the integration additionally fixes canonical parameter-schema +serialization to use JSON lists. CPU configuration tests preserve default/fused +hashes and exercise cross-backend resume rejection using simulated device metadata. + +Production trainer uninterrupted versus fresh-process resume checks in FP32 and +BF16 are pending. Actual pretrained full-weight step/save/exit/resume/reload and +the captured-gradient stock optimizer oracle are also pending. Their fixed 16 GiB +available-host-RAM admission requirement is currently unmet. Existing default +backend OOM receipts remain failures. Primitive or CPU fixture checks establish +no independent full-weight derivative parity, trained long context, +representative quality, Jev advantage or issue closure. diff --git a/schemas/train-config.schema.json b/schemas/train-config.schema.json index ef7ba6e..ca9d489 100644 --- a/schemas/train-config.schema.json +++ b/schemas/train-config.schema.json @@ -140,6 +140,18 @@ "title": "Optimizer Backend", "type": "string" }, + "optimizer_state_max_bytes": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Optimizer State Max Bytes" + }, "resource_sample_seconds": { "default": 1.0, "title": "Resource Sample Seconds", diff --git a/src/kev_laya/cpu_state_adamw.py b/src/kev_laya/cpu_state_adamw.py new file mode 100644 index 0000000..72af40d --- /dev/null +++ b/src/kev_laya/cpu_state_adamw.py @@ -0,0 +1,184 @@ +"""Explicit experimental CUDA AdamW with persistent CPU moments. + +Requires torch 2.10.0+cu128. Primitive qualification and full-weight production +qualification are separate; this backend never activates as an automatic fallback. +""" +from __future__ import annotations + +import math +import copy +import torch +from torch.optim import Optimizer +from torch.optim.adamw import adamw + +VERSION = "cpu-state-streaming-adamw-v1-torch2.10.0+cu128" +TORCH_VERSION = "2.10.0+cu128" + + +class CPUStateStreamingAdamW(Optimizer): + def __init__(self, named_parameters, *, lr: float, weight_decay: float, + max_state_bytes: int, betas=(0.9, 0.999), eps=1e-8): + if torch.__version__ != TORCH_VERSION: + raise RuntimeError(f"{VERSION} requires torch {TORCH_VERSION}") + pairs = [(name, p) for name, p in named_parameters if p.requires_grad] + if (not pairs or len({id(p) for _, p in pairs}) != len(pairs) + or len({name for name, _ in pairs}) != len(pairs)): + raise ValueError("empty or duplicate trainable parameter list") + if len({p.untyped_storage().data_ptr() for _, p in pairs}) != len(pairs): + raise ValueError("shared parameter storage is unsupported") + for name, p in pairs: + if not isinstance(name, str) or not name or p.device.type != "cuda" or p.dtype != torch.float32: + raise ValueError("CPU-state backend requires named CUDA FP32 parameters") + if type(max_state_bytes) is not int or max_state_bytes < 1: + raise ValueError("explicit positive max_state_bytes required") + required = sum(2 * p.numel() * p.element_size() + 4 for _, p in pairs) + if required > max_state_bytes: + raise MemoryError(f"CPU AdamW state requires {required} bytes, budget {max_state_bytes}") + if not (isinstance(lr, (int, float)) and math.isfinite(lr) and lr >= 0 + and isinstance(weight_decay, (int, float)) and math.isfinite(weight_decay) and weight_decay >= 0 + and isinstance(eps, (int, float)) and math.isfinite(eps) and eps > 0 + and len(betas) == 2 and all(isinstance(b, (int, float)) and 0 <= b < 1 for b in betas)): + raise ValueError("unsupported AdamW options") + specs = [(name, tuple(p.shape), str(p.dtype)) for name, p in pairs] + defaults = dict(lr=float(lr), betas=tuple(float(b) for b in betas), eps=float(eps), + weight_decay=float(weight_decay), amsgrad=False, foreach=False, + fused=False, capturable=False, differentiable=False, maximize=False, + algorithm_version=VERSION, max_state_bytes=max_state_bytes, + param_specs=specs) + super().__init__([p for _, p in pairs], defaults) + self._fixed = copy.deepcopy({key: value for key, value in defaults.items() + if key != "lr"}) + # Process-local identity. Never put addresses or tensor versions into + # config/checkpoints: fresh-process resume allocates different objects. + self._private_parameter_identity = tuple( + self._live_signature(p) for p in self.param_groups[0]["params"]) + + @staticmethod + def _live_signature(p): + return (id(p), tuple(p.shape), p.dtype, p.device, + p.untyped_storage().data_ptr(), p.storage_offset(), + tuple(p.stride()), p.requires_grad) + + def _check_group(self): + if len(self.param_groups) != 1: + raise ValueError("CPU AdamW requires one parameter group") + group = self.param_groups[0] + if tuple(self._live_signature(p) for p in group["params"]) != self._private_parameter_identity: + raise ValueError("live CPU AdamW parameter identity/order/layout changed") + if (set(group) - set(self._fixed) - {"params", "lr", "initial_lr"} + or any(group.get(key) != value for key, value in self._fixed.items()) + or not isinstance(group.get("lr"), (float, int)) + or not math.isfinite(group["lr"]) or group["lr"] < 0): + raise ValueError("CPU AdamW group options changed") + if "initial_lr" in group and (not isinstance(group["initial_lr"], (float, int)) + or not math.isfinite(group["initial_lr"]) + or group["initial_lr"] < 0): + raise ValueError("invalid initial learning rate") + return group + + @torch.no_grad() + def step(self, closure=None): + if closure is not None: + raise ValueError("closure is unsupported by CPU-state backend") + group = self._check_group() + for p in group["params"]: + grad = p.grad + if grad is None: + continue # stock AdamW semantics + if grad.is_sparse or grad.device != p.device or grad.dtype != p.dtype: + raise ValueError("unsupported optimizer gradient") + state = self.state[p] + if not state: + state["step"] = torch.tensor(0.0, dtype=torch.float32, device="cpu") + state["exp_avg"] = torch.zeros_like(p, device="cpu", memory_format=torch.preserve_format) + state["exp_avg_sq"] = torch.zeros_like(p, device="cpu", memory_format=torch.preserve_format) + self._check_state(state, p) + exp_avg = state["exp_avg"].to(p.device) + exp_avg_sq = state["exp_avg_sq"].to(p.device) + try: + adamw([p], [grad], [exp_avg], [exp_avg_sq], [], [state["step"]], + foreach=False, capturable=False, differentiable=False, fused=False, + grad_scale=None, found_inf=None, has_complex=False, + amsgrad=False, beta1=group["betas"][0], beta2=group["betas"][1], + lr=group["lr"], weight_decay=group["weight_decay"], + eps=group["eps"], maximize=False) + state["exp_avg"].copy_(exp_avg, non_blocking=False) + state["exp_avg_sq"].copy_(exp_avg_sq, non_blocking=False) + finally: + del exp_avg, exp_avg_sq + return None + + @staticmethod + def _check_state(state, p, *, deep=False): + if set(state) != {"step", "exp_avg", "exp_avg_sq"}: + raise ValueError("invalid CPU AdamW state keys") + step = state["step"] + if (not isinstance(step, torch.Tensor) or step.device.type != "cpu" + or step.dtype != torch.float32 or step.shape != () + or step.requires_grad or not math.isfinite(float(step)) + or float(step) < 0 or not float(step).is_integer()): + raise ValueError("invalid CPU AdamW step") + for key in ("exp_avg", "exp_avg_sq"): + item = state[key] + if (not isinstance(item, torch.Tensor) or item.device.type != "cpu" + or item.dtype != p.dtype or item.shape != p.shape or item.requires_grad): + raise ValueError("invalid CPU AdamW moment") + if deep and (not bool(torch.isfinite(item).all()) + or (key == "exp_avg_sq" and bool((item < 0).any()))): + raise ValueError("nonfinite or negative CPU AdamW moment") + + def load_state_dict(self, state_dict): + # Optimizer.load_state_dict would silently move all moments to CUDA. + if not isinstance(state_dict, dict) or set(state_dict) != {"state", "param_groups"}: + raise ValueError("invalid CPU AdamW optimizer payload") + self._check_group() + groups = state_dict["param_groups"] + if not isinstance(groups, list) or len(groups) != 1: + raise ValueError("CPU AdamW requires one parameter group") + saved, current = groups[0], self.param_groups[0] + ids = saved.get("params") + if (not isinstance(ids, list) or any(type(key) is not int for key in ids) + or ids != list(range(len(current["params"])))): + raise ValueError("CPU AdamW parameter count/order changed") + immutable = ("betas", "eps", "weight_decay", "amsgrad", "foreach", "fused", + "capturable", "differentiable", "maximize", "algorithm_version", + "max_state_bytes", "param_specs") + if any(saved.get(key) != self._fixed[key] for key in immutable): + raise ValueError("CPU AdamW parameter identity/options changed") + if (set(saved) - set(self._fixed) - {"lr", "params", "initial_lr"} + or not isinstance(saved.get("lr"), (float, int))): + raise ValueError("unknown CPU AdamW group option") + if (not math.isfinite(saved["lr"]) or saved["lr"] < 0 + or ("initial_lr" in saved and + (not isinstance(saved["initial_lr"], (float, int)) + or not math.isfinite(saved["initial_lr"]) or saved["initial_lr"] < 0))): + raise ValueError("invalid CPU AdamW learning rate") + raw = state_dict["state"] + if (not isinstance(raw, dict) or any(type(key) is not int for key in raw) + or set(raw) - set(ids)): + raise ValueError("unknown CPU AdamW state parameter") + mapped = {} + state_storages = set() + for key, p in zip(ids, current["params"], strict=True): + item = raw.get(key, {}) + if not isinstance(item, dict): + raise ValueError("invalid CPU AdamW per-parameter state") + if item: + self._check_state(item, p, deep=True) + for field in ("step", "exp_avg", "exp_avg_sq"): + ptr = item[field].untyped_storage().data_ptr() + if ptr in state_storages: + raise ValueError("aliased CPU AdamW state storage") + state_storages.add(ptr) + mapped[p] = item + # Commit only after full validation; reuse deserialized CPU tensors so + # resume does not duplicate the entire host optimizer state. + self.state.clear() + self.state.update(mapped) + for key in ("lr", "initial_lr"): + if key in saved: + current[key] = saved[key] + + def state_dict(self): + self._check_group() + return super().state_dict() diff --git a/src/kev_laya/training.py b/src/kev_laya/training.py index fbc2ec6..f46867a 100644 --- a/src/kev_laya/training.py +++ b/src/kev_laya/training.py @@ -19,6 +19,7 @@ from .objectives import ObjectiveConfig, objective from .quality_budget import BudgetExceeded, StepTokenBudget from .schema import canonical +from .cpu_state_adamw import CPUStateStreamingAdamW, VERSION as CPU_STATE_ADAMW_VERSION from .telemetry import TelemetryWriter, artifact, new_snapshot @@ -35,6 +36,7 @@ class TrainSettings: precision: str = "fp32" choice_permutation: bool = False optimizer_backend: str = "default" + optimizer_state_max_bytes: int | None = None resource_sample_seconds: float = 1.0 streamed_branch_vjp: bool = False def __post_init__(self): @@ -46,8 +48,13 @@ def __post_init__(self): raise ValueError("invalid resource sampling interval") if self.precision not in {"fp32", "bf16"}: raise ValueError("unsupported precision") - if self.optimizer_backend not in {"default", "fused"}: + if self.optimizer_backend not in {"default", "fused", "cpu_state_streaming_v1"}: raise ValueError("unsupported optimizer backend") + if self.optimizer_backend == "cpu_state_streaming_v1": + if type(self.optimizer_state_max_bytes) is not int or self.optimizer_state_max_bytes < 1: + raise ValueError("cpu_state_streaming_v1 requires optimizer_state_max_bytes") + elif self.optimizer_state_max_bytes is not None: + raise ValueError("optimizer_state_max_bytes requires cpu_state_streaming_v1") if type(self.streamed_branch_vjp) is not bool: raise ValueError("streamed_branch_vjp must be a boolean") @@ -110,6 +117,8 @@ def train(model: DecisionEngine, tokenizer, data: list[Datum], manifest: dict, o raise FileExistsError("output is not empty; use resume or a new directory") output.mkdir(parents=True, exist_ok=True) recorded_settings = asdict(settings) + if settings.optimizer_backend != "cpu_state_streaming_v1": + recorded_settings.pop("optimizer_state_max_bytes") if not settings.choice_permutation: recorded_settings.pop("choice_permutation") # preserve the original v1 no-augmentation config hash if settings.optimizer_backend == "default": @@ -126,6 +135,13 @@ def train(model: DecisionEngine, tokenizer, data: list[Datum], manifest: dict, o else FP32_DERIVATIVE_VERSION) if settings.precision == "fp32": cfg["attention_operator_version"] = CUDA_FP32_ATTENTION_OPERATOR_VERSION + if settings.optimizer_backend == "cpu_state_streaming_v1": + if device.type != "cuda": + raise ValueError("cpu_state_streaming_v1 requires CUDA") + specs = [[name, list(p.shape), str(p.dtype)] + for name, p in model.named_parameters() if p.requires_grad] + cfg["optimizer_algorithm_version"] = CPU_STATE_ADAMW_VERSION + cfg["optimizer_parameter_schema_sha256"] = hashlib.sha256(canonical(specs).encode()).hexdigest() data_hash = hashlib.sha256(canonical(manifest).encode()).hexdigest() config_hash = hashlib.sha256(canonical(cfg).encode()).hexdigest() if limits.branch > model.cfg.max_position_embeddings: @@ -136,9 +152,15 @@ def train(model: DecisionEngine, tokenizer, data: list[Datum], manifest: dict, o raise ValueError("streamed branch VJP requires CUDA BF16") if settings.optimizer_backend == "fused" and device.type != "cuda": raise ValueError("fused optimizer backend requires CUDA") - optimizer = torch.optim.AdamW((p for p in model.parameters() if p.requires_grad), lr=settings.learning_rate, - weight_decay=settings.weight_decay, - **({"fused": True} if settings.optimizer_backend == "fused" else {})) + if settings.optimizer_backend == "cpu_state_streaming_v1": + optimizer = CPUStateStreamingAdamW( + ((name, p) for name, p in model.named_parameters() if p.requires_grad), + lr=settings.learning_rate, weight_decay=settings.weight_decay, + max_state_bytes=settings.optimizer_state_max_bytes) + else: + optimizer = torch.optim.AdamW((p for p in model.parameters() if p.requires_grad), lr=settings.learning_rate, + weight_decay=settings.weight_decay, + **({"fused": True} if settings.optimizer_backend == "fused" else {})) scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lambda n: max(0.05, 1 - n / settings.steps)) sampler = Sampler(len(data), settings.seed) state = {"step": 0, "microbatches": 0, "examples": 0, "forward_tokens": 0, "accumulation_position": 0, @@ -190,6 +212,9 @@ def train(model: DecisionEngine, tokenizer, data: list[Datum], manifest: dict, o scheduler_ref = scheduler_ref or old.get("scheduler_identity") state["scheduler_identity"] = scheduler_ref parent_sha256 = payload["checkpoint_sha256"] + # The offload optimizer retains the needed CPU moment tensors itself. + # Release the temporary checkpoint dictionary and CPU model now. + del restored, restored_tokenizer, payload if quality_budget is not None: if quality_allocation_id is not None: quality_budget.claim_allocation(quality_allocation_id, state["run_id"]) diff --git a/tests/test_cpu_state_adamw.py b/tests/test_cpu_state_adamw.py new file mode 100644 index 0000000..48a3455 --- /dev/null +++ b/tests/test_cpu_state_adamw.py @@ -0,0 +1,214 @@ +"""Candidate qualification tests; copy under tests/ only after frozen run ends.""" +import copy +import pytest +import torch + +from kev_laya.cpu_state_adamw import CPUStateStreamingAdamW, TORCH_VERSION + + +def _optimizer(p, budget=1_000_000): + return CPUStateStreamingAdamW([("weight", p)], lr=3e-4, weight_decay=0.01, + max_state_bytes=budget) + + +def test_cpu_or_insufficient_budget_fails_closed(): + if torch.__version__ != TORCH_VERSION: + with pytest.raises(RuntimeError, match="requires torch"): + _optimizer(torch.nn.Parameter(torch.ones(4))) + return + with pytest.raises(ValueError, match="CUDA FP32"): + _optimizer(torch.nn.Parameter(torch.ones(4))) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +def test_streaming_matches_single_tensor_cuda_adamw_and_resumes(): + a = torch.nn.Parameter(torch.linspace(-1, 1, 513, device="cuda")) + b = torch.nn.Parameter(a.detach().clone()) + streamed = _optimizer(a) + native = torch.optim.AdamW([b], lr=3e-4, weight_decay=0.01, + foreach=False, fused=False, capturable=False) + saved = None + for index in range(3): + grad = torch.cos(torch.arange(a.numel(), device="cuda", dtype=torch.float32) + index) + a.grad = grad.clone(); b.grad = grad.clone() + streamed.step(); native.step() + torch.testing.assert_close(a, b, atol=0, rtol=0) + for key in ("exp_avg", "exp_avg_sq", "step"): + assert streamed.state[a][key].device.type == "cpu" + torch.testing.assert_close(streamed.state[a][key], native.state[b][key].cpu(), atol=0, rtol=0) + if index == 0: + saved = copy.deepcopy(streamed.state_dict()) + c = torch.nn.Parameter(a.detach().clone()) + resumed = _optimizer(c) + resumed.load_state_dict(saved) + assert all(v.device.type == "cpu" for v in resumed.state[c].values()) + elif index > 0: + c.grad = grad.clone(); resumed.step() + torch.testing.assert_close(c, a, atol=0, rtol=0) + assert saved is not None + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +def test_corruption_and_identity_rejected_without_cuda_migration(): + p = torch.nn.Parameter(torch.ones(13, device="cuda")) + opt = _optimizer(p) + with pytest.raises(MemoryError): + _optimizer(p, budget=1) + p.grad = torch.ones_like(p); opt.step() + clean = opt.state_dict() + for mutate in ( + lambda x: x["param_groups"][0].update(algorithm_version="wrong"), + lambda x: x["param_groups"][0].update(param_specs=[("other", (13,), "torch.float32")]), + lambda x: x["state"][0].update(exp_avg=torch.ones(13, device="cuda")), + lambda x: x["state"][0].update(exp_avg=torch.ones(12)), + lambda x: x["state"][0]["exp_avg"].fill_(float("nan")), + lambda x: x["state"][0]["exp_avg_sq"].fill_(float("inf")), + lambda x: x["state"][0]["exp_avg_sq"].fill_(-1), + lambda x: x["state"][0].update(step=torch.tensor(-1.0)), + lambda x: x["state"].update({0: None}), + lambda x: x["state"].update({0: []}), + lambda x: x["state"].update({0: 0}), + ): + broken = copy.deepcopy(clean); mutate(broken) + new = _optimizer(torch.nn.Parameter(torch.ones(13, device="cuda"))) + with pytest.raises(ValueError): + new.load_state_dict(broken) + assert not new.state + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +def test_rejected_load_preserves_populated_state_and_learning_rate(): + p = torch.nn.Parameter(torch.ones(13, device="cuda")) + opt = _optimizer(p) + p.grad = torch.full_like(p, 0.25); opt.step() + baseline = copy.deepcopy(opt.state_dict()) + rejected = copy.deepcopy(baseline) + rejected["param_groups"][0]["lr"] = 0.123 + rejected["state"][0]["exp_avg_sq"][0] = float("nan") + with pytest.raises(ValueError, match="nonfinite"): + opt.load_state_dict(rejected) + after = opt.state_dict() + assert after["param_groups"][0]["lr"] == baseline["param_groups"][0]["lr"] + for key in ("step", "exp_avg", "exp_avg_sq"): + torch.testing.assert_close(after["state"][0][key], baseline["state"][0][key], atol=0, rtol=0) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +def test_alternating_none_gradients_match_stock_and_resume(): + a = torch.nn.Parameter(torch.linspace(-1, 1, 17, device="cuda")) + b = torch.nn.Parameter(torch.linspace(1, -1, 17, device="cuda")) + x = torch.nn.Parameter(a.detach().clone()) + y = torch.nn.Parameter(b.detach().clone()) + streamed = CPUStateStreamingAdamW([("a", a), ("b", b)], lr=3e-4, + weight_decay=0.01, max_state_bytes=1_000_000) + stock = torch.optim.AdamW([x, y], lr=3e-4, weight_decay=0.01, + foreach=False, fused=False, capturable=False) + resumed = None + for index, active in enumerate(((True, True), (True, False), (False, True), (True, True))): + for parameter, reference, enabled, shift in ((a, x, active[0], 0), (b, y, active[1], 1)): + gradient = torch.cos(torch.arange(17, device="cuda", dtype=torch.float32) + index + shift) + parameter.grad = gradient.clone() if enabled else None + reference.grad = gradient.clone() if enabled else None + streamed.step(); stock.step() + for parameter, reference in ((a, x), (b, y)): + torch.testing.assert_close(parameter, reference, atol=0, rtol=0) + for field in ("step", "exp_avg", "exp_avg_sq"): + torch.testing.assert_close(streamed.state[parameter][field], + stock.state[reference][field].cpu(), atol=0, rtol=0) + if index == 0: + c = torch.nn.Parameter(a.detach().clone()) + d = torch.nn.Parameter(b.detach().clone()) + resumed = CPUStateStreamingAdamW([("a", c), ("b", d)], lr=3e-4, + weight_decay=0.01, max_state_bytes=1_000_000) + resumed.load_state_dict(copy.deepcopy(streamed.state_dict())) + else: + c.grad = a.grad.clone() if a.grad is not None else None + d.grad = b.grad.clone() if b.grad is not None else None + resumed.step() + torch.testing.assert_close(c, a, atol=0, rtol=0) + torch.testing.assert_close(d, b, atol=0, rtol=0) + assert resumed is not None + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +def test_equal_shape_parameter_id_swap_and_mutable_options_fail_closed(): + a = torch.nn.Parameter(torch.ones(8, device="cuda")) + b = torch.nn.Parameter(torch.zeros(8, device="cuda")) + opt = CPUStateStreamingAdamW([("a", a), ("b", b)], lr=3e-4, + weight_decay=0.01, max_state_bytes=1_000_000) + a.grad = torch.ones_like(a); b.grad = torch.full_like(b, 2) + opt.step() + clean = opt.state_dict() + swapped = copy.deepcopy(clean) + swapped["param_groups"][0]["params"] = [1, 0] + target = CPUStateStreamingAdamW( + [("a", torch.nn.Parameter(torch.ones(8, device="cuda"))), + ("b", torch.nn.Parameter(torch.zeros(8, device="cuda")))], + lr=3e-4, weight_decay=0.01, max_state_bytes=1_000_000) + with pytest.raises(ValueError, match="count/order"): + target.load_state_dict(swapped) + assert not target.state + for key, changed in (("foreach", True), ("fused", True), ("amsgrad", True), + ("capturable", True), ("weight_decay", 0.5), + ("algorithm_version", "wrong")): + mutation = copy.deepcopy(clean) + mutation["param_groups"][0][key] = changed + with pytest.raises(ValueError, match="options"): + target.load_state_dict(mutation) + assert not target.state + opt.param_groups[0][key] = changed + with pytest.raises(ValueError, match="options"): + opt.step() + opt.param_groups[0][key] = clean["param_groups"][0][key] + alias = copy.deepcopy(clean) + alias["state"][1]["exp_avg"] = alias["state"][0]["exp_avg"] + with pytest.raises(ValueError, match="aliased"): + target.load_state_dict(alias) + assert not target.state + step_alias = copy.deepcopy(clean) + step_alias["state"][0]["exp_avg"][0] = 1.0 + step_alias["state"][0]["step"] = step_alias["state"][0]["exp_avg"][0] + with pytest.raises(ValueError, match="aliased"): + target.load_state_dict(step_alias) + assert not target.state + cross_step_alias = copy.deepcopy(clean) + cross_step_alias["state"][1]["step"] = cross_step_alias["state"][0]["step"] + with pytest.raises(ValueError, match="aliased"): + target.load_state_dict(cross_step_alias) + assert not target.state + with pytest.raises(ValueError, match="duplicate"): + CPUStateStreamingAdamW([("same", a), ("same", b)], lr=3e-4, + weight_decay=0.01, max_state_bytes=1_000_000) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +def test_live_equal_shape_parameter_swap_rejected_before_step_or_serialization(): + a = torch.nn.Parameter(torch.ones(8, device="cuda")) + b = torch.nn.Parameter(torch.zeros(8, device="cuda")) + opt = CPUStateStreamingAdamW([("a", a), ("b", b)], lr=3e-4, + weight_decay=0.01, max_state_bytes=1_000_000) + a.grad = torch.ones_like(a); b.grad = torch.ones_like(b) + initial_a, initial_b = a.detach().clone(), b.detach().clone() + opt.param_groups[0]["params"].reverse() + with pytest.raises(ValueError, match="identity/order/layout"): + opt.step() + with pytest.raises(ValueError, match="identity/order/layout"): + opt.state_dict() + with pytest.raises(ValueError, match="identity/order/layout"): + opt.load_state_dict({"state": {}, "param_groups": []}) + torch.testing.assert_close(a, initial_a, atol=0, rtol=0) + torch.testing.assert_close(b, initial_b, atol=0, rtol=0) + opt.param_groups[0]["params"].reverse() + opt.param_groups[0]["params"][0] = torch.nn.Parameter(a.detach().clone()) + with pytest.raises(ValueError, match="identity/order/layout"): + opt.state_dict() + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +def test_none_gradient_does_not_advance_state_or_decay_parameter(): + p = torch.nn.Parameter(torch.ones(8, device="cuda")) + opt = _optimizer(p) + before = p.detach().clone() + opt.step() + assert not opt.state[p] + torch.testing.assert_close(p, before, atol=0, rtol=0) diff --git a/tests/test_cpu_state_training_contract.py b/tests/test_cpu_state_training_contract.py new file mode 100644 index 0000000..c86d1c0 --- /dev/null +++ b/tests/test_cpu_state_training_contract.py @@ -0,0 +1,119 @@ +"""CPU-only trainer integration contracts; run after candidate source integration.""" +import hashlib +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch + +import kev_laya.training as training +from kev_laya.encoding import ByteTokenizer, Limits +from kev_laya.objectives import ObjectiveConfig +from kev_laya.schema import canonical + + +LEGACY_HASHES = { + "default": "4c67e545c11fb19196a5298b56b144f2e334059b2735b6573b814521dde08100", + "fused": "1d88349ed3bf464bb39a83d257f1f2d58a8db54a8ba24fecc047c92794ee46af", +} + + +class BeforeOptimizer(Exception): + pass + + +@pytest.mark.parametrize("backend,budget", [ + ("cpu_state_streaming_v1", None), + ("cpu_state_streaming_v1", 0), + ("cpu_state_streaming_v1", -1), + ("cpu_state_streaming_v1", True), + ("default", 1), + ("fused", 1), +]) +def test_backend_budget_is_explicit_and_fail_closed(backend, budget): + with pytest.raises(ValueError): + training.TrainSettings(optimizer_backend=backend, + optimizer_state_max_bytes=budget) + + +def _capture_config_hash(monkeypatch, tmp_path: Path, tiny, backend: str, + *, pretend_cuda: bool = False) -> str: + seen = [] + original_canonical = training.canonical + + def capture(value): + if isinstance(value, dict) and set(value) >= {"settings", "objective", "limits"}: + seen.append(original_canonical(value)) + return original_canonical(value) + + def stop(*args, **kwargs): + raise BeforeOptimizer + + with monkeypatch.context() as patch: + patch.setattr(training, "canonical", capture) + patch.setattr(training.torch.optim, "AdamW", stop) + patch.setattr(training, "CPUStateStreamingAdamW", stop) + if pretend_cuda: + original_next = next + patch.setattr(training, "next", lambda iterator: SimpleNamespace( + device=torch.device("cuda")) if iterator is not None else original_next(iterator), + raising=False) + settings = training.TrainSettings(steps=1, optimizer_backend=backend, + **({"optimizer_state_max_bytes": 1_000_000} if backend == "cpu_state_streaming_v1" else {})) + try: + training.train(tiny, ByteTokenizer(), [], {"train": "frozen"}, + tmp_path / (backend + "-capture"), settings, + ObjectiveConfig(), Limits(512, 8192)) + except BeforeOptimizer: + pass + except ValueError as exc: + if backend != "fused" or pretend_cuda or "requires CUDA" not in str(exc): + raise + else: + raise AssertionError("config capture ran past optimizer construction") + assert len(seen) == 1 + return hashlib.sha256(seen[0].encode()).hexdigest() + + +@pytest.mark.parametrize("backend", ["default", "fused"]) +def test_legacy_config_hash_continuity(monkeypatch, tmp_path, tiny, backend): + assert _capture_config_hash(monkeypatch, tmp_path, tiny, backend) == LEGACY_HASHES[backend] + + +@pytest.mark.parametrize("source_backend,target_backend", [ + ("default", "cpu_state_streaming_v1"), + ("fused", "cpu_state_streaming_v1"), + ("cpu_state_streaming_v1", "default"), + ("cpu_state_streaming_v1", "fused"), +]) +def test_cross_backend_resume_refused_before_update(monkeypatch, tmp_path, tiny, + source_backend, target_backend): + source_hash = _capture_config_hash(monkeypatch, tmp_path, tiny, + source_backend, pretend_cuda=True) + target_hash = _capture_config_hash(monkeypatch, tmp_path, tiny, + target_backend, pretend_cuda=True) + assert source_hash != target_hash + manifest = {"train": "frozen"} + data_hash = hashlib.sha256(canonical(manifest).encode()).hexdigest() + + def fake_load(*args, **kwargs): + return tiny, ByteTokenizer(), {"execution": {}, "optimizer": {}, "sampler": {}, + "training_state": {"data_sha256": data_hash, "config_sha256": source_hash}} + + def cpu_optimizer(named_parameters, *, lr, weight_decay, max_state_bytes): + return torch.optim.AdamW((p for _, p in named_parameters), lr=lr, + weight_decay=weight_decay) + + with monkeypatch.context() as patch: + patch.setattr(training, "next", lambda iterator: SimpleNamespace( + device=torch.device("cuda")), raising=False) + patch.setattr(training, "load_checkpoint", fake_load) + patch.setattr(training, "CPUStateStreamingAdamW", cpu_optimizer) + target = training.TrainSettings(steps=1, optimizer_backend=target_backend, + **({"optimizer_state_max_bytes": 1_000_000} + if target_backend == "cpu_state_streaming_v1" else {})) + with pytest.raises(ValueError, match="resume configuration or frozen data differs"): + training.train(tiny, ByteTokenizer(), [object()], manifest, + tmp_path / (source_backend + "-to-" + target_backend), + target, ObjectiveConfig(), Limits(512, 8192), + resume=tmp_path / "synthetic-prior.pt") From dfdc4c12db27fff93a16976bc882d6379578e68c Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Wed, 30 Sep 2026 01:19:35 -0400 Subject: [PATCH 2/2] docs: record scoped CUDA CPU-state optimizer qualification --- docs/CPU_STATE_OPTIMIZER.md | 32 +- docs/RELEASE_READINESS.md | 13 + docs/acceptance-status.json | 17 + ...a-offload-lora-checkpoint-v2-20260930.json | 643 +++++++++++++++++ .../tiny-cuda-offload-v4-20260930.json | 648 ++++++++++++++++++ docs/research/layev-feature-mapping.json | 6 +- 6 files changed, 1355 insertions(+), 4 deletions(-) create mode 100644 docs/evidence/optimizer/tiny-cuda-offload-lora-checkpoint-v2-20260930.json create mode 100644 docs/evidence/optimizer/tiny-cuda-offload-v4-20260930.json diff --git a/docs/CPU_STATE_OPTIMIZER.md b/docs/CPU_STATE_OPTIMIZER.md index 32cd5d5..5616221 100644 --- a/docs/CPU_STATE_OPTIMIZER.md +++ b/docs/CPU_STATE_OPTIMIZER.md @@ -41,9 +41,35 @@ integration: the integration additionally fixes canonical parameter-schema serialization to use JSON lists. CPU configuration tests preserve default/fused hashes and exercise cross-backend resume rejection using simulated device metadata. -Production trainer uninterrupted versus fresh-process resume checks in FP32 and -BF16 are pending. Actual pretrained full-weight step/save/exit/resume/reload and -the captured-gradient stock optimizer oracle are also pending. Their fixed 16 GiB +The integrated source at `b5d8299b768564c09c541e3dfc04b3d28eef2fd6` subsequently +completed another owned primitive run: eight tests passed, zero skipped. The +[portable tiny qualification receipt](evidence/optimizer/tiny-cuda-offload-v4-20260930.json) +records all eight FP32/BF16 production-trainer process phases passing with no +skips: uninterrupted two steps, durable step1 exit, fresh-process resume to +step2, and reload/compare. All 33 production Python hashes and source HEAD +matched at each process start/end. Every model and optimizer tensor element, +scheduler, sampler, RNG and the whole training exposure matched exactly; resume +attempt lineage and each checkpoint's own parent chain were checked separately. + +This is a four-row synthetic byte-tokenizer fixture with 30 parameters and +28128 model tensor elements, not a pretrained fullweight result. Live persistent +CPU optimizer state was 225144 bytes; comparison covered 56286 optimizer tensor +elements. A zero tensor-element count for scalar-only fields means those fields +were asserted equal, not omitted. Effective matmul precision was highest, +TF32 was off, deterministic algorithms were enabled and the allocator cap +was 70%. The earlier external candidate primitive receipt remains distinct. + +A separate tiny **LoRA rank4 with activation checkpointing** run completed all +8/8 FP32/BF16 trainer phases with zero skips on the same frozen source. The +[portable LoRA receipt](evidence/optimizer/tiny-cuda-offload-lora-checkpoint-v2-20260930.json) +records exact model/optimizer/RNG/exposure and resume state comparisons, +12 trainable parameters, 26 unchanged frozen states, and observed LoRA/head +updates. Live CPU optimizer state was 15664 bytes. The integrated v4 primitive +proof was reused; this was not another primitive test run. These are synthetic +fixture results, with no native fullweight or acceptance claim. + +Actual pretrained full-weight step/save/exit/resume/reload and the +captured-gradient stock optimizer oracle remain pending. Their fixed 16 GiB available-host-RAM admission requirement is currently unmet. Existing default backend OOM receipts remain failures. Primitive or CPU fixture checks establish no independent full-weight derivative parity, trained long context, diff --git a/docs/RELEASE_READINESS.md b/docs/RELEASE_READINESS.md index e57b5e2..e591ccf 100644 --- a/docs/RELEASE_READINESS.md +++ b/docs/RELEASE_READINESS.md @@ -1,5 +1,18 @@ # Release readiness — intentionally incomplete +## Explicit CPU-state optimizer: partial qualification (September 30, 2026) + +The opt-in backend at signed source `b5d8299b768564c09c541e3dfc04b3d28eef2fd6` +passed the [tiny rank0 trainer qualification](evidence/optimizer/tiny-cuda-offload-v4-20260930.json) +and a separate [tiny LoRA rank4/activation-checkpointing qualification](evidence/optimizer/tiny-cuda-offload-lora-checkpoint-v2-20260930.json). +Each completed 8/8 FP32/BF16 uninterrupted/save/exit/fresh-resume/reload phases, +with zero skips and exact state comparisons. All 33 source hashes and HEAD +were unchanged at each phase start/end. The LoRA run reused the integrated v4 +8-pass primitive receipt; it did not rerun those tests. This is partial delivery +using synthetic byte-tokenizer fixtures. Actual pretrained fullweight optimizer +qualification and the captured-gradient stock oracle remain pending under their +separate 16GiB host gate. Default OOM receipts and issues #3/#4 remain unchanged. + ## Observed synthetic context decisions (September 30, 2026) The [development diagnostic](evidence/context/development-factorial96-20260930.md) diff --git a/docs/acceptance-status.json b/docs/acceptance-status.json index 8721a3b..f8cd62b 100644 --- a/docs/acceptance-status.json +++ b/docs/acceptance-status.json @@ -1,4 +1,21 @@ { + "explicit_CPU_state_optimizer_partial_delivery_2026_09_30": { + "signed_source_commit": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "scope": "partial synthetic tiny-fixture optimizer/trainer delivery; no native/fullweight/context/quality acceptance", + "rank0_receipt": "docs/evidence/optimizer/tiny-cuda-offload-v4-20260930.json", + "lora_checkpoint_receipt": "docs/evidence/optimizer/tiny-cuda-offload-lora-checkpoint-v2-20260930.json", + "rank0_precision_phases_passed": 8, + "lora_precision_phases_passed": 8, + "phase_skips": 0, + "integrated_primitive_passed": 8, + "primitive_skips": 0, + "lora_primitive_scope": "reused from rank0 v4; not rerun", + "lora_trainable_parameters": 12, + "lora_unchanged_frozen_states": 26, + "remaining_gate": "actual pretrained fullweight step/save/exit/resume/reload and captured-gradient stock optimizer oracle pending; separate16GiB available-host gate", + "issue_3": "open", + "issue_4": "open" + }, "context_development_diagnostic_2026_09_30": { "scope": "Observed synthetic short/medium development diagnostic; not acceptance, representative quality or Jev comparison", "receipt": "docs/evidence/context/development-factorial96-20260930.json", diff --git a/docs/evidence/optimizer/tiny-cuda-offload-lora-checkpoint-v2-20260930.json b/docs/evidence/optimizer/tiny-cuda-offload-lora-checkpoint-v2-20260930.json new file mode 100644 index 0000000..8a82423 --- /dev/null +++ b/docs/evidence/optimizer/tiny-cuda-offload-lora-checkpoint-v2-20260930.json @@ -0,0 +1,643 @@ +{ + "schema_version": 1, + "evidence_class": "tiny-synthetic-fixture LoRA rank4 activation-checkpointing CUDA production-trainer qualification", + "status": "complete", + "passed_within_scope": true, + "runtime_source_head": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "source_hashes": { + "src/kev_laya/__init__.py": "fccca0741142d06495613c3a03f56aee4ac89b7b409c1f86587c43f717f28a35", + "src/kev_laya/__main__.py": "79ee6f134c5c5193377cf9b189cbdcc3e2095e49ba9cec310114441c15a917c4", + "src/kev_laya/backbone_provenance.py": "06c6205d2d5a6b8681f5753393ebb39ff9041d9f98f5f74887b8096e79c3160c", + "src/kev_laya/checkpoint.py": "f7cd8cb15e130756751c53d6cb55ec2b7c3ef0c608da764d10b000552f1aab95", + "src/kev_laya/cli.py": "208502e86bdaf565d59bf9b72f6181d183027f0c60c4a43dcb6316b30234eccd", + "src/kev_laya/client.py": "ca2f5c798ebf76fbea61026242a626919b6a83f37e14368f3497c857596ebcac", + "src/kev_laya/counterfactual.py": "107c5ab7268ef657b6ac112696398752d40e25121cf4e1cb36aaeb7bbd84dcc5", + "src/kev_laya/cpu_state_adamw.py": "1a0ae5003728aa7acade8a92c33f9293134b1afe7bd5487890c9a4e47eca9105", + "src/kev_laya/data.py": "f954fa8cdeb3164ba56a4d2c77541df6aaa2b860a2f0f979d182213e04268b06", + "src/kev_laya/encoding.py": "3c945bd0390d3b0949a337ac651434942379e0d76d5d2fdff6fdb722b10f35e1", + "src/kev_laya/evaluation.py": "4b30d41328bb275b571492a1dcdcc16106d0ffa36863fc79d1bf0aac7ed3b402", + "src/kev_laya/execution.py": "41d7eaa6b3bf33c6e59c30fd2b36c7e086b29116179109a75dcbdc7d24d1756a", + "src/kev_laya/exposure.py": "3894954f8b84bf1008d6c928f35248b86c6d796a9acb0715f3e90d3974083500", + "src/kev_laya/io.py": "89c547a6399680dfa8d47511e3e5414875c61e97d87a6270efc871aa581850fe", + "src/kev_laya/jev_comparison.py": "4c256c3c8b5b3ae7aa3f6f7c72f8ad1eb1ee94800473255416b82a4448bd5753", + "src/kev_laya/lineage.py": "44e37b3838e4f2973aa74ed4e7ea4a18268cf46ce9dd6ce821dda2d3cc25bd1c", + "src/kev_laya/model.py": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "src/kev_laya/native_gradients.py": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f", + "src/kev_laya/native_validation.py": "9a55e053b8beee9aa12948faf317d1c6c2f59a2b4ffe4a3ae5a31e641fd2afa7", + "src/kev_laya/objectives.py": "d4f6089cfb3e9d6fe0e7bdaa75eb4e68d3571a78f278a193263756b2de3b5884", + "src/kev_laya/provenance.py": "5c0ef25846aab9d8b142eceda9c729f46308dd67e3881ffca2eb2d85064ede80", + "src/kev_laya/publishing.py": "bdaae8210b06b5e9e352c7a7e5cb0615aa3473fcf1c9c660d7b1187abda8b407", + "src/kev_laya/quality_budget.py": "4c57d637b4a31c829fef42df709076f96834b9f1f71033fefd23966772eab763", + "src/kev_laya/quality_protocol.py": "ad7aca42b57b58113f1831585dc42ab834a34c7a95e26b9aaa0d99af7c7f5f50", + "src/kev_laya/quality_readout.py": "8e0c862d7182c039a8ea1da6797be1a45fd4a2b02a42a21c9680ee502cf2d11b", + "src/kev_laya/quality_supervisor.py": "ef8b0be98cc03ade6673088a354aed634a10dcf7828b13243d845d33f8e2e917", + "src/kev_laya/registration.py": "63d6006c42eaea9f60e3b335d512328ba0cb1677e481d268fc6426c11cfa000e", + "src/kev_laya/schema.py": "105e02bb9d22bc3d90fb48ccbb3687b8e402ae0e6179a0570bf9a8391a92c644", + "src/kev_laya/service.py": "02ecec7f3729b318a2601cc14e8bf2165a2c0687151f10aedac648cab4561f5d", + "src/kev_laya/stages.py": "cba03c48edb2d5b2cf53042d5eebcac8b042bc81fb510757f66c0431ed8fb4c0", + "src/kev_laya/telemetry.py": "4b108a1dee06ce00b222ce77a4b18758a9c46167934dbe5097e902263ebd61f1", + "src/kev_laya/telemetry_contract.py": "9019ffa5e3d2e42b2b1e27248e8801413ff540007efb9c1448dc29b0e8b3646e", + "src/kev_laya/training.py": "e04f1ab3874f94699d8cb6e9695195d448decd56da86972f553388afe8052473" + }, + "all33_source_hashes_unchanged_at_each_phase_start_end": true, + "supervisor_terminal_sha256": "323d6cf8c4dd6718f0bf0a061c0fb0d13767dc669c3d61139ce9d0474291b3c9", + "frozen_inputs_sha256": "97d6ce7150c4c22d8b6056a6af23e3c911e336e59e5498111bdf2d8633b0966a", + "runner_sha256": "59e3d7c1c6dd753a1af9fd2127cc9fa9daacca00c10bd65d9095acf1ac5988d0", + "pins_sha256": "f4eac4d80d4a886c6734aec0c16fe48635320522d8e612ccfa4ad05d89bff3fb", + "primitive_test_hashes": { + "tests/test_cpu_state_adamw.py": "04e30b65d5b169f34001d0b2ec7a43136d61de460dc5efb0fda914ffa3a54646", + "tests/conftest.py": "9edead45d7071253e2f5d4c7f9ec2ac3291d453dd540588b34ed35f1bf530589" + }, + "fixture": { + "initializer_sha256": "df44414e200f37683591c64c0154ecbbda410b23a3165089e06f0b09f19a9ede", + "data_sha256": "d68a8d3a1f1c127372fe25d99708240141a341b21abe62dc332237a66cef60ae", + "evidence_class": "tiny-synthetic-fixture", + "seed": 19, + "init_seed": 47 + }, + "fixture_receipt_sha256": "33c119cfcd661f0710aaa52820358f75f62c1277e9881facabbdc017b40df472", + "initializer_manifest": { + "file": "init.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": null, + "resumable": false, + "sha256": "df44414e200f37683591c64c0154ecbbda410b23a3165089e06f0b09f19a9ede", + "size_bytes": 140398, + "training_steps": 0 + }, + "primitive_reused_not_rerun": { + "supervisor_sha256": "213a91d9e8153d9cfb388053efd8ac637e867f51459ca771ca61f6ec583fd19d", + "primitive_sha256": "3f3f403a0931be67e82f67723f0bec3ed8fd49bf5502ca1dc8d7fac23f9a9a37" + }, + "primitive_passed_in_prior_run": 8, + "primitive_skipped_in_prior_run": 0, + "precision_phase_successes": 8, + "precision_phase_required": 8, + "precision_phase_skips": 0, + "phases": [ + { + "status": "complete", + "phase": "init", + "head_start": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "head_end": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "receipt_file": "cpu-init.json", + "receipt_sha256": "8fa7afe45330a7cb5be17d5b4025188b91a8a060a3dc044a0862fa965f9b84cf", + "supervisor_elapsed_seconds": 14.759992099978263, + "worker_exitcode": 0, + "timeout": false, + "source_start_end_match": true + }, + { + "status": "complete", + "phase": "full", + "precision": "fp32", + "checkpoint_sha256": "32ad69fc50432444adde75ebfc2a9aaa5b93368d57e3bbdd823d7174a587cbc7", + "step": 2, + "live_cpu_moment_parameter_counts": [ + 12, + 12 + ], + "live_cpu_optimizer_state_bytes": [ + 15664, + 15664 + ], + "CPU_optimizer_state_ceiling_bytes": 16777216, + "effective_settings_start": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "effective_settings_end": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "start_resources": { + "available_host_bytes": 4127641600, + "scope": "tiny fixture only; actual-weight12/16GiB floors unchanged", + "GPU_free_bytes": 11799625728, + "GPU_total_bytes": 12884377600 + }, + "end_GPU_free_total_bytes": [ + 11615076352, + 12884377600 + ], + "peak_allocated_bytes": 86953472, + "peak_reserved_bytes": 96468992, + "head_start": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "head_end": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "native_fullweight_claim": false, + "checkpoint": "fp32-full/checkpoint-000002.pt", + "receipt_file": "fp32-full.json", + "receipt_sha256": "ecbc6bcad41fd47b0d327c6d1d6ad094cb5c02cc4348a51affef0b52f6aa2eec", + "supervisor_elapsed_seconds": 7.786693300004117, + "worker_exitcode": 0, + "timeout": false, + "source_start_end_match": true + }, + { + "status": "complete", + "phase": "split", + "precision": "fp32", + "checkpoint_sha256": "76bc9006d5954b5e5a3b4ab044739e9d04599b9fbc73f3080813c97af600828e", + "step": 1, + "live_cpu_moment_parameter_counts": [ + 12 + ], + "live_cpu_optimizer_state_bytes": [ + 15664 + ], + "CPU_optimizer_state_ceiling_bytes": 16777216, + "effective_settings_start": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "effective_settings_end": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "start_resources": { + "available_host_bytes": 4993875968, + "scope": "tiny fixture only; actual-weight12/16GiB floors unchanged", + "GPU_free_bytes": 11799625728, + "GPU_total_bytes": 12884377600 + }, + "end_GPU_free_total_bytes": [ + 11615076352, + 12884377600 + ], + "peak_allocated_bytes": 86950912, + "peak_reserved_bytes": 96468992, + "head_start": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "head_end": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "native_fullweight_claim": false, + "checkpoint": "fp32-split/checkpoint-000001.pt", + "receipt_file": "fp32-split.json", + "receipt_sha256": "676ab69e4caee00c9d95528bf5cd57de661e98e44a26781daea40d1fcd0c0c02", + "supervisor_elapsed_seconds": 4.579358100017998, + "worker_exitcode": 0, + "timeout": false, + "source_start_end_match": true + }, + { + "status": "complete", + "phase": "resume", + "precision": "fp32", + "checkpoint_sha256": "8cf25bd55d26271b3e7d61665c6b50a998f65076199e6b9f5503da0f5725b744", + "step": 2, + "live_cpu_moment_parameter_counts": [ + 12 + ], + "live_cpu_optimizer_state_bytes": [ + 15664 + ], + "CPU_optimizer_state_ceiling_bytes": 16777216, + "effective_settings_start": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "effective_settings_end": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "start_resources": { + "available_host_bytes": 4318109696, + "scope": "tiny fixture only; actual-weight12/16GiB floors unchanged", + "GPU_free_bytes": 11799625728, + "GPU_total_bytes": 12884377600 + }, + "end_GPU_free_total_bytes": [ + 11615076352, + 12884377600 + ], + "peak_allocated_bytes": 86952960, + "peak_reserved_bytes": 96468992, + "head_start": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "head_end": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "native_fullweight_claim": false, + "checkpoint": "fp32-split/checkpoint-000002.pt", + "receipt_file": "fp32-resume.json", + "receipt_sha256": "c159e296403dbc20b531335254cd18ab499d65e48dd7e81ddf2be0ad70e41bd5", + "supervisor_elapsed_seconds": 5.189867899985984, + "worker_exitcode": 0, + "timeout": false, + "source_start_end_match": true + }, + { + "status": "complete", + "phase": "compare", + "precision": "fp32", + "scope": "tiny LoRA rank4 activation checkpointing", + "exact_comparison_elements": { + "model": 29024, + "optimizer": 3916, + "scheduler": 0, + "sampler": 0, + "rng": 5072, + "training_exposure": 0, + "config": 0, + "execution": 0, + "preprocessing": 0 + }, + "optimizer_parameter_count": 12, + "trainable_parameter_names": [ + "backbone.layers.0.self_attn.q_proj.a", + "backbone.layers.0.self_attn.q_proj.b", + "backbone.layers.0.self_attn.v_proj.a", + "backbone.layers.0.self_attn.v_proj.b", + "backbone.layers.1.self_attn.q_proj.a", + "backbone.layers.1.self_attn.q_proj.b", + "backbone.layers.1.self_attn.v_proj.a", + "backbone.layers.1.self_attn.v_proj.b", + "head.q.weight", + "head.q.bias", + "head.k.weight", + "head.k.bias" + ], + "frozen_backbone_parameter_count": 26, + "frozen_model_state_count": 26, + "model_state_count": 38, + "full_manifest": { + "file": "checkpoint-000002.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "76bd701c907209cea8affe262b033649843213a566ab77b1671ba19ab965e14c", + "resumable": true, + "sha256": "32ad69fc50432444adde75ebfc2a9aaa5b93368d57e3bbdd823d7174a587cbc7", + "size_bytes": 177919, + "training_steps": 2 + }, + "split_manifest": { + "file": "checkpoint-000002.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "76bc9006d5954b5e5a3b4ab044739e9d04599b9fbc73f3080813c97af600828e", + "resumable": true, + "sha256": "8cf25bd55d26271b3e7d61665c6b50a998f65076199e6b9f5503da0f5725b744", + "size_bytes": 178751, + "training_steps": 2 + }, + "split_step1_manifest": { + "file": "checkpoint-000001.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "df44414e200f37683591c64c0154ecbbda410b23a3165089e06f0b09f19a9ede", + "resumable": true, + "sha256": "76bc9006d5954b5e5a3b4ab044739e9d04599b9fbc73f3080813c97af600828e", + "size_bytes": 177919, + "training_steps": 1 + }, + "head_start": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "head_end": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "native_fullweight_claim": false, + "receipt_file": "fp32-compare.json", + "receipt_sha256": "64978b49b17cd9ab9401f58d3a2905676b9b793c71434d1fa23af7a0aeb657fb", + "supervisor_elapsed_seconds": 3.598090500017861, + "worker_exitcode": 0, + "timeout": false, + "source_start_end_match": true + }, + { + "status": "complete", + "phase": "full", + "precision": "bf16", + "checkpoint_sha256": "3150e703a172b17c53fa86cf30836f4c4878c85ce8003bea29008eba00f94248", + "step": 2, + "live_cpu_moment_parameter_counts": [ + 12, + 12 + ], + "live_cpu_optimizer_state_bytes": [ + 15664, + 15664 + ], + "CPU_optimizer_state_ceiling_bytes": 16777216, + "effective_settings_start": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "effective_settings_end": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "start_resources": { + "available_host_bytes": 4425535488, + "scope": "tiny fixture only; actual-weight12/16GiB floors unchanged", + "GPU_free_bytes": 11799625728, + "GPU_total_bytes": 12884377600 + }, + "end_GPU_free_total_bytes": [ + 11610882048, + 12884377600 + ], + "peak_allocated_bytes": 74546176, + "peak_reserved_bytes": 94371840, + "head_start": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "head_end": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "native_fullweight_claim": false, + "checkpoint": "bf16-full/checkpoint-000002.pt", + "receipt_file": "bf16-full.json", + "receipt_sha256": "fe01d05239f9a4a567a5816c429528a2df959b9006958b8595e15a8b048f93bc", + "supervisor_elapsed_seconds": 6.878780400002142, + "worker_exitcode": 0, + "timeout": false, + "source_start_end_match": true + }, + { + "status": "complete", + "phase": "split", + "precision": "bf16", + "checkpoint_sha256": "9bcdc9e2750a682c1fe9d82e34313661b32a69eb6aab09d84ed65798482388b4", + "step": 1, + "live_cpu_moment_parameter_counts": [ + 12 + ], + "live_cpu_optimizer_state_bytes": [ + 15664 + ], + "CPU_optimizer_state_ceiling_bytes": 16777216, + "effective_settings_start": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "effective_settings_end": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "start_resources": { + "available_host_bytes": 4537671680, + "scope": "tiny fixture only; actual-weight12/16GiB floors unchanged", + "GPU_free_bytes": 11799625728, + "GPU_total_bytes": 12884377600 + }, + "end_GPU_free_total_bytes": [ + 11610882048, + 12884377600 + ], + "peak_allocated_bytes": 74518528, + "peak_reserved_bytes": 94371840, + "head_start": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "head_end": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "native_fullweight_claim": false, + "checkpoint": "bf16-split/checkpoint-000001.pt", + "receipt_file": "bf16-split.json", + "receipt_sha256": "5fed23684702ac7e52a3df9a1b838ab4d145ebbc74572fce431891c829b59b73", + "supervisor_elapsed_seconds": 5.160096300009172, + "worker_exitcode": 0, + "timeout": false, + "source_start_end_match": true + }, + { + "status": "complete", + "phase": "resume", + "precision": "bf16", + "checkpoint_sha256": "baf0b053d5994fca6c54495d8b2feb5fac1b28930b219f314968ae8b19309729", + "step": 2, + "live_cpu_moment_parameter_counts": [ + 12 + ], + "live_cpu_optimizer_state_bytes": [ + 15664 + ], + "CPU_optimizer_state_ceiling_bytes": 16777216, + "effective_settings_start": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "effective_settings_end": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "start_resources": { + "available_host_bytes": 4634771456, + "scope": "tiny fixture only; actual-weight12/16GiB floors unchanged", + "GPU_free_bytes": 11799625728, + "GPU_total_bytes": 12884377600 + }, + "end_GPU_free_total_bytes": [ + 11610882048, + 12884377600 + ], + "peak_allocated_bytes": 74545664, + "peak_reserved_bytes": 94371840, + "head_start": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "head_end": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "native_fullweight_claim": false, + "checkpoint": "bf16-split/checkpoint-000002.pt", + "receipt_file": "bf16-resume.json", + "receipt_sha256": "c028ffb83bc887ff8436f8f70690df20818e5faeee80a3eed5b4a48d31b49e99", + "supervisor_elapsed_seconds": 5.178113699977985, + "worker_exitcode": 0, + "timeout": false, + "source_start_end_match": true + }, + { + "status": "complete", + "phase": "compare", + "precision": "bf16", + "scope": "tiny LoRA rank4 activation checkpointing", + "exact_comparison_elements": { + "model": 29024, + "optimizer": 3916, + "scheduler": 0, + "sampler": 0, + "rng": 5072, + "training_exposure": 0, + "config": 0, + "execution": 0, + "preprocessing": 0 + }, + "optimizer_parameter_count": 12, + "trainable_parameter_names": [ + "backbone.layers.0.self_attn.q_proj.a", + "backbone.layers.0.self_attn.q_proj.b", + "backbone.layers.0.self_attn.v_proj.a", + "backbone.layers.0.self_attn.v_proj.b", + "backbone.layers.1.self_attn.q_proj.a", + "backbone.layers.1.self_attn.q_proj.b", + "backbone.layers.1.self_attn.v_proj.a", + "backbone.layers.1.self_attn.v_proj.b", + "head.q.weight", + "head.q.bias", + "head.k.weight", + "head.k.bias" + ], + "frozen_backbone_parameter_count": 26, + "frozen_model_state_count": 26, + "model_state_count": 38, + "full_manifest": { + "file": "checkpoint-000002.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "2bc8b6b21e76bc00c84ae8c3c31a759b70e4e2f14351997912c0fb4f29d22370", + "resumable": true, + "sha256": "3150e703a172b17c53fa86cf30836f4c4878c85ce8003bea29008eba00f94248", + "size_bytes": 177919, + "training_steps": 2 + }, + "split_manifest": { + "file": "checkpoint-000002.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "9bcdc9e2750a682c1fe9d82e34313661b32a69eb6aab09d84ed65798482388b4", + "resumable": true, + "sha256": "baf0b053d5994fca6c54495d8b2feb5fac1b28930b219f314968ae8b19309729", + "size_bytes": 178751, + "training_steps": 2 + }, + "split_step1_manifest": { + "file": "checkpoint-000001.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "df44414e200f37683591c64c0154ecbbda410b23a3165089e06f0b09f19a9ede", + "resumable": true, + "sha256": "9bcdc9e2750a682c1fe9d82e34313661b32a69eb6aab09d84ed65798482388b4", + "size_bytes": 177919, + "training_steps": 1 + }, + "head_start": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "head_end": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "native_fullweight_claim": false, + "receipt_file": "bf16-compare.json", + "receipt_sha256": "65a9dee14a5b2d5788509361cc69f8571338df07deafce98226212f7ee6c9c1d", + "supervisor_elapsed_seconds": 3.2711413000070024, + "worker_exitcode": 0, + "timeout": false, + "source_start_end_match": true + } + ], + "checkpoint_chains": { + "fp32-full": [ + { + "file": "checkpoint-000001.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "df44414e200f37683591c64c0154ecbbda410b23a3165089e06f0b09f19a9ede", + "resumable": true, + "sha256": "76bd701c907209cea8affe262b033649843213a566ab77b1671ba19ab965e14c", + "size_bytes": 177919, + "training_steps": 1, + "manifest_sha256": "84199d566d644d5f51229d1da79289cca4545b645a3170bef02f53a50797a1c2" + }, + { + "file": "checkpoint-000002.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "76bd701c907209cea8affe262b033649843213a566ab77b1671ba19ab965e14c", + "resumable": true, + "sha256": "32ad69fc50432444adde75ebfc2a9aaa5b93368d57e3bbdd823d7174a587cbc7", + "size_bytes": 177919, + "training_steps": 2, + "manifest_sha256": "f69dc4d1d0046b77664d9e346159733744eeae0a9973b410d736c80cde7eb236" + } + ], + "fp32-split": [ + { + "file": "checkpoint-000001.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "df44414e200f37683591c64c0154ecbbda410b23a3165089e06f0b09f19a9ede", + "resumable": true, + "sha256": "76bc9006d5954b5e5a3b4ab044739e9d04599b9fbc73f3080813c97af600828e", + "size_bytes": 177919, + "training_steps": 1, + "manifest_sha256": "11b7049933531c5fc1bf98e5621a8da5c86000cc8fb05686b71d380cd24382ca" + }, + { + "file": "checkpoint-000002.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "76bc9006d5954b5e5a3b4ab044739e9d04599b9fbc73f3080813c97af600828e", + "resumable": true, + "sha256": "8cf25bd55d26271b3e7d61665c6b50a998f65076199e6b9f5503da0f5725b744", + "size_bytes": 178751, + "training_steps": 2, + "manifest_sha256": "78ad28e4109cce9f35a4d18aa31f07e198675bc7ae1828abc3c013b43ef4d7ab" + } + ], + "bf16-full": [ + { + "file": "checkpoint-000001.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "df44414e200f37683591c64c0154ecbbda410b23a3165089e06f0b09f19a9ede", + "resumable": true, + "sha256": "2bc8b6b21e76bc00c84ae8c3c31a759b70e4e2f14351997912c0fb4f29d22370", + "size_bytes": 177919, + "training_steps": 1, + "manifest_sha256": "2b70ab26e13bf148000e7eaf0c912c382ee30ff30ac29a0543ee363b8d3750eb" + }, + { + "file": "checkpoint-000002.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "2bc8b6b21e76bc00c84ae8c3c31a759b70e4e2f14351997912c0fb4f29d22370", + "resumable": true, + "sha256": "3150e703a172b17c53fa86cf30836f4c4878c85ce8003bea29008eba00f94248", + "size_bytes": 177919, + "training_steps": 2, + "manifest_sha256": "d32af05e3053edddd0257fe2c907066eea51dddcd9eeac1eaa1ffffe27995122" + } + ], + "bf16-split": [ + { + "file": "checkpoint-000001.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "df44414e200f37683591c64c0154ecbbda410b23a3165089e06f0b09f19a9ede", + "resumable": true, + "sha256": "9bcdc9e2750a682c1fe9d82e34313661b32a69eb6aab09d84ed65798482388b4", + "size_bytes": 177919, + "training_steps": 1, + "manifest_sha256": "33f878f4383f799f5d881e6597d977fd04cdbe932cf89f1c80d08191b38515ff" + }, + { + "file": "checkpoint-000002.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "9bcdc9e2750a682c1fe9d82e34313661b32a69eb6aab09d84ed65798482388b4", + "resumable": true, + "sha256": "baf0b053d5994fca6c54495d8b2feb5fac1b28930b219f314968ae8b19309729", + "size_bytes": 178751, + "training_steps": 2, + "manifest_sha256": "dff72f3e1620f925aee25b48f8654882ae008680e8737c07cedde88f392384c0" + } + ] + }, + "LoRA_rank": 4, + "activation_checkpointing": true, + "frozen_states_unchanged_asserted_in_worker": 26, + "LoRA_and_head_updates_asserted_in_worker": true, + "checkpoint_bytes_and_manifest_links_verified_CPU_readback": true, + "comparison_count_semantics": "Counts report tensor elements only. Zero for scheduler/sampler/exposure/config/execution/preprocessing means those recursive non-tensor fields were asserted equal; it does not mean they were omitted. Every model/optimizer element and whole training_exposure was compared exactly. Attempt identities differ legitimately; resume lineage was checked separately.", + "native_fullweight_claim": false, + "acceptance_claim": false, + "representative_quality_claim": false, + "limitations": [ + "Four synthetic byte-tokenizer rows and a tiny LoRA rank4 architecture with activation checkpointing; no pretrained/native fullweight inference.", + "Primitive8 proof was reused from the prior integrated v4 run; no new primitive run occurred. Earlier external candidate primitive8 remains separate.", + "Actual-weight step/save/exit/resume/reload and captured-gradient stock optimizer oracle remain pending under their separate16GiB available-host gate.", + "Short fixture optimizer/resume equivalence does not establish independent native derivative parity, exact-limit context, representative quality, Jev benefit or issue closure.", + "Portable receipt omits local absolute paths; private raw receipts are identified by SHA256." + ] +} diff --git a/docs/evidence/optimizer/tiny-cuda-offload-v4-20260930.json b/docs/evidence/optimizer/tiny-cuda-offload-v4-20260930.json new file mode 100644 index 0000000..c26ba3d --- /dev/null +++ b/docs/evidence/optimizer/tiny-cuda-offload-v4-20260930.json @@ -0,0 +1,648 @@ +{ + "schema_version": 1, + "evidence_class": "tiny-synthetic-fixture CUDA production-trainer qualification", + "status": "complete", + "passed_within_scope": true, + "runtime_source_head": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "source_hashes": { + "src/kev_laya/__init__.py": "fccca0741142d06495613c3a03f56aee4ac89b7b409c1f86587c43f717f28a35", + "src/kev_laya/__main__.py": "79ee6f134c5c5193377cf9b189cbdcc3e2095e49ba9cec310114441c15a917c4", + "src/kev_laya/backbone_provenance.py": "06c6205d2d5a6b8681f5753393ebb39ff9041d9f98f5f74887b8096e79c3160c", + "src/kev_laya/checkpoint.py": "f7cd8cb15e130756751c53d6cb55ec2b7c3ef0c608da764d10b000552f1aab95", + "src/kev_laya/cli.py": "208502e86bdaf565d59bf9b72f6181d183027f0c60c4a43dcb6316b30234eccd", + "src/kev_laya/client.py": "ca2f5c798ebf76fbea61026242a626919b6a83f37e14368f3497c857596ebcac", + "src/kev_laya/counterfactual.py": "107c5ab7268ef657b6ac112696398752d40e25121cf4e1cb36aaeb7bbd84dcc5", + "src/kev_laya/cpu_state_adamw.py": "1a0ae5003728aa7acade8a92c33f9293134b1afe7bd5487890c9a4e47eca9105", + "src/kev_laya/data.py": "f954fa8cdeb3164ba56a4d2c77541df6aaa2b860a2f0f979d182213e04268b06", + "src/kev_laya/encoding.py": "3c945bd0390d3b0949a337ac651434942379e0d76d5d2fdff6fdb722b10f35e1", + "src/kev_laya/evaluation.py": "4b30d41328bb275b571492a1dcdcc16106d0ffa36863fc79d1bf0aac7ed3b402", + "src/kev_laya/execution.py": "41d7eaa6b3bf33c6e59c30fd2b36c7e086b29116179109a75dcbdc7d24d1756a", + "src/kev_laya/exposure.py": "3894954f8b84bf1008d6c928f35248b86c6d796a9acb0715f3e90d3974083500", + "src/kev_laya/io.py": "89c547a6399680dfa8d47511e3e5414875c61e97d87a6270efc871aa581850fe", + "src/kev_laya/jev_comparison.py": "4c256c3c8b5b3ae7aa3f6f7c72f8ad1eb1ee94800473255416b82a4448bd5753", + "src/kev_laya/lineage.py": "44e37b3838e4f2973aa74ed4e7ea4a18268cf46ce9dd6ce821dda2d3cc25bd1c", + "src/kev_laya/model.py": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "src/kev_laya/native_gradients.py": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f", + "src/kev_laya/native_validation.py": "9a55e053b8beee9aa12948faf317d1c6c2f59a2b4ffe4a3ae5a31e641fd2afa7", + "src/kev_laya/objectives.py": "d4f6089cfb3e9d6fe0e7bdaa75eb4e68d3571a78f278a193263756b2de3b5884", + "src/kev_laya/provenance.py": "5c0ef25846aab9d8b142eceda9c729f46308dd67e3881ffca2eb2d85064ede80", + "src/kev_laya/publishing.py": "bdaae8210b06b5e9e352c7a7e5cb0615aa3473fcf1c9c660d7b1187abda8b407", + "src/kev_laya/quality_budget.py": "4c57d637b4a31c829fef42df709076f96834b9f1f71033fefd23966772eab763", + "src/kev_laya/quality_protocol.py": "ad7aca42b57b58113f1831585dc42ab834a34c7a95e26b9aaa0d99af7c7f5f50", + "src/kev_laya/quality_readout.py": "8e0c862d7182c039a8ea1da6797be1a45fd4a2b02a42a21c9680ee502cf2d11b", + "src/kev_laya/quality_supervisor.py": "ef8b0be98cc03ade6673088a354aed634a10dcf7828b13243d845d33f8e2e917", + "src/kev_laya/registration.py": "63d6006c42eaea9f60e3b335d512328ba0cb1677e481d268fc6426c11cfa000e", + "src/kev_laya/schema.py": "105e02bb9d22bc3d90fb48ccbb3687b8e402ae0e6179a0570bf9a8391a92c644", + "src/kev_laya/service.py": "02ecec7f3729b318a2601cc14e8bf2165a2c0687151f10aedac648cab4561f5d", + "src/kev_laya/stages.py": "cba03c48edb2d5b2cf53042d5eebcac8b042bc81fb510757f66c0431ed8fb4c0", + "src/kev_laya/telemetry.py": "4b108a1dee06ce00b222ce77a4b18758a9c46167934dbe5097e902263ebd61f1", + "src/kev_laya/telemetry_contract.py": "9019ffa5e3d2e42b2b1e27248e8801413ff540007efb9c1448dc29b0e8b3646e", + "src/kev_laya/training.py": "e04f1ab3874f94699d8cb6e9695195d448decd56da86972f553388afe8052473" + }, + "all33_source_hashes_unchanged_at_each_phase_start_end": true, + "supervisor_terminal_sha256": "213a91d9e8153d9cfb388053efd8ac637e867f51459ca771ca61f6ec583fd19d", + "frozen_inputs_sha256": "03eb1c8e44e5182058934e000981bb46bf89ffde4d523145154cbb2c82d07af1", + "runner_sha256": "eb9bd625590f5db1f5664a5a054a14802f3e602788d219fa700c5eb2cc26960b", + "pins_sha256": "f3c10df524322ebdaf5fa1e77e8e203316451dae899293197c34dc10f42a0b25", + "primitive_test_hashes": { + "tests/test_cpu_state_adamw.py": "04e30b65d5b169f34001d0b2ec7a43136d61de460dc5efb0fda914ffa3a54646", + "tests/conftest.py": "9edead45d7071253e2f5d4c7f9ec2ac3291d453dd540588b34ed35f1bf530589" + }, + "fixture": { + "initializer_sha256": "50c30c260349affd9a67e341d33e71eebd18e693d88f9131d91c48203e315d1f", + "data_sha256": "d68a8d3a1f1c127372fe25d99708240141a341b21abe62dc332237a66cef60ae", + "evidence_class": "tiny-synthetic-fixture", + "seed": 19, + "init_seed": 47 + }, + "fixture_receipt_sha256": "f968142ba7b9b9a279c68b3ffc5fca5ab849eba3510072847987028832da22da", + "initializer_manifest": { + "file": "init.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": null, + "resumable": false, + "sha256": "50c30c260349affd9a67e341d33e71eebd18e693d88f9131d91c48203e315d1f", + "size_bytes": 134022, + "training_steps": 0 + }, + "primitive_passed": 8, + "primitive_skipped": 0, + "precision_phase_successes": 8, + "precision_phase_required": 8, + "precision_phase_skips": 0, + "phases": [ + { + "status": "complete", + "phase": "primitive", + "passed_tests": 8, + "collected_tests": 8, + "skipped_tests": 0, + "failed_tests": 0, + "effective_settings_start": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "effective_settings_end": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "start_resources": { + "available_host_bytes": 7578529792, + "scope": "tiny fixture only; actual-weight12/16GiB floors unchanged", + "GPU_free_bytes": 11799625728, + "GPU_total_bytes": 12884377600 + }, + "end_GPU_free_total_bytes": [ + 11780751360, + 12884377600 + ], + "peak_allocated_bytes": 33280, + "peak_reserved_bytes": 2097152, + "head_start": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "head_end": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "primitive_test_hashes": { + "tests/test_cpu_state_adamw.py": "04e30b65d5b169f34001d0b2ec7a43136d61de460dc5efb0fda914ffa3a54646", + "tests/conftest.py": "9edead45d7071253e2f5d4c7f9ec2ac3291d453dd540588b34ed35f1bf530589" + }, + "native_fullweight_claim": false, + "receipt_file": "cpu-primitive.json", + "receipt_sha256": "3f3f403a0931be67e82f67723f0bec3ed8fd49bf5502ca1dc8d7fac23f9a9a37", + "supervisor_elapsed_seconds": 15.835129100014456, + "worker_exitcode": 0, + "timeout": false, + "source_start_end_match": true + }, + { + "status": "complete", + "phase": "init", + "head_start": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "head_end": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "receipt_file": "cpu-init.json", + "receipt_sha256": "8fa7afe45330a7cb5be17d5b4025188b91a8a060a3dc044a0862fa965f9b84cf", + "supervisor_elapsed_seconds": 3.96942440001294, + "worker_exitcode": 0, + "timeout": false, + "source_start_end_match": true + }, + { + "status": "complete", + "phase": "full", + "precision": "fp32", + "checkpoint_sha256": "30d1597a22e085bb18dbc09f7fe25cbcbdcdd952b738cfefcf3ae1624362b05e", + "step": 2, + "live_cpu_moment_parameter_counts": [ + 30, + 30 + ], + "live_cpu_optimizer_state_bytes": [ + 225144, + 225144 + ], + "CPU_optimizer_state_ceiling_bytes": 16777216, + "effective_settings_start": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "effective_settings_end": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "start_resources": { + "available_host_bytes": 7492796416, + "scope": "tiny fixture only; actual-weight12/16GiB floors unchanged", + "GPU_free_bytes": 11799625728, + "GPU_total_bytes": 12884377600 + }, + "end_GPU_free_total_bytes": [ + 11612979200, + 12884377600 + ], + "peak_allocated_bytes": 87295488, + "peak_reserved_bytes": 98566144, + "head_start": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "head_end": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "native_fullweight_claim": false, + "checkpoint": "fp32-full/checkpoint-000002.pt", + "receipt_file": "fp32-full.json", + "receipt_sha256": "f6a235937042b8cc523737464f7f8795d8437dc79034ff529fd04fcff80ce20f", + "supervisor_elapsed_seconds": 7.333125500008464, + "worker_exitcode": 0, + "timeout": false, + "source_start_end_match": true + }, + { + "status": "complete", + "phase": "split", + "precision": "fp32", + "checkpoint_sha256": "5102e79697ee21091fd5351855e785c06c3402d0ba76cc6eec5fe8ce02ad2ada", + "step": 1, + "live_cpu_moment_parameter_counts": [ + 30 + ], + "live_cpu_optimizer_state_bytes": [ + 225144 + ], + "CPU_optimizer_state_ceiling_bytes": 16777216, + "effective_settings_start": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "effective_settings_end": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "start_resources": { + "available_host_bytes": 7241297920, + "scope": "tiny fixture only; actual-weight12/16GiB floors unchanged", + "GPU_free_bytes": 11799625728, + "GPU_total_bytes": 12884377600 + }, + "end_GPU_free_total_bytes": [ + 11615076352, + 12884377600 + ], + "peak_allocated_bytes": 87290368, + "peak_reserved_bytes": 96468992, + "head_start": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "head_end": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "native_fullweight_claim": false, + "checkpoint": "fp32-split/checkpoint-000001.pt", + "receipt_file": "fp32-split.json", + "receipt_sha256": "4d23c48620d7f848fff6e3712170ed5080cbfed25c5688b39574f59f7706aa47", + "supervisor_elapsed_seconds": 4.678749099984998, + "worker_exitcode": 0, + "timeout": false, + "source_start_end_match": true + }, + { + "status": "complete", + "phase": "resume", + "precision": "fp32", + "checkpoint_sha256": "a489e297c63f2a420519a78880a0ce54bce015e80106fe48e40795e8831f08e6", + "step": 2, + "live_cpu_moment_parameter_counts": [ + 30 + ], + "live_cpu_optimizer_state_bytes": [ + 225144 + ], + "CPU_optimizer_state_ceiling_bytes": 16777216, + "effective_settings_start": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "effective_settings_end": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "start_resources": { + "available_host_bytes": 7478607872, + "scope": "tiny fixture only; actual-weight12/16GiB floors unchanged", + "GPU_free_bytes": 11799625728, + "GPU_total_bytes": 12884377600 + }, + "end_GPU_free_total_bytes": [ + 11612979200, + 12884377600 + ], + "peak_allocated_bytes": 87294976, + "peak_reserved_bytes": 98566144, + "head_start": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "head_end": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "native_fullweight_claim": false, + "checkpoint": "fp32-split/checkpoint-000002.pt", + "receipt_file": "fp32-resume.json", + "receipt_sha256": "9ed4d913072fc2dd9c26787e66b777eff8987255951e2ccf686709d3c16d5719", + "supervisor_elapsed_seconds": 4.661549299984472, + "worker_exitcode": 0, + "timeout": false, + "source_start_end_match": true + }, + { + "status": "complete", + "phase": "compare", + "precision": "fp32", + "exact_comparison_elements": { + "model": 28128, + "optimizer": 56286, + "scheduler": 0, + "sampler": 0, + "rng": 5072, + "training_exposure": 0, + "config": 0, + "execution": 0, + "preprocessing": 0 + }, + "optimizer_parameter_count": 30, + "model_state_count": 30, + "full_manifest": { + "file": "checkpoint-000002.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "5093393ec873f6dfb9d500dd205862620a4bf0545df376de07e56184eb6fd873", + "resumable": true, + "sha256": "30d1597a22e085bb18dbc09f7fe25cbcbdcdd952b738cfefcf3ae1624362b05e", + "size_bytes": 397003, + "training_steps": 2 + }, + "split_manifest": { + "file": "checkpoint-000002.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "5102e79697ee21091fd5351855e785c06c3402d0ba76cc6eec5fe8ce02ad2ada", + "resumable": true, + "sha256": "a489e297c63f2a420519a78880a0ce54bce015e80106fe48e40795e8831f08e6", + "size_bytes": 397835, + "training_steps": 2 + }, + "split_step1_manifest": { + "file": "checkpoint-000001.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "50c30c260349affd9a67e341d33e71eebd18e693d88f9131d91c48203e315d1f", + "resumable": true, + "sha256": "5102e79697ee21091fd5351855e785c06c3402d0ba76cc6eec5fe8ce02ad2ada", + "size_bytes": 397003, + "training_steps": 1 + }, + "head_start": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "head_end": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "native_fullweight_claim": false, + "receipt_file": "fp32-compare.json", + "receipt_sha256": "3eec3e07a74e67af966eee88a4ed9f40f2aa7808fd38ed8c25a3f9816bde90f7", + "supervisor_elapsed_seconds": 3.6139021000126377, + "worker_exitcode": 0, + "timeout": false, + "source_start_end_match": true + }, + { + "status": "complete", + "phase": "full", + "precision": "bf16", + "checkpoint_sha256": "38e6a3d0489eed4a0518fdd0ec4f5924ba8a169c28325ff9f58a903cb4f70abd", + "step": 2, + "live_cpu_moment_parameter_counts": [ + 30, + 30 + ], + "live_cpu_optimizer_state_bytes": [ + 225144, + 225144 + ], + "CPU_optimizer_state_ceiling_bytes": 16777216, + "effective_settings_start": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "effective_settings_end": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "start_resources": { + "available_host_bytes": 6954450944, + "scope": "tiny fixture only; actual-weight12/16GiB floors unchanged", + "GPU_free_bytes": 11799625728, + "GPU_total_bytes": 12884377600 + }, + "end_GPU_free_total_bytes": [ + 11608784896, + 12884377600 + ], + "peak_allocated_bytes": 76699648, + "peak_reserved_bytes": 96468992, + "head_start": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "head_end": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "native_fullweight_claim": false, + "checkpoint": "bf16-full/checkpoint-000002.pt", + "receipt_file": "bf16-full.json", + "receipt_sha256": "d474ba74dbbfaae126a60a7b693add90c3dfce22e3f1652887817d302ab4123f", + "supervisor_elapsed_seconds": 7.150515199988149, + "worker_exitcode": 0, + "timeout": false, + "source_start_end_match": true + }, + { + "status": "complete", + "phase": "split", + "precision": "bf16", + "checkpoint_sha256": "7600e8dcdd34a2f05e6d4f2b62f530cdd291924f94a63735cec36851dfd0a4d7", + "step": 1, + "live_cpu_moment_parameter_counts": [ + 30 + ], + "live_cpu_optimizer_state_bytes": [ + 225144 + ], + "CPU_optimizer_state_ceiling_bytes": 16777216, + "effective_settings_start": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "effective_settings_end": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "start_resources": { + "available_host_bytes": 7093039104, + "scope": "tiny fixture only; actual-weight12/16GiB floors unchanged", + "GPU_free_bytes": 11799625728, + "GPU_total_bytes": 12884377600 + }, + "end_GPU_free_total_bytes": [ + 11608784896, + 12884377600 + ], + "peak_allocated_bytes": 76667392, + "peak_reserved_bytes": 96468992, + "head_start": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "head_end": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "native_fullweight_claim": false, + "checkpoint": "bf16-split/checkpoint-000001.pt", + "receipt_file": "bf16-split.json", + "receipt_sha256": "ef44b93b5daf03a3b3b18c918f1d4e59f4fdf6b4383ac97ea78d9f2e71cc405e", + "supervisor_elapsed_seconds": 5.353276000008918, + "worker_exitcode": 0, + "timeout": false, + "source_start_end_match": true + }, + { + "status": "complete", + "phase": "resume", + "precision": "bf16", + "checkpoint_sha256": "0202bf3827e03be46dc94cc804cd0ca477f8395a6d90fd1f4d16c4781aeb0b20", + "step": 2, + "live_cpu_moment_parameter_counts": [ + 30 + ], + "live_cpu_optimizer_state_bytes": [ + 225144 + ], + "CPU_optimizer_state_ceiling_bytes": 16777216, + "effective_settings_start": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "effective_settings_end": { + "float32_matmul_precision": "highest", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "deterministic_algorithms": true, + "allocator_fraction": 0.7 + }, + "start_resources": { + "available_host_bytes": 6866132992, + "scope": "tiny fixture only; actual-weight12/16GiB floors unchanged", + "GPU_free_bytes": 11799625728, + "GPU_total_bytes": 12884377600 + }, + "end_GPU_free_total_bytes": [ + 11608784896, + 12884377600 + ], + "peak_allocated_bytes": 76699136, + "peak_reserved_bytes": 96468992, + "head_start": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "head_end": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "native_fullweight_claim": false, + "checkpoint": "bf16-split/checkpoint-000002.pt", + "receipt_file": "bf16-resume.json", + "receipt_sha256": "b468386a007165893f54c4443d841522ad2612c42b2f5f3baaebe0d22cbd11dd", + "supervisor_elapsed_seconds": 5.403744600014761, + "worker_exitcode": 0, + "timeout": false, + "source_start_end_match": true + }, + { + "status": "complete", + "phase": "compare", + "precision": "bf16", + "exact_comparison_elements": { + "model": 28128, + "optimizer": 56286, + "scheduler": 0, + "sampler": 0, + "rng": 5072, + "training_exposure": 0, + "config": 0, + "execution": 0, + "preprocessing": 0 + }, + "optimizer_parameter_count": 30, + "model_state_count": 30, + "full_manifest": { + "file": "checkpoint-000002.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "a29f62d5a1c7e22c16c1a34fb97b66f26804147862018a43ca16181765d2abf9", + "resumable": true, + "sha256": "38e6a3d0489eed4a0518fdd0ec4f5924ba8a169c28325ff9f58a903cb4f70abd", + "size_bytes": 397003, + "training_steps": 2 + }, + "split_manifest": { + "file": "checkpoint-000002.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "7600e8dcdd34a2f05e6d4f2b62f530cdd291924f94a63735cec36851dfd0a4d7", + "resumable": true, + "sha256": "0202bf3827e03be46dc94cc804cd0ca477f8395a6d90fd1f4d16c4781aeb0b20", + "size_bytes": 397835, + "training_steps": 2 + }, + "split_step1_manifest": { + "file": "checkpoint-000001.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "50c30c260349affd9a67e341d33e71eebd18e693d88f9131d91c48203e315d1f", + "resumable": true, + "sha256": "7600e8dcdd34a2f05e6d4f2b62f530cdd291924f94a63735cec36851dfd0a4d7", + "size_bytes": 397003, + "training_steps": 1 + }, + "head_start": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "head_end": "b5d8299b768564c09c541e3dfc04b3d28eef2fd6", + "native_fullweight_claim": false, + "receipt_file": "bf16-compare.json", + "receipt_sha256": "344afed841ac979a976f8fca201c2fa573df13b1957cc9f25e19feb015f99e5b", + "supervisor_elapsed_seconds": 3.438619699998526, + "worker_exitcode": 0, + "timeout": false, + "source_start_end_match": true + } + ], + "checkpoint_chains": { + "fp32-full": [ + { + "file": "checkpoint-000001.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "50c30c260349affd9a67e341d33e71eebd18e693d88f9131d91c48203e315d1f", + "resumable": true, + "sha256": "5093393ec873f6dfb9d500dd205862620a4bf0545df376de07e56184eb6fd873", + "size_bytes": 397003, + "training_steps": 1, + "manifest_sha256": "126c6b02fe141dac17b52de82694b6646e0c296fa20127175f2f8a87c7d2b4de" + }, + { + "file": "checkpoint-000002.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "5093393ec873f6dfb9d500dd205862620a4bf0545df376de07e56184eb6fd873", + "resumable": true, + "sha256": "30d1597a22e085bb18dbc09f7fe25cbcbdcdd952b738cfefcf3ae1624362b05e", + "size_bytes": 397003, + "training_steps": 2, + "manifest_sha256": "5d2177419a1dfb605e4270b077af24c7f7b5bbab0dd1b674bbd2df17b148ec0b" + } + ], + "fp32-split": [ + { + "file": "checkpoint-000001.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "50c30c260349affd9a67e341d33e71eebd18e693d88f9131d91c48203e315d1f", + "resumable": true, + "sha256": "5102e79697ee21091fd5351855e785c06c3402d0ba76cc6eec5fe8ce02ad2ada", + "size_bytes": 397003, + "training_steps": 1, + "manifest_sha256": "bf94b2067b049fb82711126b589a37728f15dc5779136b93ebc6bc8f5fd8a511" + }, + { + "file": "checkpoint-000002.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "5102e79697ee21091fd5351855e785c06c3402d0ba76cc6eec5fe8ce02ad2ada", + "resumable": true, + "sha256": "a489e297c63f2a420519a78880a0ce54bce015e80106fe48e40795e8831f08e6", + "size_bytes": 397835, + "training_steps": 2, + "manifest_sha256": "de70b74e52a31a8f0710ff7cd8afc4a4190d285d4d13967cd0edadef5ca1daa5" + } + ], + "bf16-full": [ + { + "file": "checkpoint-000001.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "50c30c260349affd9a67e341d33e71eebd18e693d88f9131d91c48203e315d1f", + "resumable": true, + "sha256": "a29f62d5a1c7e22c16c1a34fb97b66f26804147862018a43ca16181765d2abf9", + "size_bytes": 397003, + "training_steps": 1, + "manifest_sha256": "56fed2395f3f25ac636b5ade8d31df1282e799d65ac0446cba14981b3fe82ff0" + }, + { + "file": "checkpoint-000002.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "a29f62d5a1c7e22c16c1a34fb97b66f26804147862018a43ca16181765d2abf9", + "resumable": true, + "sha256": "38e6a3d0489eed4a0518fdd0ec4f5924ba8a169c28325ff9f58a903cb4f70abd", + "size_bytes": 397003, + "training_steps": 2, + "manifest_sha256": "88462d4f8f3b982d6c0e50a0d5b8b9834769d33d97e99616c34aa965edc0ce4c" + } + ], + "bf16-split": [ + { + "file": "checkpoint-000001.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "50c30c260349affd9a67e341d33e71eebd18e693d88f9131d91c48203e315d1f", + "resumable": true, + "sha256": "7600e8dcdd34a2f05e6d4f2b62f530cdd291924f94a63735cec36851dfd0a4d7", + "size_bytes": 397003, + "training_steps": 1, + "manifest_sha256": "662df663df8d68bd06938f765a430c103f7e0fa05227cc1365bbcbae43714084" + }, + { + "file": "checkpoint-000002.pt", + "format": "kev-laya-checkpoint/1", + "parent_sha256": "7600e8dcdd34a2f05e6d4f2b62f530cdd291924f94a63735cec36851dfd0a4d7", + "resumable": true, + "sha256": "0202bf3827e03be46dc94cc804cd0ca477f8395a6d90fd1f4d16c4781aeb0b20", + "size_bytes": 397835, + "training_steps": 2, + "manifest_sha256": "8c3f4bdd49c4a84097b136c6f1d9ca9ec4db20416f2ac103864b1a368cf42d36" + } + ] + }, + "checkpoint_bytes_and_manifest_links_verified_CPU_readback": true, + "comparison_count_semantics": "Counts report tensor elements only. Zero for scheduler/sampler/exposure/config/execution/preprocessing means those recursive non-tensor fields were asserted equal; it does not mean they were omitted. Every model/optimizer element and whole training_exposure was compared exactly. Attempt identities differ legitimately; resume lineage was checked separately.", + "native_fullweight_claim": false, + "acceptance_claim": false, + "representative_quality_claim": false, + "limitations": [ + "Four synthetic byte-tokenizer rows and a tiny rank0 architecture; no pretrained/native fullweight inference.", + "Integrated primitive8 and precision phases are distinct from the earlier external candidate primitive8 receipt.", + "Actual-weight step/save/exit/resume/reload and captured-gradient stock optimizer oracle remain pending under their separate16GiB available-host gate.", + "Short fixture optimizer/resume equivalence does not establish independent native derivative parity, exact-limit context, representative quality, Jev benefit or issue closure.", + "Portable receipt omits local absolute paths; private raw receipts are identified by SHA256." + ] +} diff --git a/docs/research/layev-feature-mapping.json b/docs/research/layev-feature-mapping.json index 42e006a..2783753 100644 --- a/docs/research/layev-feature-mapping.json +++ b/docs/research/layev-feature-mapping.json @@ -607,7 +607,11 @@ "https://github.com/CompleteDotTech/layev/issues/3", "https://github.com/CompleteDotTech/layev/issues/5" ], - "limitations": "Source/test presence is not a passing current test or trained runtime result." + "limitations": "Source/test presence is not a passing current test or trained runtime result. Explicit CPU-state backend synthetic rank0 and LoRA rank4/activation-checkpointing FP32/BF16 trainer matrices each passed8/8 process phases, zero skips, with exact resume state comparisons on signedb5d8299. This does not establish actual pretrained fullweight optimizer feasibility, native derivative parity, context or representative quality; actualweight/oracle16GiB gate remains pending.", + "code_test_evidence": [ + "docs/evidence/optimizer/tiny-cuda-offload-v4-20260930.json", + "docs/evidence/optimizer/tiny-cuda-offload-lora-checkpoint-v2-20260930.json" + ] }, { "id": "calibration",