From 636ab1203d78e001203b51f7a5e2134323a7f1a4 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 29 Sep 2026 11:29:39 -0400 Subject: [PATCH] Add calibrated native fixed-limit service probe and receipt --- ...xed-limit-calibrated-service-20260929.json | 91 +++++++++++++ scripts/probe_native_context_service.py | 128 ++++++++++++++++++ 2 files changed, 219 insertions(+) create mode 100644 docs/evidence/native/fixed-limit-calibrated-service-20260929.json create mode 100644 scripts/probe_native_context_service.py diff --git a/docs/evidence/native/fixed-limit-calibrated-service-20260929.json b/docs/evidence/native/fixed-limit-calibrated-service-20260929.json new file mode 100644 index 0000000..2f79eac --- /dev/null +++ b/docs/evidence/native/fixed-limit-calibrated-service-20260929.json @@ -0,0 +1,91 @@ +{ + "schema_version": "native-context-service-evidence-v1", + "status": "partial_context_acceptance", + "source_base_commit": "9504b6ee37a778e4e8567bc4a0f642ea660a59f3", + "runner": "scripts/probe_native_context_service.py", + "scope": "Generated local authenticated service diagnostics on a calibrated pretrained checkpoint; no representative quality or Jev comparison claim.", + "identity": { + "backbone": "Qwen/Qwen2.5-0.5B", + "backbone_revision": "060db6499f32faf8b98477b0a26969ef7d8b9987", + "tokenizer_sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "training_checkpoint_sha256": "8472f22b8f02b14f27ccded353c2f378d71579a927324a25d9e5c818d4aaad90", + "calibrated_checkpoint_sha256": "a731903c8e91adc6206ecf60325b6ebbf4cdedd1349f0ab831c85069cf37df3d", + "verified_exposure_receipt_sha256": "6d7c1358b8c9d8d493f1883624c0bef7cec942d48afd14eaa6218e5f4b0849fb", + "observed_optimizer_steps": 2, + "observed_branch_32768_examples": 2, + "observed_aggregate_65536_examples": 2 + }, + "environment": { + "device": "NVIDIA GeForce RTX 3060", + "torch": "2.10.0+cu128", + "gpu_process_memory_fraction": 0.7 + }, + "fixed_limit_service": { + "private_receipt_sha256": "008e6f21b48a108acaeeb0b38a2e138608ae84240100385e02848871d8080f69", + "position": "beginning", + "language": "en", + "state_tokens": 27259, + "questions": 19, + "cases": [ + { + "name": "exact", + "max_branch_tokens": 32768, + "aggregate_tokens": 65536, + "serialized_bytes": 161058, + "http_status": 200, + "answers": 19, + "reported_input_tokens": 65536, + "reported_forward_tokens": 65536, + "calibration_status": "fitted-held-out", + "elapsed_seconds": 67.4210898, + "peak_gpu_allocated_bytes": 4666471936, + "peak_gpu_reserved_bytes": 5739905024 + }, + { + "name": "aggregate_plus_one", + "max_branch_tokens": 32768, + "aggregate_tokens": 65537, + "serialized_bytes": 161059, + "http_status": 422, + "rejection_limit": "aggregate_input", + "elapsed_seconds": 0.1323966 + }, + { + "name": "branch_plus_one", + "max_branch_tokens": 32769, + "aggregate_tokens": 65537, + "serialized_bytes": 161059, + "http_status": 422, + "rejection_limit": "state_plus_branch", + "elapsed_seconds": 0.0498554 + } + ] + }, + "related_edge_shape_service": { + "private_receipt_sha256": "080f23617c529f49bce6b87ee7cf94fc9c60c0713bd922546c3580b19c4cda1f", + "choice_options": 255, + "score_levels": 10, + "noul_options": 2, + "logical_tokens": 5492, + "serialized_bytes": 11219, + "elapsed_seconds": 2.2627882, + "peak_gpu_allocated_bytes": 2403747328 + }, + "six_stratum_native_validator": { + "private_report_sha256": "9b3f05edb105c2a09f80d0acf6cb28178ae64dd3329ba93415b505fcc240bc60", + "positions": ["beginning", "middle", "end"], + "languages": ["en", "es"], + "numerical_cases_passed": 6, + "overflow_cases_passed": 6, + "marker_decisions_correct": 6, + "marker_decisions_total": 18, + "marker_accuracy": 0.3333333333333333, + "required_marker_accuracy": 0.8, + "overall_passed": false + }, + "limitations": [ + "The exact and edge-shape service probes use generated requests and do not establish representative decision quality.", + "The six-stratum diagnostic remains below its frozen decision threshold.", + "Full FP32/BF16 native mode parity and Jev-relative quality remain open." + ] +} diff --git a/scripts/probe_native_context_service.py b/scripts/probe_native_context_service.py new file mode 100644 index 0000000..a76727a --- /dev/null +++ b/scripts/probe_native_context_service.py @@ -0,0 +1,128 @@ +"""Probe fixed native context limits through the local authenticated service. + +This generated diagnostic measures request handling, not decision quality. +It never sends a request to a remote service or writes model weights. +""" +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import time +from pathlib import Path + +import torch +from fastapi.testclient import TestClient + +from kev_laya.checkpoint import load_checkpoint +from kev_laya.encoding import Limits, encode_request +from kev_laya.io import atomic_json, sha256_file +from kev_laya.native_validation import make_case +from kev_laya.schema import SystemOneRequest, SystemOneResponse +from kev_laya.service import InferenceRuntime, ServeSettings, create_app + + +def measured_shape(data: dict, tokenizer) -> dict: + encoded = encode_request(SystemOneRequest.model_validate(data), tokenizer, + Limits(262144, 524288, 1024)) + return {"state_tokens": len(encoded.state), + "max_branch_tokens": max(len(encoded.state) + len(branch.ids) for branch in encoded.branches), + "logical_tokens": encoded.logical_tokens, + "questions": len(encoded.branches)} + + +def one_token_over(exact: dict, tokenizer, question_id: str) -> tuple[dict, dict]: + for suffix in ("a", "x", "0", ".", "!", "?", " x", " a", " z", "\n", " q", " zz"): + candidate = copy.deepcopy(exact) + candidate["questions"][question_id]["instructions"] += suffix + shape = measured_shape(candidate, tokenizer) + if shape["logical_tokens"] == 65537: + return candidate, shape + raise ValueError(f"no measured one-token suffix for {question_id}") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--checkpoint", type=Path, required=True) + parser.add_argument("--checkpoint-sha256", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--device", default="cuda:0") + parser.add_argument("--gpu-process-fraction", type=float, default=0.70) + parser.add_argument("--position", choices=("beginning", "middle", "end"), default="beginning") + parser.add_argument("--language", choices=("en", "es"), default="en") + args = parser.parse_args() + if args.output.exists(): + raise FileExistsError(args.output) + if not 0 < args.gpu_process_fraction <= 1: + raise ValueError("GPU process fraction must be in (0, 1]") + if sha256_file(args.checkpoint) != args.checkpoint_sha256: + raise ValueError("checkpoint hash mismatch") + device = torch.device(args.device) + if device.type != "cuda" or not torch.cuda.is_available(): + raise ValueError("native context probe requires CUDA") + torch.cuda.set_per_process_memory_fraction(args.gpu_process_fraction, device) + torch.backends.cuda.matmul.allow_tf32 = False + model, tokenizer, _ = load_checkpoint(args.checkpoint, device) + model.eval() + exact = make_case(tokenizer, args.position, args.language).model_dump() + exact_shape = measured_shape(exact, tokenizer) + if exact_shape["max_branch_tokens"] != 32768 or exact_shape["logical_tokens"] != 65536: + raise ValueError("generated request no longer sits on both boundaries") + aggregate_over, aggregate_shape = one_token_over( + exact, tokenizer, next(reversed(exact["questions"]))) + if aggregate_shape["max_branch_tokens"] != 32768: + raise ValueError("aggregate overflow also changed maximum branch") + branch_over, branch_shape = one_token_over(exact, tokenizer, "route") + if branch_shape["max_branch_tokens"] != 32769: + raise ValueError("branch request is not one token beyond its limit") + + runtime = InferenceRuntime(model, tokenizer, "kev-laya-native-context-probe", Limits()) + client = TestClient(create_app(runtime, ServeSettings(api_keys=("local-generated-probe",)))) + cases = [] + for name, data, shape in (("exact", exact, exact_shape), + ("aggregate_plus_one", aggregate_over, aggregate_shape), + ("branch_plus_one", branch_over, branch_shape)): + torch.cuda.reset_peak_memory_stats(device) + torch.cuda.synchronize(device) + started = time.perf_counter() + response = client.post("/v1/systemone", json=data, + headers={"Authorization": "Bearer local-generated-probe"}) + torch.cuda.synchronize(device) + row = {"name": name, "shape": shape, + "request_sha256": hashlib.sha256(SystemOneRequest.model_validate(data).model_dump_json().encode()).hexdigest(), + "serialized_bytes": len(json.dumps(data, ensure_ascii=False).encode()), + "status_code": response.status_code, "elapsed_seconds": time.perf_counter() - started, + "peak_allocated_bytes": torch.cuda.max_memory_allocated(device), + "peak_reserved_bytes": torch.cuda.max_memory_reserved(device)} + if name == "exact": + if response.status_code != 200: + raise ValueError(f"exact request failed: {response.status_code} {response.text[:300]}") + result = SystemOneResponse.model_validate(response.json()) + if result.usage.input_tokens != 65536 or len(result.answers) != shape["questions"]: + raise ValueError("exact response lost questions or tokens") + row.update({"answers": len(result.answers), "returned_input_tokens": result.usage.input_tokens, + "returned_forward_tokens": result.usage.forward_tokens, + "calibration_status": result.calibration_status}) + else: + detail = response.json().get("detail") + expected = "aggregate_input" if name == "aggregate_plus_one" else "state_plus_branch" + if (response.status_code != 422 or not isinstance(detail, dict) or + detail.get("code") != "context_overflow" or detail.get("limit") != expected): + raise ValueError(f"over-limit request was not rejected as {expected}: {response.status_code} {detail}") + row["rejection"] = detail + cases.append(row) + print(f"{name}: HTTP {response.status_code}", flush=True) + atomic_json(args.output, {"status": "passed_fixed_limit_native_probe", + "evidence_class": "generated_exact_context_mechanics_not_quality", + "checkpoint_sha256": args.checkpoint_sha256, + "position": args.position, "language": args.language, + "device": torch.cuda.get_device_name(device), + "torch_version": torch.__version__, + "gpu_process_memory_fraction": args.gpu_process_fraction, + "cases": cases}) + print(sha256_file(args.output), flush=True) + + +if __name__ == "__main__": + main()