Skip to content

Add calibrated native fixed-limit service evidence #55

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
CompleteDotTech merged 1 commit into from
Sep 29, 2026
Merged

Add calibrated native fixed-limit service evidence #55

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 91 additions & 0 deletions docs/evidence/native/fixed-limit-calibrated-service-20260929.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
{
"schema_version": "native-context-service-evidence-v1",
"status": "partial_context_acceptance",
"source_base_commit": "9504b6ee37a778e4e8567bc4a0f642ea660a59f3",
"runner": "scripts/probe_native_context_service.py",
"scope": "Generated local authenticated service diagnostics on a calibrated pretrained checkpoint; no representative quality or Jev comparison claim.",
"identity": {
"backbone": "Qwen/Qwen2.5-0.5B",
"backbone_revision": "060db6499f32faf8b98477b0a26969ef7d8b9987",
"tokenizer_sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539",
"training_checkpoint_sha256": "8472f22b8f02b14f27ccded353c2f378d71579a927324a25d9e5c818d4aaad90",
"calibrated_checkpoint_sha256": "a731903c8e91adc6206ecf60325b6ebbf4cdedd1349f0ab831c85069cf37df3d",
"verified_exposure_receipt_sha256": "6d7c1358b8c9d8d493f1883624c0bef7cec942d48afd14eaa6218e5f4b0849fb",
"observed_optimizer_steps": 2,
"observed_branch_32768_examples": 2,
"observed_aggregate_65536_examples": 2
},
"environment": {
"device": "NVIDIA GeForce RTX 3060",
"torch": "2.10.0+cu128",
"gpu_process_memory_fraction": 0.7
},
"fixed_limit_service": {
"private_receipt_sha256": "008e6f21b48a108acaeeb0b38a2e138608ae84240100385e02848871d8080f69",
"position": "beginning",
"language": "en",
"state_tokens": 27259,
"questions": 19,
"cases": [
{
"name": "exact",
"max_branch_tokens": 32768,
"aggregate_tokens": 65536,
"serialized_bytes": 161058,
"http_status": 200,
"answers": 19,
"reported_input_tokens": 65536,
"reported_forward_tokens": 65536,
"calibration_status": "fitted-held-out",
"elapsed_seconds": 67.4210898,
"peak_gpu_allocated_bytes": 4666471936,
"peak_gpu_reserved_bytes": 5739905024
},
{
"name": "aggregate_plus_one",
"max_branch_tokens": 32768,
"aggregate_tokens": 65537,
"serialized_bytes": 161059,
"http_status": 422,
"rejection_limit": "aggregate_input",
"elapsed_seconds": 0.1323966
},
{
"name": "branch_plus_one",
"max_branch_tokens": 32769,
"aggregate_tokens": 65537,
"serialized_bytes": 161059,
"http_status": 422,
"rejection_limit": "state_plus_branch",
"elapsed_seconds": 0.0498554
}
]
},
"related_edge_shape_service": {
"private_receipt_sha256": "080f23617c529f49bce6b87ee7cf94fc9c60c0713bd922546c3580b19c4cda1f",
"choice_options": 255,
"score_levels": 10,
"noul_options": 2,
"logical_tokens": 5492,
"serialized_bytes": 11219,
"elapsed_seconds": 2.2627882,
"peak_gpu_allocated_bytes": 2403747328
},
"six_stratum_native_validator": {
"private_report_sha256": "9b3f05edb105c2a09f80d0acf6cb28178ae64dd3329ba93415b505fcc240bc60",
"positions": ["beginning", "middle", "end"],
"languages": ["en", "es"],
"numerical_cases_passed": 6,
"overflow_cases_passed": 6,
"marker_decisions_correct": 6,
"marker_decisions_total": 18,
"marker_accuracy": 0.3333333333333333,
"required_marker_accuracy": 0.8,
"overall_passed": false
},
"limitations": [
"The exact and edge-shape service probes use generated requests and do not establish representative decision quality.",
"The six-stratum diagnostic remains below its frozen decision threshold.",
"Full FP32/BF16 native mode parity and Jev-relative quality remain open."
]
}
128 changes: 128 additions & 0 deletions scripts/probe_native_context_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
"""Probe fixed native context limits through the local authenticated service.

This generated diagnostic measures request handling, not decision quality.
It never sends a request to a remote service or writes model weights.
"""
from __future__ import annotations

import argparse
import copy
import hashlib
import json
import time
from pathlib import Path

import torch
from fastapi.testclient import TestClient

from kev_laya.checkpoint import load_checkpoint
from kev_laya.encoding import Limits, encode_request
from kev_laya.io import atomic_json, sha256_file
from kev_laya.native_validation import make_case
from kev_laya.schema import SystemOneRequest, SystemOneResponse
from kev_laya.service import InferenceRuntime, ServeSettings, create_app


def measured_shape(data: dict, tokenizer) -> dict:
encoded = encode_request(SystemOneRequest.model_validate(data), tokenizer,
Limits(262144, 524288, 1024))
return {"state_tokens": len(encoded.state),
"max_branch_tokens": max(len(encoded.state) + len(branch.ids) for branch in encoded.branches),
"logical_tokens": encoded.logical_tokens,
"questions": len(encoded.branches)}


def one_token_over(exact: dict, tokenizer, question_id: str) -> tuple[dict, dict]:
for suffix in ("a", "x", "0", ".", "!", "?", " x", " a", " z", "\n", " q", " zz"):
candidate = copy.deepcopy(exact)
candidate["questions"][question_id]["instructions"] += suffix
shape = measured_shape(candidate, tokenizer)
if shape["logical_tokens"] == 65537:
return candidate, shape
raise ValueError(f"no measured one-token suffix for {question_id}")


def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--checkpoint", type=Path, required=True)
parser.add_argument("--checkpoint-sha256", required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--device", default="cuda:0")
parser.add_argument("--gpu-process-fraction", type=float, default=0.70)
parser.add_argument("--position", choices=("beginning", "middle", "end"), default="beginning")
parser.add_argument("--language", choices=("en", "es"), default="en")
args = parser.parse_args()
if args.output.exists():
raise FileExistsError(args.output)
if not 0 < args.gpu_process_fraction <= 1:
raise ValueError("GPU process fraction must be in (0, 1]")
if sha256_file(args.checkpoint) != args.checkpoint_sha256:
raise ValueError("checkpoint hash mismatch")
device = torch.device(args.device)
if device.type != "cuda" or not torch.cuda.is_available():
raise ValueError("native context probe requires CUDA")
torch.cuda.set_per_process_memory_fraction(args.gpu_process_fraction, device)
torch.backends.cuda.matmul.allow_tf32 = False
model, tokenizer, _ = load_checkpoint(args.checkpoint, device)
model.eval()
exact = make_case(tokenizer, args.position, args.language).model_dump()
exact_shape = measured_shape(exact, tokenizer)
if exact_shape["max_branch_tokens"] != 32768 or exact_shape["logical_tokens"] != 65536:
raise ValueError("generated request no longer sits on both boundaries")
aggregate_over, aggregate_shape = one_token_over(
exact, tokenizer, next(reversed(exact["questions"])))
if aggregate_shape["max_branch_tokens"] != 32768:
raise ValueError("aggregate overflow also changed maximum branch")
branch_over, branch_shape = one_token_over(exact, tokenizer, "route")
if branch_shape["max_branch_tokens"] != 32769:
raise ValueError("branch request is not one token beyond its limit")

runtime = InferenceRuntime(model, tokenizer, "kev-laya-native-context-probe", Limits())
client = TestClient(create_app(runtime, ServeSettings(api_keys=("local-generated-probe",))))
cases = []
for name, data, shape in (("exact", exact, exact_shape),
("aggregate_plus_one", aggregate_over, aggregate_shape),
("branch_plus_one", branch_over, branch_shape)):
torch.cuda.reset_peak_memory_stats(device)
torch.cuda.synchronize(device)
started = time.perf_counter()
response = client.post("/v1/systemone", json=data,
headers={"Authorization": "Bearer local-generated-probe"})
torch.cuda.synchronize(device)
row = {"name": name, "shape": shape,
"request_sha256": hashlib.sha256(SystemOneRequest.model_validate(data).model_dump_json().encode()).hexdigest(),
"serialized_bytes": len(json.dumps(data, ensure_ascii=False).encode()),
"status_code": response.status_code, "elapsed_seconds": time.perf_counter() - started,
"peak_allocated_bytes": torch.cuda.max_memory_allocated(device),
"peak_reserved_bytes": torch.cuda.max_memory_reserved(device)}
if name == "exact":
if response.status_code != 200:
raise ValueError(f"exact request failed: {response.status_code} {response.text[:300]}")
result = SystemOneResponse.model_validate(response.json())
if result.usage.input_tokens != 65536 or len(result.answers) != shape["questions"]:
raise ValueError("exact response lost questions or tokens")
row.update({"answers": len(result.answers), "returned_input_tokens": result.usage.input_tokens,
"returned_forward_tokens": result.usage.forward_tokens,
"calibration_status": result.calibration_status})
else:
detail = response.json().get("detail")
expected = "aggregate_input" if name == "aggregate_plus_one" else "state_plus_branch"
if (response.status_code != 422 or not isinstance(detail, dict) or
detail.get("code") != "context_overflow" or detail.get("limit") != expected):
raise ValueError(f"over-limit request was not rejected as {expected}: {response.status_code} {detail}")
row["rejection"] = detail
cases.append(row)
print(f"{name}: HTTP {response.status_code}", flush=True)
atomic_json(args.output, {"status": "passed_fixed_limit_native_probe",
"evidence_class": "generated_exact_context_mechanics_not_quality",
"checkpoint_sha256": args.checkpoint_sha256,
"position": args.position, "language": args.language,
"device": torch.cuda.get_device_name(device),
"torch_version": torch.__version__,
"gpu_process_memory_fraction": args.gpu_process_fraction,
"cases": cases})
print(sha256_file(args.output), flush=True)


if __name__ == "__main__":
main()
Loading