diff --git a/.github/workflows/no-gpu.yml b/.github/workflows/no-gpu.yml index 597ff6b522..a4c558e9dc 100644 --- a/.github/workflows/no-gpu.yml +++ b/.github/workflows/no-gpu.yml @@ -24,6 +24,10 @@ jobs: # The kernel contract's limits gate: every registered launch against M1-M5 and sm_80/86/89/120/121, controls red. - name: zig build test-kernel-contract run: zig build test-kernel-contract + - name: Clef head inspection CLI (CPU only) + run: | + zig build tf-clef-inspect + python3 tools/zig/test_clef_inspect.py zig-out/bin/tf-clef-inspect # The frozen server goldens: request parsing, rendering and streaming against recorded answers. - name: zig build test-golden run: zig build test-golden diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 74546d20cd..bad38d8ccb 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -232,3 +232,12 @@ TensorFold. What they follow or copy: - The Level Zero loader and the Intel compute runtime are loaded at run time and are not bundled; `ocloc` compiles the kernels at build time and is not distributed. +## Clef head checkpoint + +`zig/src/families/clef/weights.zig` reads the separate bf16 head format from Cloudflare's +[`joint_schema_model.py`](https://huggingface.co/Cloudflare/clef-flash/blob/8b2e5fd17c09fd49bc2880b805d5323ec7ff4ff3/joint_schema_model.py), +Apache License 2.0. The earlier TensorFold work by @outcastofmusic in +[#241](https://github.com/ashhart/TensorFold/pull/241) and +[#251](https://github.com/ashhart/TensorFold/pull/251) established the separate head and backbone interface. +`zig/tests/fixtures/clef/` contains a small synthetic head generated with the release code at seed 317. +Its source metadata identifies the release revision. It contains no trained model weights or agreement corpus. diff --git a/build.zig b/build.zig index 380a38b3de..085f30381a 100644 --- a/build.zig +++ b/build.zig @@ -161,6 +161,22 @@ pub fn build(b: *std.Build) void { b.step("test-cost-rule", "CPU scripted window and interleaved head timing").dependOn(&cost_run.step); test_step.dependOn(&cost_run.step); merge_build.targets(b, target, optimize); // tf-merge-check and test-merge, host only on every OS + const clef_head = b.addTest(.{ .root_module = b.createModule(.{ + .root_source_file = b.path("zig/clef_checkpoint_cases.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + }) }); + const clef_head_run = b.addRunArtifact(clef_head); + b.step("test-clef-checkpoint", "CPU Clef checkpoint and disposable-pass contracts").dependOn(&clef_head_run.step); + test_step.dependOn(&clef_head_run.step); + const clef_inspect = b.addExecutable(.{ .name = "tf-clef-inspect", .root_module = b.createModule(.{ + .root_source_file = b.path("zig/clef_inspect.zig"), + .target = target, + .optimize = .safe, + .link_libc = true, + }) }); + b.step("tf-clef-inspect", "Build CPU-only Clef head and backbone-config inspection").dependOn(&b.addInstallArtifact(clef_inspect, .{}).step); switch (target.result.os.tag) { .macos => metalTargets(b, target, optimize, draft_ids, build_options, test_step), .linux => cuda_build.targets(b, target, optimize, draft_ids, build_options), diff --git a/tools/zig/test_clef_inspect.py b/tools/zig/test_clef_inspect.py new file mode 100644 index 0000000000..06111a8b0c --- /dev/null +++ b/tools/zig/test_clef_inspect.py @@ -0,0 +1,79 @@ +"""CPU CLI receipts distinguish a validated head from a plain or damaged checkpoint.""" +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +BINARY = Path(sys.argv.pop(1)).resolve() if len(sys.argv) > 1 and not sys.argv[1].startswith("-") else ROOT / "zig-out/bin/tf-clef-inspect" +FIXTURE = ROOT / "zig/tests/fixtures/clef" +CONFIG = { + "model_type": "qwen3_5_text", "hidden_size": 16, "intermediate_size": 32, + "num_hidden_layers": 4, "num_attention_heads": 2, "num_key_value_heads": 1, + "head_dim": 8, "vocab_size": 97, "linear_num_key_heads": 2, + "linear_num_value_heads": 4, "linear_key_head_dim": 4, + "linear_value_head_dim": 4, "linear_conv_kernel_dim": 4, "eos_token_id": 2, +} + + +class InspectTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.path = Path(self.tmp.name) + (self.path / "config.json").write_text(json.dumps(CONFIG)) + + def pair(self): + for name in ("joint_head_config.json", "joint_head.safetensors"): + shutil.copyfile(FIXTURE / name, self.path / name) + + def run_cli(self): + return subprocess.run([str(BINARY), str(self.path), "--check"], text=True, capture_output=True, timeout=30) + + def rejected(self): + result = self.run_cli() + self.assertNotEqual(0, result.returncode) + self.assertEqual("", result.stdout, "a failed check must not emit a success receipt") + + def test_complete_pair_has_verified_payload_hash_and_no_model_readiness_claim(self): + self.pair() + result = self.run_cli() + self.assertEqual(0, result.returncode, result.stderr) + record = json.loads(result.stdout) + self.assertTrue(record["head_present"] and record["head_validated"]) + self.assertEqual(122, record["head_tensors"]) + self.assertEqual(16, record["backbone_hidden"]) + self.assertEqual(8, record["head_config"]["width"]) + self.assertEqual(hashlib.sha256((self.path / "joint_head.safetensors").read_bytes()).hexdigest(), record["head_file_sha256"]) + self.assertFalse(record["backbone_weights_checked"] or record["gpu"] or record["model_ready"]) + + def test_plain_pack_has_no_validated_head_or_hash(self): + result = self.run_cli() + self.assertEqual(0, result.returncode, result.stderr) + record = json.loads(result.stdout) + self.assertFalse(record["head_present"] or record["head_validated"] or record["model_ready"]) + self.assertEqual(0, record["head_tensors"]) + self.assertIsNone(record["head_config"]) + self.assertIsNone(record["head_file_sha256"]) + + def test_incomplete_pair_and_truncated_payload_are_errors(self): + shutil.copyfile(FIXTURE / "joint_head_config.json", self.path / "joint_head_config.json") + self.rejected() + self.pair() + payload = self.path / "joint_head.safetensors" + payload.write_bytes(payload.read_bytes()[:-2]) + self.rejected() + + def test_incompatible_backbone_is_an_error(self): + self.pair() + (self.path / "config.json").write_text(json.dumps({**CONFIG, "hidden_size": 32})) + self.rejected() + + +if __name__ == "__main__": + unittest.main() diff --git a/zig/build/qwen27.zig b/zig/build/qwen27.zig index 842e0073b2..4c19645d67 100644 --- a/zig/build/qwen27.zig +++ b/zig/build/qwen27.zig @@ -111,6 +111,7 @@ pub fn targets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.bu _ = program(b, "tf-qwen27-dflash-run", "Build the explicit draft prototype and its plain comparison on real weights", "zig/tests/qwen27/dflash_run.zig", target, true, family); const engine_only: []const std.Build.Module.Import = &.{ .{ .name = "tensorfold", .module = engine }, .{ .name = "metal", .module = metal } }; _ = program(b, "tf-qwen27-synthetic", "Build bounded synthetic native hybrid chunk checks, no models", "zig/tests/qwen27/synthetic.zig", target, false, engine_only); + _ = program(b, "tf-qwen27-hidden-rows-check", "Build disposable normalized-row and chat/resume checks without running a GPU", "zig/tests/qwen27/hidden_rows.zig", target, false, engine_only); _ = program(b, "tf-qwen27-dflash-synthetic", "Build model-free native draft context/forward/tree check", "zig/tests/qwen27/dflash_runtime.zig", target, false, engine_only); _ = program(b, "tf-qwen27-widths", "Build the projections at every oQ width before tensor units: float64, row windows, mixed stacks, embedding", "zig/tests/qwen27/widths.zig", target, false, engine_only); _ = program(b, "tf-qwen27-dflash-refine", "Build the drafter refinement probe (top-1 by block position, masked versus given)", "zig/tests/qwen27/dflash_refine.zig", target, false, engine_only); @@ -118,6 +119,7 @@ pub fn targets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.bu const api = b.createModule(.{ .root_source_file = b.path("zig/src/core/engine_api.zig"), .target = target, .optimize = .ReleaseSafe, .link_libc = true, .imports = &.{.{ .name = "lanes", .module = lanes }} }); const host = b.createModule(.{ .root_source_file = b.path("zig/src/native/qwen27_served.zig"), .target = target, .optimize = .ReleaseSafe, .link_libc = true, .imports = &.{ .{ .name = "engine_api", .module = api }, .{ .name = "metal", .module = metal }, .{ .name = "tensorfold", .module = engine } } }); const served: []const std.Build.Module.Import = &.{ .{ .name = "tensorfold", .module = engine }, .{ .name = "metal", .module = metal }, .{ .name = "engine_api", .module = api }, .{ .name = "qwen27_served", .module = host } }; + _ = program(b, "tf-qwen27-hidden-rows-served-check", "Build queued capture versus solo and concurrent chat checks without running a GPU", "zig/tests/qwen27/hidden_rows_served.zig", target, false, served); _ = program(b, "tf-qwen27-served-check", "Build synthetic CLI versus served-host token check", "zig/tests/qwen27/served.zig", target, false, served); _ = program(b, "tf-qwen27-dflash-served-check", "Build synthetic drafted CLI versus served-host token check", "zig/tests/qwen27/dflash_served.zig", target, false, served); _ = program(b, "tf-qwen27-lanes-check", "Build synthetic lane-core versus plain greedy token and state check", "zig/tests/qwen27/lanes_check.zig", target, false, served); diff --git a/zig/clef_checkpoint_cases.zig b/zig/clef_checkpoint_cases.zig new file mode 100644 index 0000000000..652325692a --- /dev/null +++ b/zig/clef_checkpoint_cases.zig @@ -0,0 +1,7 @@ +//! CPU-only Clef head admission and disposable-pass contracts; no device or scoring arithmetic. +test { + _ = @import("src/families/clef/weights.zig"); + _ = @import("src/families/clef/checkpoint.zig"); + _ = @import("src/families/qwen27/ane_handoff.zig"); + _ = @import("src/families/qwen27/hidden_rows_plan.zig"); +} diff --git a/zig/clef_inspect.zig b/zig/clef_inspect.zig new file mode 100644 index 0000000000..4ea2763254 --- /dev/null +++ b/zig/clef_inspect.zig @@ -0,0 +1,44 @@ +//! CPU inspection of a separate Clef head against its backbone config; no backbone payload or device is opened. +const std = @import("std"); +const checkpoint = @import("src/families/clef/checkpoint.zig"); +const qwen = @import("src/families/qwen27/config.zig"); + +fn digest(bytes: []const u8) [64]u8 { + var sum: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(bytes, &sum, .{}); + return std.fmt.bytesToHex(sum, .lower); +} + +pub fn main(init: std.process.Init) !void { + const a = init.arena.allocator(); + const io = init.io; + const args = try init.minimal.args.toSlice(a); + if (args.len != 3 or !std.mem.eql(u8, args[2], "--check")) return error.CheckModeRequired; + const path = try std.fs.path.join(a, &.{ args[1], "config.json" }); + const text = try std.Io.Dir.cwd().readFileAlloc(io, path, a, .limited(1 << 20)); + const c = try qwen.parse(a, text, null); + var head = try checkpoint.loadOptional(init.gpa, io, args[1], c.hidden); + defer if (head) |*h| h.deinit(io); + const config_sha = digest(text); + var head_sha: [64]u8 = undefined; + const sha: ?[]const u8 = if (head) |h| blk: { + head_sha = digest(h.file.map.memory); + break :blk &head_sha; + } else null; + const record = try std.json.Stringify.valueAlloc(a, .{ + .kind = "clef_head_checkpoint", + .version = 1, + .head_present = head != null, + .head_validated = head != null, + .head_tensors = if (head) |h| h.file.names.count() else @as(usize, 0), + .head_config = if (head) |h| h.config else null, + .head_file_sha256 = sha, + .backbone_hidden = c.hidden, + .backbone_config_sha256 = @as([]const u8, &config_sha), + .backbone_weights_checked = false, + .gpu = false, + .model_ready = false, + }, .{}); + try std.Io.File.stdout().writeStreamingAll(io, record); + try std.Io.File.stdout().writeStreamingAll(io, "\n"); +} diff --git a/zig/src/core/lane_host.zig b/zig/src/core/lane_host.zig index 4a8f7d3493..fee64f995c 100644 --- a/zig/src/core/lane_host.zig +++ b/zig/src/core/lane_host.zig @@ -11,6 +11,7 @@ const admit = @import("lane_host_admit.zig"); const recalls = @import("lane_host_recall.zig"); const ends = @import("lane_host_end.zig"); const rows_host = @import("lane_host_rows.zig"); +const calls = @import("lane_host_calls.zig"); const Allocator = std.mem.Allocator; const Id = api.Id; const Request = api.Request; @@ -36,6 +37,8 @@ pub const LaneHost = struct { closing: bool = false, stale: bool = false, // the learner's last step: a put-back has not reached the device, so replies are refused thread: ?std.Thread = null, + owner_thread: ?std.Thread.Id = null, + calls: calls.Queue = .{}, decoded: std.ArrayList(status.Mark) = .empty, // tokens a round landed, for the 2 s decode rate prefill_rate: f64 = 0, prefill_at: i96 = 0, @@ -141,6 +144,11 @@ pub const LaneHost = struct { } }; } + /// Family work waits for the engine thread; ctx must remain valid until this returns, including cancellation. + pub fn callOwner(h: *LaneHost, ctx: *anyopaque, operation: calls.Run) calls.Error!void { + return calls.submit(h, ctx, operation); + } + /// The family's queue as a keepalive target, when it set one. fn keepaliveFn(ctx: *anyopaque) ?api.keepalive.Target { return self(ctx).keepalive_target; @@ -207,9 +215,9 @@ pub const LaneHost = struct { return job.host.foregroundWaiting() or job.host.core.activeCount() > 0; } - /// A foreground request is queued (they queue ahead of every background one); the caller holds the lock. + /// A foreground request or family call waits; the caller holds the lock. pub fn foregroundWaiting(h: *const LaneHost) bool { - return h.queued.items.len > 0 and !h.queued.items[0].request.background; + return h.calls.first != null or (h.queued.items.len > 0 and !h.queued.items[0].request.background); } /// Waits `warm_quiet` for other work (a request, a decoding stream, a cancel): true if none came. @@ -495,10 +503,14 @@ pub const LaneHost = struct { } fn run(h: *LaneHost) void { + h.lock(); + h.owner_thread = std.Thread.getCurrentId(); + h.unlock(); while (true) { ends.takeCancels(h); rows_host.answer(h); learn.refuse(h); + calls.one(h); while (h.admitOne()) {} h.fillOne(); status.noteLive(h); @@ -525,7 +537,7 @@ pub const LaneHost = struct { continue; } if (rows_host.idle(h)) continue; - if (h.cancels.items.len == 0 and (h.queued.items.len == 0 or h.admitted.items.len >= h.info_.lanes)) + if (h.calls.first == null and h.cancels.items.len == 0 and (h.queued.items.len == 0 or h.admitted.items.len >= h.info_.lanes)) h.wake.waitTimeout(h.io, &h.mutex, .{ .duration = .{ .raw = .fromMilliseconds(100), .clock = .awake } }) catch {}; h.unlock(); continue; diff --git a/zig/src/core/lane_host_calls.zig b/zig/src/core/lane_host_calls.zig new file mode 100644 index 0000000000..3d21c0a219 --- /dev/null +++ b/zig/src/core/lane_host_calls.zig @@ -0,0 +1,111 @@ +//! Family operations run on the engine thread between rounds, with caller-owned completion storage. +const std = @import("std"); +const LaneHost = @import("lane_host.zig").LaneHost; + +pub const Error = error{ Closed, NotStarted, ReentrantOwnerCall, StaleModel }; +pub const Run = *const fn (*anyopaque) void; + +const Call = struct { + next: ?*Call = null, + io: std.Io, + ctx: *anyopaque, + run: Run, + mutex: std.Io.Mutex = .init, + wake: std.Io.Condition = .init, + done: bool = false, + failure: ?Error = null, + + fn finish(c: *Call, failure: ?Error) void { + const io = c.io; + c.mutex.lockUncancelable(io); + c.failure = failure; + c.done = true; + c.wake.signal(io); + c.mutex.unlock(io); + // The caller can now return and destroy this node. Never touch c again. + } + + fn wait(c: *Call) Error!void { + const io = c.io; + c.mutex.lockUncancelable(io); + defer c.mutex.unlock(io); + while (!c.done) c.wake.waitUncancelable(io, &c.mutex); + if (c.failure) |err| return err; + } +}; + +/// Intrusive nodes stay on their callers' stacks until their own completion is observed. +pub const Queue = struct { + first: ?*Call = null, + last: ?*Call = null, + + fn append(q: *Queue, c: *Call) void { + if (q.last) |last| last.next = c else q.first = c; + q.last = c; + } + + fn pop(q: *Queue) ?*Call { + const c = q.first orelse return null; + q.first = c.next; + if (q.first == null) q.last = null; + c.next = null; + return c; + } +}; + +/// Once accepted, even a canceled caller waits until the owner has finished using ctx. +pub fn submit(h: *LaneHost, ctx: *anyopaque, run: Run) Error!void { + var c = Call{ .io = h.io, .ctx = ctx, .run = run }; + h.lock(); + if (h.closing) { + h.unlock(); + return error.Closed; + } + if (h.stale) { + h.unlock(); + return error.StaleModel; + } + if (h.thread == null) { + h.unlock(); + return error.NotStarted; + } + if (h.owner_thread == std.Thread.getCurrentId()) { + h.unlock(); + return error.ReentrantOwnerCall; + } + h.calls.append(&c); + h.wake.signal(h.io); + h.unlock(); + // No host access after enqueue: stop may free it while this caller observes its own completion. + try c.wait(); +} + +/// At most one operation per ordinary loop iteration; a running lesson reaches its safe end first. +pub fn one(h: *LaneHost) void { + h.lock(); + if (h.stale) { + const pending = detach(&h.calls); + h.unlock(); + return reject(pending, error.StaleModel); + } + const c = if (h.closing or h.lesson_open) null else h.calls.pop(); + h.unlock(); + const call = c orelse return; + call.run(call.ctx); + call.finish(null); +} + +/// Detach pending work under the host lock, then finish it without that lock or host storage. +pub fn detach(q: *Queue) ?*Call { + const first = q.first; + q.* = .{}; + return first; +} + +pub fn reject(first: ?*Call, failure: Error) void { + var next = first; + while (next) |c| { + next = c.next; + c.finish(failure); + } +} diff --git a/zig/src/core/lane_host_calls_test.zig b/zig/src/core/lane_host_calls_test.zig new file mode 100644 index 0000000000..274c624488 --- /dev/null +++ b/zig/src/core/lane_host_calls_test.zig @@ -0,0 +1,393 @@ +//! Owner calls through a real host thread: serialization, shutdown, reentrancy and chat fairness. +const std = @import("std"); +const lanes = @import("lanes"); +const api = @import("engine_api.zig"); +const LaneHost = @import("lane_host.zig").LaneHost; +const a = std.testing.allocator; +const io = std.testing.io; + +const Rig = struct { + cfg: lanes.Config = undefined, + target: lanes.fake.Fake = .{ .gpa = a }, + clock: lanes.fake.FixedClock = .{}, + core: lanes.Engine = undefined, + host: LaneHost = undefined, + stopped: bool = false, + + fn init(r: *Rig) !void { + r.cfg = try lanes.Config.init(a, .{ .exact_width = 8, .gpu_tokens = true, .hidden_rows = true }, 8, 7); + r.core = lanes.Engine.init(a, &r.cfg, r.target.backend(), r.clock.clock()); + r.host = LaneHost.init(a, io, &r.core, .{ .lanes = 2 }); + } + fn deinit(r: *Rig) void { + if (!r.stopped) r.host.stop(); + r.core.deinit(); + r.target.deinit(); + r.cfg.deinit(a); + } + fn stop(r: *Rig) void { + r.host.stop(); + r.stopped = true; + } +}; + +const Flag = std.atomic.Value(bool); +fn wait(flag: *const Flag) !void { + for (0..5000) |_| { + if (flag.load(.acquire)) return; + std.Io.sleep(io, .fromMilliseconds(1), .awake) catch {}; + } + return error.TestTimeout; +} + +const Threads = struct { + threads: [32]?std.Thread = @splat(null), + count: usize = 0, + fn add(t: *Threads, comptime f: anytype, args: anytype) !void { + t.threads[t.count] = try std.Thread.spawn(.{}, f, args); + t.count += 1; + } + fn join(t: *Threads) void { + for (t.threads[0..t.count]) |thread| thread.?.join(); + t.count = 0; + } +}; + +const Job = struct { + host: *LaneHost, + gate: ?*const Flag = null, + cancel: ?*const Flag = null, + entered: Flag = .init(false), + returned: Flag = .init(false), + ran: bool = false, + saw_cancel: bool = false, + reenter: bool = false, + failure: ?anyerror = null, + nested_failure: ?anyerror = null, + owner: ?std.Thread.Id = null, + counter: ?*std.atomic.Value(usize) = null, + observed_count: usize = 0, + + fn exec(ptr: *anyopaque) void { + const j: *Job = @ptrCast(@alignCast(ptr)); + j.owner = std.Thread.getCurrentId(); + if (j.counter) |counter| j.observed_count = counter.load(.acquire); + j.entered.store(true, .release); + if (j.gate) |gate| wait(gate) catch @panic("owner-call gate timed out"); + if (j.cancel) |cancel| j.saw_cancel = cancel.load(.acquire); + if (j.reenter) j.host.callOwner(j, exec) catch |err| { + j.nested_failure = err; + }; + j.ran = true; + } + fn run(j: *Job) void { + j.host.callOwner(j, exec) catch |err| { + j.failure = err; + }; + j.returned.store(true, .release); + } +}; + +fn pending(h: *LaneHost, want: usize) !void { + for (0..5000) |_| { + h.lock(); + var count: usize = 0; + var next = h.calls.first; + while (next) |c| : (next = c.next) count += 1; + h.unlock(); + if (count >= want) return; + std.Io.sleep(io, .fromMilliseconds(1), .awake) catch {}; + } + return error.TestTimeout; +} + +test "owner call refuses an unstarted host and owner reentrancy" { + var rig: Rig = .{}; + try rig.init(); + defer rig.deinit(); + var job = Job{ .host = &rig.host, .reenter = true }; + try std.testing.expectError(error.NotStarted, rig.host.callOwner(&job, Job.exec)); + try rig.host.start(); + try rig.host.callOwner(&job, Job.exec); + try std.testing.expectEqual(error.ReentrantOwnerCall, job.nested_failure.?); + try std.testing.expect(job.owner.? != std.Thread.getCurrentId()); + rig.stop(); + try std.testing.expectError(error.Closed, rig.host.callOwner(&job, Job.exec)); +} + +test "owner calls serialize and cancellation never releases a caller while its callback uses the context" { + var rig: Rig = .{}; + try rig.init(); + defer rig.deinit(); + try rig.host.start(); + var gate: Flag = .init(false); + var cancel: Flag = .init(false); + var threads: Threads = .{}; + defer { + gate.store(true, .release); + threads.join(); + } + var first = Job{ .host = &rig.host, .gate = &gate, .cancel = &cancel }; + var second = Job{ .host = &rig.host }; + try threads.add(Job.run, .{&first}); + try wait(&first.entered); + try threads.add(Job.run, .{&second}); + try pending(&rig.host, 1); + cancel.store(true, .release); + try std.testing.expect(!first.returned.load(.acquire) and !second.entered.load(.acquire)); + gate.store(true, .release); + threads.join(); + try std.testing.expect(first.ran and first.saw_cancel and second.ran); + try std.testing.expect(first.failure == null and second.failure == null); + try std.testing.expectEqual(first.owner, second.owner); +} + +test "owner stop rejects pending calls while active callback finishes before shutdown returns" { + var rig: Rig = .{}; + try rig.init(); + defer rig.deinit(); + try rig.host.start(); + var gate: Flag = .init(false); + var threads: Threads = .{}; + defer { + gate.store(true, .release); + threads.join(); + } + var first = Job{ .host = &rig.host, .gate = &gate }; + var second = Job{ .host = &rig.host }; + try threads.add(Job.run, .{&first}); + try wait(&first.entered); + try threads.add(Job.run, .{&second}); + try pending(&rig.host, 1); + try threads.add(Rig.stop, .{&rig}); + try wait(&second.returned); + try std.testing.expectEqual(error.Closed, second.failure.?); + try std.testing.expect(!second.ran and !first.returned.load(.acquire)); + gate.store(true, .release); + threads.join(); + try std.testing.expect(first.ran and first.failure == null and rig.stopped); +} + +test "engine drain rejects pending calls before joining an active owner callback" { + var rig: Rig = .{}; + try rig.init(); + defer rig.deinit(); + try rig.host.start(); + var gate: Flag = .init(false); + var threads: Threads = .{}; + defer { + gate.store(true, .release); + threads.join(); + } + var first = Job{ .host = &rig.host, .gate = &gate }; + var second = Job{ .host = &rig.host }; + try threads.add(Job.run, .{&first}); + try wait(&first.entered); + try threads.add(Job.run, .{&second}); + try pending(&rig.host, 1); + try threads.add(api.Engine.drain, .{rig.host.engine()}); + try wait(&second.returned); + try std.testing.expectEqual(error.Closed, second.failure.?); + try std.testing.expect(!second.ran and !first.returned.load(.acquire)); + gate.store(true, .release); + threads.join(); + try std.testing.expect(first.ran and first.failure == null); + try std.testing.expect(rig.host.thread == null); +} + +test "owner enqueue racing stop completes every accepted or refused caller" { + var rig: Rig = .{}; + try rig.init(); + defer rig.deinit(); + try rig.host.start(); + var threads: Threads = .{}; + defer threads.join(); + var jobs: [24]Job = undefined; + for (&jobs, 0..) |*job, i| { + job.* = .{ .host = &rig.host }; + try threads.add(Job.run, .{job}); + if (i == 12) try threads.add(Rig.stop, .{&rig}); + } + threads.join(); + for (jobs) |job| { + try std.testing.expect(job.returned.load(.acquire)); + if (job.failure) |err| try std.testing.expect(err == error.Closed) else try std.testing.expect(job.ran); + } +} + +const Box = struct { + done: Flag = .init(false), + tokens: std.atomic.Value(usize) = .init(0), + fn event(ptr: *anyopaque, _: api.Id, e: *const api.Event) void { + const b: *Box = @ptrCast(@alignCast(ptr)); + if (e.* == .finished) b.done.store(true, .release); + if (e.* == .tokens) _ = b.tokens.fetchAdd(e.tokens.len, .release); + } +}; + +test "owner queue lets a chat round run between two waiting family calls" { + var rig: Rig = .{}; + try rig.init(); + defer rig.deinit(); + try rig.host.start(); + var gate: Flag = .init(false); + var threads: Threads = .{}; + defer { + gate.store(true, .release); + threads.join(); + } + var box: Box = .{}; + var first = Job{ .host = &rig.host, .gate = &gate, .counter = &box.tokens }; + var second = Job{ .host = &rig.host, .counter = &box.tokens }; + try threads.add(Job.run, .{&first}); + try wait(&first.entered); + const request = api.Request{ .prompt = &.{ 2, 7, 1, 8 }, .max_tokens = 64 }; + try rig.host.engine().submit(1, &request, .{ .ctx = &box, .event = Box.event }); + try threads.add(Job.run, .{&second}); + try pending(&rig.host, 1); + gate.store(true, .release); + threads.join(); + try std.testing.expect(second.observed_count > first.observed_count and second.observed_count >= 2); + try wait(&box.done); +} + +test "owner call makes a running lone driver yield before its long reply ends" { + const Lone = struct { + backend: lanes.backend.Backend, + entered: Flag = .init(false), + yielded: Flag = .init(false), + fn run(ptr: *anyopaque, s: *lanes.Stream, hooks: api.LoneHooks) !bool { + const l: *@This() = @ptrCast(@alignCast(ptr)); + _ = try l.backend.opening(a, s); + hooks.committed(hooks.ctx); + s.pending = s.context.items[s.context.items.len - 1]; + s.cache_len = s.prompt_len; + l.entered.store(true, .release); + for (0..5000) |_| { + if (hooks.yield(hooks.ctx)) { + l.yielded.store(true, .release); + return true; + } + std.Io.sleep(io, .fromMilliseconds(1), .awake) catch {}; + } + return error.TestTimeout; + } + }; + var rig: Rig = .{}; + try rig.init(); + defer rig.deinit(); + var lone = Lone{ .backend = rig.target.backend() }; + rig.host.lone = .{ .ctx = &lone, .run = Lone.run }; + try rig.host.start(); + var box: Box = .{}; + const request = api.Request{ .prompt = &.{ 2, 7, 1, 8 }, .max_tokens = 64 }; + try rig.host.engine().submit(1, &request, .{ .ctx = &box, .event = Box.event }); + try wait(&lone.entered); + var job = Job{ .host = &rig.host, .counter = &box.tokens }; + try rig.host.callOwner(&job, Job.exec); + try std.testing.expect(lone.yielded.load(.acquire)); + try std.testing.expect(job.observed_count > 0 and job.observed_count < request.max_tokens); + try wait(&box.done); +} + +test "owner call waits for an open lesson without preventing that lesson from completing" { + const Learner = struct { + gate: Flag = .init(false), + entered: Flag = .init(false), + steps: std.atomic.Value(usize) = .init(0), + fn begin(_: *anyopaque, _: *const api.LearnRequest, _: api.LearnSink) !void { + return error.UnexpectedBegin; + } + fn step(ptr: *anyopaque) api.Learner.Step { + const l: *@This() = @ptrCast(@alignCast(ptr)); + if (l.steps.fetchAdd(1, .acq_rel) == 0) { + l.entered.store(true, .release); + wait(&l.gate) catch @panic("lesson gate timed out"); + return .{ .done = false, .changed = false }; + } + return .{ .done = true, .changed = false }; + } + fn abort(_: *anyopaque) void {} + }; + var rig: Rig = .{}; + try rig.init(); + defer rig.deinit(); + var learner: Learner = .{}; + rig.host.learner = .{ .ctx = &learner, .begin = Learner.begin, .step = Learner.step, .abort = Learner.abort }; + rig.host.lesson_open = true; + try rig.host.start(); + var threads: Threads = .{}; + defer { + learner.gate.store(true, .release); + if (!rig.stopped) rig.stop(); + threads.join(); + } + try wait(&learner.entered); + var job = Job{ .host = &rig.host, .counter = &learner.steps }; + try threads.add(Job.run, .{&job}); + try pending(&rig.host, 1); + learner.gate.store(true, .release); + try wait(&job.returned); + threads.join(); + try std.testing.expect(job.failure == null and job.observed_count >= 2); +} + +test "owner calls queued before a failed weight put-back are refused until the model heals" { + const Learner = struct { + gate: Flag = .init(false), + entered: Flag = .init(false), + healing: bool = false, + fn begin(ptr: *anyopaque, _: *const api.LearnRequest, _: api.LearnSink) !void { + const l: *@This() = @ptrCast(@alignCast(ptr)); + l.healing = true; + } + fn step(ptr: *anyopaque) api.Learner.Step { + const l: *@This() = @ptrCast(@alignCast(ptr)); + if (l.healing) return .{ .done = true, .changed = false, .stale = false }; + l.entered.store(true, .release); + wait(&l.gate) catch @panic("stale lesson gate timed out"); + return .{ .done = true, .changed = false, .stale = true }; + } + fn abort(_: *anyopaque) void {} + fn event(_: *anyopaque, _: *const api.LearnEvent) void {} + }; + var rig: Rig = .{}; + try rig.init(); + defer rig.deinit(); + var learner: Learner = .{}; + rig.host.learner = .{ .ctx = &learner, .begin = Learner.begin, .step = Learner.step, .abort = Learner.abort }; + rig.host.lesson_open = true; + try rig.host.start(); + var threads: Threads = .{}; + defer { + learner.gate.store(true, .release); + if (!rig.stopped) rig.stop(); + threads.join(); + } + try wait(&learner.entered); + var jobs = [_]Job{ .{ .host = &rig.host }, .{ .host = &rig.host } }; + for (&jobs) |*job| try threads.add(Job.run, .{job}); + try pending(&rig.host, 2); + learner.gate.store(true, .release); + for (&jobs) |*job| try wait(&job.returned); + threads.join(); + for (jobs) |job| { + try std.testing.expectEqual(error.StaleModel, job.failure.?); + try std.testing.expect(!job.ran); + } + var healed = Job{ .host = &rig.host }; + try std.testing.expectError(error.StaleModel, rig.host.callOwner(&healed, Job.exec)); + const heal: api.LearnRequest = .{}; + try rig.host.engine().learn(&heal, .{ .ctx = &learner, .event = Learner.event }); + var still_stale = true; + for (0..5000) |_| { + rig.host.lock(); + still_stale = rig.host.stale; + rig.host.unlock(); + if (!still_stale) break; + std.Io.sleep(io, .fromMilliseconds(1), .awake) catch {}; + } + try std.testing.expect(!still_stale); + try rig.host.callOwner(&healed, Job.exec); + try std.testing.expect(healed.ran); +} diff --git a/zig/src/core/lane_host_learn.zig b/zig/src/core/lane_host_learn.zig index dde774bfbd..b9b7224d50 100644 --- a/zig/src/core/lane_host_learn.zig +++ b/zig/src/core/lane_host_learn.zig @@ -28,7 +28,7 @@ pub fn refuse(h: *LaneHost) void { /// Learning runs only when nothing else waits: no stream, queued request or cancel (called under the lock). pub fn ready(h: *const LaneHost) bool { const work = h.lesson_open or h.lessons.items.len > 0; - return h.learner != null and work and h.queued.items.len == 0 and h.admitted.items.len == 0 and h.cancels.items.len == 0; + return h.learner != null and work and (h.lesson_open or h.calls.first == null) and h.queued.items.len == 0 and h.admitted.items.len == 0 and h.cancels.items.len == 0; } /// One unit of learning: open the next lesson if none is open, then step it; moved weights drop every kept prompt. diff --git a/zig/src/core/lane_host_rows.zig b/zig/src/core/lane_host_rows.zig index a45b9cfd2f..360a8e4f71 100644 --- a/zig/src/core/lane_host_rows.zig +++ b/zig/src/core/lane_host_rows.zig @@ -3,6 +3,7 @@ const std = @import("std"); const api = @import("engine_api.zig"); const LaneHost = @import("lane_host.zig").LaneHost; const rows = @import("prompt_cache.zig").rows; +const calls = @import("lane_host_calls.zig"); /// A planned stop's time to finish writing what waits. pub const stop_ns: i128 = 20 * std.time.ns_per_s; @@ -17,8 +18,10 @@ pub fn drain(ctx: *anyopaque) void { pub fn drainThread(h: *LaneHost) void { h.lock(); h.closing = true; + const pending = calls.detach(&h.calls); h.wake.broadcast(h.io); h.unlock(); + calls.reject(pending, error.Closed); if (h.thread) |t| t.join(); h.thread = null; } @@ -52,7 +55,7 @@ pub fn ready(h: *LaneHost) bool { const store = h.cache orelse return false; const r = store.disk orelse return false; if (!rows.write.pending(r)) return false; - if (h.queued.items.len > 0 or h.admitted.items.len > 0 or h.cancels.items.len > 0 or h.closing) return false; + if (h.calls.first != null or h.queued.items.len > 0 or h.admitted.items.len > 0 or h.cancels.items.len > 0 or h.closing) return false; return h.disk.quiet(h.admissions, std.Io.Clock.awake.now(h.io).toNanoseconds()); } diff --git a/zig/src/core/lane_host_test.zig b/zig/src/core/lane_host_test.zig index 85a34c0e15..2a6836a6fe 100644 --- a/zig/src/core/lane_host_test.zig +++ b/zig/src/core/lane_host_test.zig @@ -570,4 +570,5 @@ test { _ = @import("lane_host_borrow_test.zig"); _ = @import("lane_host_warm_test.zig"); _ = @import("lane_host_rows_test.zig"); + _ = @import("lane_host_calls_test.zig"); } diff --git a/zig/src/core/lanes/engine.zig b/zig/src/core/lanes/engine.zig index f11d8dbec5..105ac3ebe1 100644 --- a/zig/src/core/lanes/engine.zig +++ b/zig/src/core/lanes/engine.zig @@ -490,7 +490,8 @@ pub const Engine = struct { /// One token, the next forward queued first; a copied continuation ahead switches to verify windows. fn pipelinedRound(e: *Engine, s: *Stream) !Result { const before = s.mode; - const mode = s.mode orelse .pipe; + // an adopted stream holds neither a mode nor a queued step: it starts where a drain step leaves one + const mode: sm.Mode = s.mode orelse if (s.inflight == null) .verify else .pipe; var result: Result = .{ .rows = 1, .keep = 1, .got = &.{} }; if (mode == .verify or mode == .exit) { const verifying = mode == .verify and e.cfg.family_width >= 2; diff --git a/zig/src/core/lanes/engine_test.zig b/zig/src/core/lanes/engine_test.zig index c17758753b..5b7d468de2 100644 --- a/zig/src/core/lanes/engine_test.zig +++ b/zig/src/core/lanes/engine_test.zig @@ -88,9 +88,14 @@ const Classes = struct { }; fn model() !Config { + return modelWith(true); +} + +/// The test family drafting with its head (`mtp`) or by copies alone, when drafted streams step pipelined. +fn modelWith(mtp: bool) !Config { var costs: [16]@import("config.zig").Cost = undefined; for (&costs, 1..) |*c, w| c.* = .{ .width = @intCast(w), .ms = 5.0 + 0.8 * @as(f64, @floatFromInt(w)) }; - return Config.init(gpa, .{ .exact_width = 16, .gpu_tokens = true, .mtp = true, .speculate = true, .speculate_early = false, .drafts = 4, .window_costs = &costs, .mtp_step_ms = 0.5, .hidden_rows = true, .batch_rows = 32, .max_streams = 8, .draft_streams = true }, 16, 15); + return Config.init(gpa, .{ .exact_width = 16, .gpu_tokens = true, .mtp = mtp, .speculate = true, .speculate_early = false, .drafts = 4, .window_costs = &costs, .mtp_step_ms = 0.5, .hidden_rows = true, .batch_rows = 32, .max_streams = 8, .draft_streams = true }, 16, 15); } /// Every case's emitted tokens, the cases admitted together and stepped until done. @@ -440,6 +445,108 @@ test "structured output: where the grammar spells its own text, its drafts fill } } +/// The fake target's one-token decode after `c.prompt`, to `c.max_new` tokens or an eos one: `s`'s reply and reason. +fn expectDecode(c: Case, s: *const sm.Stream) !void { + var history: std.ArrayList(u32) = .empty; + defer history.deinit(gpa); + try history.appendSlice(gpa, c.prompt); + var reason: sm.Reason = .length; + while (history.items.len - c.prompt.len < c.max_new) { + const t = fake.next(history.items, c.sampling, history.items.len); + try history.append(gpa, t); + if (std.mem.indexOfScalar(u32, c.eos, t) != null) { + reason = .stop; + break; + } + } + try std.testing.expectEqualSlices(u32, history.items[c.prompt.len..], s.emitted()); + try std.testing.expectEqual(reason, s.reason); +} + +test "a lone stream handed to another driver between rounds and back replies as if kept, pipelined or windowed" { + // a plain stream left in verify by a shared round, a windowed drafted one, a copy-drafted one left there by a drain + const Set = struct { cases: []const Case, mtp: bool }; + const sets = [_]Set{ + .{ .cases = &.{ .{ .prompt = &p1, .drafts = false }, .{ .prompt = &p2, .max_new = 6 } }, .mtp = true }, + .{ .cases = &.{.{ .prompt = &p2 }}, .mtp = true }, + .{ .cases = &.{.{ .prompt = &p2, .max_new = 60 }}, .mtp = false }, + }; + for (sets) |set| { + var cfg = try modelWith(set.mtp); + defer cfg.deinit(gpa); + var target: fake.Fake = .{ .gpa = gpa }; + defer target.deinit(); + var clock: fake.FixedClock = .{}; + var engine = Engine.init(gpa, &cfg, target.backend(), clock.clock()); + defer engine.deinit(); + var proposers: [2]SuffixLookup = undefined; + var streams: [2]sm.Stream = undefined; + for (set.cases, proposers[0..set.cases.len], streams[0..set.cases.len]) |c, *p, *s| { + p.* = try SuffixLookup.init(gpa, .{ .min_match = 4 }); + s.* = try sm.Stream.init(gpa, .{ + .id = "s", + .prompt = c.prompt, + .max_new = c.max_new, + .eos = c.eos, + .drafts = c.drafts, + .proposer = p.proposer(), + }); + } + defer for (proposers[0..set.cases.len], streams[0..set.cases.len]) |*p, *s| { + s.deinit(gpa); + p.deinit(); + }; + for (streams[0..set.cases.len]) |*s| try engine.addStream(s); + var handed: usize = 0; + var settle = false; // a round runs on what the adopted round leaves before the next hand-back + while (engine.activeCount() > 0) { + try engine.step(); + if (engine.live.items.len != 1) continue; + if (settle) { + settle = false; + continue; + } + const s = engine.live.items[0]; + if (!engine.detach(s)) continue; + handed += 1; + try engine.adopt(s); + settle = true; + } + try std.testing.expect(handed > 0); + for (set.cases, streams[0..set.cases.len]) |c, *s| try expectDecode(c, s); + } +} + +test "a stream another driver opened and hands to the lane core replies as its one-token decode" { + const c: Case = .{ .prompt = &p2 }; + for ([_]bool{ true, false }) |mtp| { + var cfg = try modelWith(mtp); + defer cfg.deinit(gpa); + var target: fake.Fake = .{ .gpa = gpa }; + defer target.deinit(); + var clock: fake.FixedClock = .{}; + var engine = Engine.init(gpa, &cfg, target.backend(), clock.clock()); + defer engine.deinit(); + var p = try SuffixLookup.init(gpa, .{ .min_match = 4 }); + defer p.deinit(); + var s = try sm.Stream.init(gpa, .{ + .id = "s", + .prompt = c.prompt, + .max_new = c.max_new, + .eos = c.eos, + .proposer = p.proposer(), + }); + defer s.deinit(gpa); + // as a lone driver hands a stream over after its first token (lane_host_lone_test.zig's Lone) + _ = try target.backend().opening(gpa, &s); + s.pending = s.context.items[s.context.items.len - 1]; + s.cache_len = s.prompt_len; + try engine.adopt(&s); + while (engine.activeCount() > 0) try engine.step(); + try expectDecode(c, &s); + } +} + test "a family that keeps copies to shared rounds proposes none for a lone stream" { for ([_]bool{ true, false }) |lone_copies| { var cfg = try Config.init(gpa, .{ .exact_width = 16, .mtp = true, .speculate = true, .drafts = 4, .lone_copies = lone_copies }, 16, 15); diff --git a/zig/src/families/clef/README.md b/zig/src/families/clef/README.md new file mode 100644 index 0000000000..a7ff04e510 --- /dev/null +++ b/zig/src/families/clef/README.md @@ -0,0 +1,44 @@ +# Clef first-slice work + +Work for [issue 692](https://github.com/ashhart/TensorFold/issues/692), not ready for an upstream PR or deployment. The bf16 head reader is separate from the qwen27 backbone. The original 45-record, 128-question layout gate is still required; the older 15-record prototype does not replace it. The encoding and head port retain the credit for outcastofmusic's #241 and #251. + +`checkpoint.loadOptional` returns no head only when both head sidecars are absent. A missing model directory, incomplete pair, broken symlink, wrong file type or invalid head remains an error. Valid Hugging Face cache symlinks are accepted. The caller owns a returned head and must call `deinit`. + +The CPU inspector checks the separate head against the hidden width in the backbone's `config.json`: + +```sh +zig build tf-clef-inspect +zig-out/bin/tf-clef-inspect /path/to/checkpoint --check +python3 tools/zig/test_clef_inspect.py zig-out/bin/tf-clef-inspect +``` + +Its JSON includes the head's tensor count, geometry and file SHA256, plus the backbone config SHA256. A plain pack reports no head. A failed check exits with an error and emits no success JSON. The inspector opens no backbone weights or GPU; `backbone_weights_checked`, `gpu` and `model_ready` remain false. Head detection is not connected to the serving path yet. + +`qwen27.hidden_rows.captureExclusive` returns owned bf16 final normalized rows in token order. It runs the existing prompt kernels with a fresh frame and fresh GDN/KV runner. It copies `frame.input` after each completed `advance(.none)` and restores the caller's frame and previous prompt flag on return. The caller must own the model for the entire call. + +Capture currently accepts text token IDs. It has no image input. Isolation after a real vision-chat prefill requires its own device test. + +`qwen27_served.Host.hiddenRows` queues this operation on the engine thread between chat rounds. Each caller keeps its own completion storage alive until the operation ends. Pending calls make lone and background drivers yield. An open learning transaction finishes before capture starts. Both drain and stop reject queued calls and wait for an active call to finish safely. A failed learning put-back refuses calls with `StaleModel` until the model heals. A complete capture can pause chat for its duration; the queue does not promise a short pause. HTTP decision scoring is not connected yet. + +Before allocating capture buffers, the owner thread checks output rows, private KV state and a conservative fixed-buffer bound against the server's remaining device working set, preserving its 8 GiB reserve. Insufficient room returns `HiddenRowsMemory` without evicting chat snapshots. The current host's larger buffers bound the private single-slot pass; this can refuse a capture that a tighter estimate would admit. The bound covers buffers, not measured peak process memory or compiler/pipeline overhead. + +The ANE handoff reserves storage before input encoding, then registers and publishes each committed command buffer without a fallible allocation. The worker can release an even wait after a terminal input-CB failure even when the odd signal never arrived. Drain waits for every registered CB and the last worker completion before releasing retained handles. A poisoned share stays unavailable until explicitly rebuilt. Restoring the chat frame after such a device failure does not guarantee that the server can continue serving. + +CPU validation uses Zig 0.17.0: + +```sh +zig build test-clef-checkpoint +zig build test +zig build test-golden fuzz-cortheon +zig fmt --check build.zig zig +``` + +The targeted check covers head admission, optional-head detection, hidden-row bounds and the actual ANE worker/drain protocol. Its two-thread ring test publishes 1,024 tickets through a ring of four entries. Capture-budget cases cover the exact boundary, insufficient room and overflowing totals. The CLI has four additional CPU tests. Whole-tree validation must be repeated after each upstream rebase. + +Nine additional owner-call cases run with the host tests: owner identity/reentrancy, serialization, cancellation lifetime, stop and enqueue races, chat progress between calls, lone-driver yield, completion of an open lesson, direct engine drain and stale-weight refusal. Mutants that remove lone yield, block an open lesson behind a call, or serve two calls before chat each fail their corresponding regression test. + +`zig build tf-qwen27-hidden-rows-check` builds the synthetic device harness without executing it. The harness compares capture against normal prompt arithmetic with the LM head projected; this proves capture parity, not independent Clef numerical accuracy. It also checks output ownership, chat/snapshot continuation, cancellation, allocation failure and nonfinite final rows. Late cancel/OOM triggers must observe the CPU output of an already completed first chunk. Its optional `--ane` argument exercises the ANE path. + +`zig build tf-qwen27-hidden-rows-served-check` builds a second device harness. It queues three captures while a synthetic chat is open, compares their rows with a solo capture and compares all 64 chat tokens with CLI generation. Its optional `--ane` argument keeps the same capture path with the Neural Engine enabled. + +Both device harnesses have only been compiled. Run them only with exclusive use of the Metal engine. Pending gates include ANE on/off runs, device-fault behavior, performance against the pinned base, queued concurrent-request isolation on a device, real-model chat/resume parity, the original layout corpus and an agreed numerical head tolerance. No probability tolerance has been selected. diff --git a/zig/src/families/clef/checkpoint.zig b/zig/src/families/clef/checkpoint.zig new file mode 100644 index 0000000000..16167160b8 --- /dev/null +++ b/zig/src/families/clef/checkpoint.zig @@ -0,0 +1,28 @@ +//! A joint head is optional only when both sidecars are absent; damaged decision packs never become chat packs. +const std = @import("std"); +const Weights = @import("weights.zig").Weights; + +fn present(dir: std.Io.Dir, io: std.Io, name: []const u8) !bool { + const entry = dir.statFile(io, name, .{ .follow_symlinks = false }) catch |err| switch (err) { + error.FileNotFound => return false, + else => return err, + }; + const file = if (entry.kind == .sym_link) try dir.statFile(io, name, .{}) else entry; + if (file.kind != .file) return error.InvalidHeadSidecar; + return true; +} + +/// Plain chat packs return null; a complete decision pack owns a fully validated separate bf16 head. +pub fn loadOptional(a: std.mem.Allocator, io: std.Io, path: []const u8, backbone_hidden: usize) !?Weights { + var dir = try std.Io.Dir.cwd().openDir(io, path, .{}); + defer dir.close(io); + const config = try present(dir, io, "joint_head_config.json"); + const weights = try present(dir, io, "joint_head.safetensors"); + if (!config and !weights) return null; + if (!config or !weights) return error.IncompleteJointHead; + return try Weights.load(a, io, path, backbone_hidden); +} + +test { + _ = @import("checkpoint_test.zig"); +} diff --git a/zig/src/families/clef/checkpoint_test.zig b/zig/src/families/clef/checkpoint_test.zig new file mode 100644 index 0000000000..fe2c3501d8 --- /dev/null +++ b/zig/src/families/clef/checkpoint_test.zig @@ -0,0 +1,89 @@ +//! Presence, corruption and cache-symlink checks keep decision checkpoints from silently taking the chat path. +const std = @import("std"); +const checkpoint = @import("checkpoint.zig"); +const a = std.testing.allocator; +const io = std.testing.io; +const config = @embedFile("../../../tests/fixtures/clef/joint_head_config.json"); +const weights = @embedFile("../../../tests/fixtures/clef/joint_head.safetensors"); + +fn loadClose(gpa: std.mem.Allocator, path: []const u8) !void { + var head = (try checkpoint.loadOptional(gpa, io, path, 16)) orelse return error.ExpectedHead; + defer head.deinit(io); +} + +test "a missing model directory is not a plain checkpoint" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const path = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(path); + const missing = try std.fs.path.join(a, &.{ path, "missing-model" }); + defer a.free(missing); + try std.testing.expectError(error.FileNotFound, checkpoint.loadOptional(a, io, missing, 16)); +} + +test "a plain pack has neither head sidecar, not an incomplete pair" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const path = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(path); + try std.testing.expect((try checkpoint.loadOptional(a, io, path, 16)) == null); + try tmp.dir.writeFile(io, .{ .sub_path = "joint_head_config.json", .data = config }); + try std.testing.expectError(error.IncompleteJointHead, checkpoint.loadOptional(a, io, path, 16)); + try tmp.dir.deleteFile(io, "joint_head_config.json"); + try tmp.dir.writeFile(io, .{ .sub_path = "joint_head.safetensors", .data = weights }); + try std.testing.expectError(error.IncompleteJointHead, checkpoint.loadOptional(a, io, path, 16)); +} + +test "a complete pair uses the same geometry and finite tensor checks as the strict reader" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const path = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(path); + try tmp.dir.writeFile(io, .{ .sub_path = "joint_head_config.json", .data = config }); + try tmp.dir.writeFile(io, .{ .sub_path = "joint_head.safetensors", .data = weights }); + try loadClose(a, path); + try std.testing.expectError(error.IncompatibleHeadBackbone, checkpoint.loadOptional(a, io, path, 5120)); + try tmp.dir.writeFile(io, .{ .sub_path = "joint_head.safetensors", .data = weights[0 .. weights.len - 2] }); + if (checkpoint.loadOptional(a, io, path, 16)) |head_| { + if (head_) |head__| { + var head = head__; + head.deinit(io); + } + return error.CorruptPackAccepted; + } else |_| {} +} + +test "head sidecar directories fail, while valid cache symlinks remain supported" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const path = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(path); + try tmp.dir.createDir(io, "joint_head_config.json", .default_dir); + try std.testing.expectError(error.InvalidHeadSidecar, checkpoint.loadOptional(a, io, path, 16)); + try tmp.dir.deleteDir(io, "joint_head_config.json"); + try tmp.dir.writeFile(io, .{ .sub_path = "config-blob", .data = config }); + try tmp.dir.writeFile(io, .{ .sub_path = "weights-blob", .data = weights }); + try tmp.dir.symLink(io, "config-blob", "joint_head_config.json", .{}); + try tmp.dir.symLink(io, "weights-blob", "joint_head.safetensors", .{}); + try loadClose(a, path); +} + +test "a dangling head symlink is a broken pack even when neither target exists" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const path = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(path); + try tmp.dir.symLink(io, "missing-config", "joint_head_config.json", .{}); + try tmp.dir.symLink(io, "missing-weights", "joint_head.safetensors", .{}); + try std.testing.expectError(error.FileNotFound, checkpoint.loadOptional(a, io, path, 16)); +} + +test "optional head loading preserves allocation failures and releases its map" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const path = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(path); + try tmp.dir.writeFile(io, .{ .sub_path = "joint_head_config.json", .data = config }); + try tmp.dir.writeFile(io, .{ .sub_path = "joint_head.safetensors", .data = weights }); + try std.testing.checkAllAllocationFailures(a, loadClose, .{path}); +} diff --git a/zig/src/families/clef/weights.zig b/zig/src/families/clef/weights.zig new file mode 100644 index 0000000000..fa265d0e3c --- /dev/null +++ b/zig/src/families/clef/weights.zig @@ -0,0 +1,114 @@ +//! The joint head owns a read-only bf16 safetensors map and validates every trained tensor. +const std = @import("std"); +const safetensors = @import("../../core/safetensors.zig"); +const convert = @import("../../core/quant/convert.zig"); +const Tensor = safetensors.Tensor; +const A = std.mem.Allocator; + +pub const Config = struct { + hidden_size: usize, + width: usize, + routing_layers: usize, + layers: usize, + heads: usize, + feedforward: usize, + dropout: f32 = 0, + pub fn validate(c: Config) !void { + if (c.hidden_size == 0 or c.hidden_size > 16384 or c.width == 0 or c.width > 4096 or c.heads == 0 or c.width % c.heads != 0 or c.feedforward == 0 or c.feedforward > 16384 or c.routing_layers > 16 or c.layers > 16 or c.dropout != 0) return error.InvalidHeadConfig; + } +}; + +pub const Weights = struct { + config: Config, + file: safetensors.File, + pub fn load(a: A, io: std.Io, dir: []const u8, backbone_hidden: usize) !Weights { + const config_path = try std.fs.path.join(a, &.{ dir, "joint_head_config.json" }); + defer a.free(config_path); + const text = try std.Io.Dir.cwd().readFileAlloc(io, config_path, a, .limited(1 << 20)); + defer a.free(text); + const parsed = try std.json.parseFromSlice(Config, a, text, .{}); + defer parsed.deinit(); + try parsed.value.validate(); + if (parsed.value.hidden_size != backbone_hidden) return error.IncompatibleHeadBackbone; + const weight_path = try std.fs.path.join(a, &.{ dir, "joint_head.safetensors" }); + defer a.free(weight_path); + var out = Weights{ .config = parsed.value, .file = try safetensors.File.open(a, io, weight_path) }; + errdefer out.file.close(io); + try out.validate(); + return out; + } + pub fn deinit(w: *Weights, io: std.Io) void { + w.file.close(io); + } + pub fn get(w: *const Weights, name: []const u8) Tensor { + return w.file.get(name).?; + } + pub fn named(w: *const Weights, prefix: []const u8, suffix: []const u8) Tensor { + var buf: [192]u8 = undefined; + return w.get(std.fmt.bufPrint(&buf, "{s}.{s}", .{ prefix, suffix }) catch unreachable); + } + fn tensor(w: *const Weights, prefix: []const u8, suffix: []const u8, shape: []const usize) !void { + var buf: [192]u8 = undefined; + const t = w.file.get(try std.fmt.bufPrint(&buf, "{s}{s}{s}", .{ prefix, if (suffix.len == 0) "" else ".", suffix })) orelse return error.MissingHeadWeight; + if (!t.is(.bf16, shape)) return error.InvalidHeadWeight; + for (0..t.numel()) |i| if (!std.math.isFinite(convert.load(t.dtype, t.bytes, i))) return error.NonfiniteHeadWeight; + } + fn norm(w: *const Weights, prefix: []const u8, n: usize) !void { + try w.tensor(prefix, "weight", &.{n}); + try w.tensor(prefix, "bias", &.{n}); + } + fn linear(w: *const Weights, prefix: []const u8, n: usize, k: usize, bias: bool) !void { + try w.tensor(prefix, "weight", &.{ n, k }); + if (bias) try w.tensor(prefix, "bias", &.{n}); + } + fn attention(w: *const Weights, prefix: []const u8) !void { + const d = w.config.width; + try w.tensor(prefix, "in_proj_weight", &.{ 3 * d, d }); + try w.tensor(prefix, "in_proj_bias", &.{3 * d}); + var buf: [192]u8 = undefined; + try w.linear(try std.fmt.bufPrint(&buf, "{s}.out_proj", .{prefix}), d, d, true); + } + pub fn validate(w: *const Weights) !void { + const c = w.config; + try c.validate(); + try w.norm("hidden_norm", c.hidden_size); + for ([_][]const u8{ "memory_projection", "question_projection", "option_question_projection", "global_projection", "option_context_projection", "option_lexical_projection" }) |name| try w.linear(name, c.width, c.hidden_size, false); + try w.tensor("type_embedding", "weight", &.{ 3, c.width }); + var buf: [192]u8 = undefined; + for (0..c.routing_layers) |i| { + for ([_][]const u8{ "query_norm", "memory_norm", "feedforward_norm" }) |name| try w.norm(try std.fmt.bufPrint(&buf, "evidence_layers.{d}.{s}", .{ i, name }), c.width); + try w.attention(try std.fmt.bufPrint(&buf, "evidence_layers.{d}.attention", .{i})); + try w.linear(try std.fmt.bufPrint(&buf, "evidence_layers.{d}.feedforward.0", .{i}), c.feedforward, c.width, true); + try w.linear(try std.fmt.bufPrint(&buf, "evidence_layers.{d}.feedforward.3", .{i}), c.width, c.feedforward, true); + } + try w.norm("option_summary_norm", c.width); + for (0..c.layers) |i| { + for ([_][]const u8{ "self_attn", "multihead_attn" }) |name| try w.attention(try std.fmt.bufPrint(&buf, "layers.{d}.{s}", .{ i, name })); + for ([_][]const u8{ "norm1", "norm2", "norm3" }) |name| try w.norm(try std.fmt.bufPrint(&buf, "layers.{d}.{s}", .{ i, name }), c.width); + try w.linear(try std.fmt.bufPrint(&buf, "layers.{d}.linear1", .{i}), c.feedforward, c.width, true); + try w.linear(try std.fmt.bufPrint(&buf, "layers.{d}.linear2", .{i}), c.width, c.feedforward, true); + } + try w.norm("field_norm", c.width); + try w.norm("option_norm", c.width); + try w.linear("residual_scorer.0", c.width, 4 * c.width, true); + try w.linear("residual_scorer.3", 1, c.width, true); + for ([_][]const u8{ "prior_logit_scale", "joint_logit_scale", "residual_gate" }) |name| try w.tensor(name, "", &.{}); + const expected = 22 + c.routing_layers * 14 + c.layers * 18; + if (w.file.names.count() != expected) return error.UnexpectedHeadWeight; + } +}; + +test "head configuration refuses invalid attention geometry and training dropout" { + const c = Config{ .hidden_size = 16, .width = 8, .routing_layers = 2, .layers = 4, .heads = 2, .feedforward = 16 }; + try c.validate(); + var bad = c; + bad.heads = 3; + try std.testing.expectError(error.InvalidHeadConfig, bad.validate()); + bad = c; + bad.dropout = 0.1; + try std.testing.expectError(error.InvalidHeadConfig, bad.validate()); +} + +test { + _ = @import("weights_test.zig"); +} diff --git a/zig/src/families/clef/weights_test.zig b/zig/src/families/clef/weights_test.zig new file mode 100644 index 0000000000..276608c20e --- /dev/null +++ b/zig/src/families/clef/weights_test.zig @@ -0,0 +1,127 @@ +//! A small seeded release head tests the complete tensor graph and every load refusal without a device. +const std = @import("std"); +const Weights = @import("weights.zig").Weights; +const a = std.testing.allocator; +const io = std.testing.io; +const config = @embedFile("../../../tests/fixtures/clef/joint_head_config.json"); +const fixture = @embedFile("../../../tests/fixtures/clef/joint_head.safetensors"); + +fn write(tmp: std.testing.TmpDir, bytes: []const u8) !void { + try tmp.dir.writeFile(io, .{ .sub_path = "joint_head_config.json", .data = config }); + try tmp.dir.writeFile(io, .{ .sub_path = "joint_head.safetensors", .data = bytes }); +} + +fn changed(needle: []const u8, replacement: []const u8) ![]u8 { + const n: usize = @intCast(std.mem.readInt(u64, fixture[0..8], .little)); + try std.testing.expect(std.mem.indexOf(u8, fixture[8..][0..n], needle) != null); + const header = try std.mem.replaceOwned(u8, a, fixture[8..][0..n], needle, replacement); + defer a.free(header); + const padded = std.mem.alignForward(usize, header.len, 8); + const out = try a.alloc(u8, 8 + padded + fixture.len - 8 - n); + std.mem.writeInt(u64, out[0..8], padded, .little); + @memcpy(out[8..][0..header.len], header); + @memset(out[8 + header.len .. 8 + padded], ' '); + @memcpy(out[8 + padded ..], fixture[8 + n ..]); + return out; +} + +test "the separate bf16 head validates every tensor and matches its backbone width" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try write(tmp, fixture); + const dir = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(dir); + var w = try Weights.load(a, io, dir, 16); + defer w.deinit(io); + try std.testing.expectEqual(@as(usize, 122), w.file.names.count()); + try std.testing.expectEqual(@as(usize, 16), w.config.hidden_size); + try std.testing.expectError(error.IncompatibleHeadBackbone, Weights.load(a, io, dir, 5120)); +} + +test "the trained head refuses missing tensors, changed ranks and a changed storage dtype" { + for ([_]struct { from: []const u8, to: []const u8, err: anyerror }{ + .{ .from = "prior_logit_scale", .to = "other_logit_scale", .err = error.MissingHeadWeight }, + .{ .from = "\"shape\":[16]", .to = "\"shape\":[8,2]", .err = error.InvalidHeadWeight }, + .{ .from = "\"BF16\"", .to = "\"F16\"", .err = error.InvalidHeadWeight }, + }) |change| { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const bytes = try changed(change.from, change.to); + defer a.free(bytes); + try write(tmp, bytes); + const dir = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(dir); + try std.testing.expectError(change.err, Weights.load(a, io, dir, 16)); + } +} + +test "a nonfinite trained weight is refused before any head execution" { + for ([_]u16{ 0x7f80, 0xff80, 0x7fc0 }) |value| { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const bytes = try a.dupe(u8, fixture); + defer a.free(bytes); + const n: usize = @intCast(std.mem.readInt(u64, bytes[0..8], .little)); + std.mem.writeInt(u16, bytes[8 + n ..][0..2], value, .little); + try write(tmp, bytes); + const dir = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(dir); + try std.testing.expectError(error.NonfiniteHeadWeight, Weights.load(a, io, dir, 16)); + } +} + +test "an unrelated tensor is refused even when all required trained tensors are present" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const n: usize = @intCast(std.mem.readInt(u64, fixture[0..8], .little)); + const end = fixture.len - 8 - n; + const prefix = try std.fmt.allocPrint(a, "{{\"unused\":{{\"dtype\":\"BF16\",\"shape\":[1],\"data_offsets\":[{d},{d}]}},\"evidence_layers.0.attention.in_proj_bias\"", .{ end, end + 2 }); + defer a.free(prefix); + const header = try changed("{\"evidence_layers.0.attention.in_proj_bias\"", prefix); + defer a.free(header); + const bytes = try a.alloc(u8, header.len + 2); + defer a.free(bytes); + @memcpy(bytes[0..header.len], header); + std.mem.writeInt(u16, bytes[header.len..][0..2], 0x3f80, .little); + try write(tmp, bytes); + const dir = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(dir); + try std.testing.expectError(error.UnexpectedHeadWeight, Weights.load(a, io, dir, 16)); +} + +test "a truncated head payload is refused" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try write(tmp, fixture[0 .. fixture.len - 1]); + const dir = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(dir); + if (Weights.load(a, io, dir, 16)) |raw| { + var w = raw; + defer w.deinit(io); + return error.TruncatedHeadAccepted; + } else |_| {} +} + +test "head backbone compatibility is checked before opening its payload" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.writeFile(io, .{ .sub_path = "joint_head_config.json", .data = config }); + const dir = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(dir); + try std.testing.expectError(error.IncompatibleHeadBackbone, Weights.load(a, io, dir, 5120)); + try std.testing.expectError(error.FileNotFound, Weights.load(a, io, dir, 16)); +} + +fn loadClose(gpa: std.mem.Allocator, dir: []const u8) !void { + var w = try Weights.load(gpa, io, dir, 16); + w.deinit(io); +} + +test "a failed allocation during a head load leaves no live allocation behind" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try write(tmp, fixture); + const dir = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(dir); + try std.testing.checkAllAllocationFailures(a, loadClose, .{dir}); +} diff --git a/zig/src/families/qwen27/ane_handoff.zig b/zig/src/families/qwen27/ane_handoff.zig new file mode 100644 index 0000000000..a5147ede52 --- /dev/null +++ b/zig/src/families/qwen27/ane_handoff.zig @@ -0,0 +1,207 @@ +//! A handoff completes even when its input command buffer fails, so later GPU waits can end. +const std = @import("std"); + +/// One producer and one consumer; completion makes both payload reuse and retained-CB release safe. +pub fn Ring(comptime Payload: type, comptime capacity: usize) type { + return struct { + slots: [capacity]Entry = @splat(.{}), + issued: u64 = 0, + completed: std.atomic.Value(u64) = .init(0), + const Self = @This(); + const Entry = struct { payload: Payload = undefined, published: std.atomic.Value(u64) = .init(0) }; + + pub fn prepare(q: *Self, a: std.mem.Allocator, retained: anytype) !void { + if (q.issued >= std.math.maxInt(u64) / 2) return error.AneHandoffOverflow; + if (q.issued - q.completed.load(.acquire) >= capacity) return error.AneHandoffFull; + try retained.ensureUnusedCapacity(a, 1); + } + + pub fn publish(q: *Self, payload: Payload) void { + const entry = &q.slots[q.issued % capacity]; + entry.payload = payload; + q.issued += 1; + entry.published.store(q.issued, .release); + } + + pub fn get(q: *Self, t: u64) ?*const Payload { + const entry = &q.slots[t % capacity]; + return if (entry.published.load(.acquire) == t + 1) &entry.payload else null; + } + + pub fn complete(q: *Self, t: u64) void { + q.completed.store(t + 1, .release); + } + }; +} + +/// `job` owns its retained input CB until the batch has also observed worker completion. +pub fn execute(job: anytype) void { + while (!job.ready()) { + if (job.inputFailed()) { + job.poison(); + break; + } + job.waitInput(); + } + if (!job.poisoned()) job.evaluate(); + job.complete(); +} + +/// Wait every input CB and the worker before releasing any CB, including on multiple failures. +pub fn drain(batch: anytype) bool { + var failed = false; + for (0..batch.count()) |i| { + batch.waitBuffer(i); + failed = batch.bufferFailed(i) or failed; + } + batch.waitWorker(); + for (0..batch.count()) |i| batch.releaseBuffer(i); + batch.clear(); + return failed; +} + +const FakeJob = struct { + input: enum { pending, ready, failed } = .pending, + fail_after_wait: bool = false, + poison_: bool = false, + ran: bool = false, + completed: bool = false, + waits: usize = 0, + pub fn ready(j: *FakeJob) bool { + return j.input == .ready; + } + pub fn inputFailed(j: *FakeJob) bool { + return j.input == .failed; + } + pub fn poison(j: *FakeJob) void { + j.poison_ = true; + } + pub fn poisoned(j: *FakeJob) bool { + return j.poison_; + } + pub fn waitInput(j: *FakeJob) void { + j.waits += 1; + j.input = if (j.fail_after_wait) .failed else .ready; + } + pub fn evaluate(j: *FakeJob) void { + std.debug.assert(j.input == .ready and !j.poison_); + j.ran = true; + } + pub fn complete(j: *FakeJob) void { + std.debug.assert(j.input != .pending); + std.debug.assert(j.poison_ or j.ran); + j.completed = true; + } +}; + +test "failed input with no odd signal releases its even wait without running ANE" { + var j: FakeJob = .{ .fail_after_wait = true }; + execute(&j); + try std.testing.expect(j.poison_ and j.completed and !j.ran); + try std.testing.expectEqual(@as(usize, 1), j.waits); +} + +test "a poisoned followup still waits its input CB before releasing its even wait" { + var j: FakeJob = .{ .poison_ = true }; + execute(&j); + try std.testing.expect(j.completed and !j.ran); + try std.testing.expectEqual(@as(usize, 1), j.waits); +} + +test "successful input runs before worker completion" { + var j: FakeJob = .{}; + execute(&j); + try std.testing.expect(j.ran and j.completed and !j.poison_); +} + +const FakeBatch = struct { + waited: [3]bool = @splat(false), + released: [3]bool = @splat(false), + worker_done: bool = false, + cleared: bool = false, + pub fn count(_: *FakeBatch) usize { + return 3; + } + pub fn waitBuffer(b: *FakeBatch, i: usize) void { + b.waited[i] = true; + } + pub fn bufferFailed(_: *FakeBatch, i: usize) bool { + return i != 1; + } + pub fn waitWorker(b: *FakeBatch) void { + std.debug.assert(std.mem.allEqual(bool, &b.waited, true)); + b.worker_done = true; + } + pub fn releaseBuffer(b: *FakeBatch, i: usize) void { + std.debug.assert(b.worker_done); + b.released[i] = true; + } + pub fn clear(b: *FakeBatch) void { + std.debug.assert(std.mem.allEqual(bool, &b.released, true)); + b.cleared = true; + } +}; + +test "drain waits all CBs and the worker despite first and last CB failures" { + var b: FakeBatch = .{}; + try std.testing.expect(drain(&b)); + try std.testing.expect(b.cleared and b.worker_done); +} + +test "failed reservation publishes no ticket and the next successful ticket is zero" { + const a = std.testing.allocator; + var q: Ring(u32, 2) = .{}; + var retained: std.ArrayList(u32) = .empty; + defer retained.deinit(a); + var failing = std.testing.FailingAllocator.init(a, .{ .fail_index = 0 }); + try std.testing.expectError(error.OutOfMemory, q.prepare(failing.allocator(), &retained)); + try std.testing.expectEqual(@as(u64, 0), q.issued); + try std.testing.expect(q.get(0) == null); + try q.prepare(a, &retained); + retained.appendAssumeCapacity(42); + q.publish(42); + try std.testing.expectEqual(@as(u32, 42), q.get(0).?.*); + try std.testing.expectEqual(@as(u64, 1), q.issued); + q.complete(0); +} + +test "real producer and consumer publish complete payloads through repeated ring wraparound" { + const Payload = struct { t: u64, checksum: u64 }; + const Queue = Ring(Payload, 4); + const Consumer = struct { + fn run(q: *Queue, bad: *std.atomic.Value(bool)) void { + for (0..1024) |t| { + while (true) { + if (q.get(t)) |p| { + if (p.t != t or p.checksum != t ^ 0xaabbccdd) bad.store(true, .release); + q.complete(t); + break; + } + std.atomic.spinLoopHint(); + } + } + } + }; + var q: Queue = .{}; + var bad: std.atomic.Value(bool) = .init(false); + var retained: std.ArrayList(u32) = .empty; + defer retained.deinit(std.testing.allocator); + // Reserve once so an allocation error cannot leave the consumer waiting for unpublished work. + try retained.ensureTotalCapacity(std.testing.allocator, 1024); + const thread = try std.Thread.spawn(.{}, Consumer.run, .{ &q, &bad }); + defer thread.join(); + for (0..1024) |t| { + while (true) { + q.prepare(std.testing.allocator, &retained) catch |err| { + std.debug.assert(err == error.AneHandoffFull); + std.atomic.spinLoopHint(); + continue; + }; + break; + } + retained.appendAssumeCapacity(@intCast(t)); + q.publish(.{ .t = t, .checksum = t ^ 0xaabbccdd }); + } + while (q.completed.load(.acquire) < 1024) std.atomic.spinLoopHint(); + try std.testing.expect(!bad.load(.acquire)); +} diff --git a/zig/src/families/qwen27/ane_mlp.zig b/zig/src/families/qwen27/ane_mlp.zig index 64a183e2c7..b0fb01c9ec 100644 --- a/zig/src/families/qwen27/ane_mlp.zig +++ b/zig/src/families/qwen27/ane_mlp.zig @@ -5,6 +5,7 @@ const ane = mtl.ane; const cp = @import("core").checkpoint_metal; const Lane = @import("core").segments.Lane; const Config = @import("config.zig").Config; +const handoff = @import("ane_handoff.zig"); /// Program rows: a prompt chunk of up to 128 rows, zero past its last (rows never depend on each other). pub const width = 128; @@ -24,12 +25,17 @@ pub const Share = struct { /// Command buffers committed at handoffs during the current prompt chunk; drain() checks them. committed: std.ArrayList(mtl.CommandBuffer) = .empty, gpu_seconds: f64 = 0, - calls: u64 = 0, - ring: [256]std.atomic.Value(u32) = @splat(.init(0)), + tickets: handoff.Ring(Ticket, 256) = .{}, + poisoned: std.atomic.Value(bool) = .init(false), stop: std.atomic.Value(bool) = .init(false), failed: std.atomic.Value(bool) = .init(false), thread: ?std.Thread = null, + const Ticket = struct { + layer: usize, + cb: mtl.CommandBuffer, + }; + /// The fp16 weights a share of `columns` holds: every layer's gate, up and down over those columns. pub fn bytes(c: Config, columns: u32) u64 { return 3 * @as(u64, columns) * c.hidden * 2 * c.layers; @@ -78,6 +84,7 @@ pub const Share = struct { } pub fn deinit(s: *Share) void { + s.drain() catch {}; s.stop.store(true, .release); if (s.thread) |t| t.join(); for (s.requests) |r| r.deinit(); @@ -93,24 +100,26 @@ pub const Share = struct { s.allocator.destroy(s); } - /// Queues `layer`'s program; the GPU signals [0] once inputs are written and waits for [1] before the partial. - pub fn next(s: *Share, layer: usize) [2]u64 { - const t = s.calls; - s.calls += 1; - s.ring[t % s.ring.len].store(@intCast(layer), .release); - return .{ 2 * t + 1, 2 * t + 2 }; + /// Reserve before encoding inputs: no ticket or committed CB exists if allocation fails here. + pub fn prepare(s: *Share) !void { + if (s.poisoned.load(.acquire)) return error.AnePoisoned; + try s.tickets.prepare(s.allocator, &s.committed); } - /// Signals `value` at the buffer's end and commits it, then opens the next behind the fence. - pub fn signal(s: *Share, lane: *Lane, value: u64) !void { + /// After prepare and successful input encoding: publish a retained CB, then return its worker's even signal. + pub fn submit(s: *Share, lane: *Lane, layer: usize) u64 { + const t = s.tickets.issued; + const cb = lane.cb.retain(); + s.committed.appendAssumeCapacity(cb); lane.fence.update(lane.enc); lane.enc.end(); - lane.cb.signal(s.event, value); + lane.cb.signal(s.event, 2 * t + 1); lane.cb.commit(); - try s.committed.append(s.allocator, .{ .id = mtl.objc.retain(lane.cb.id) }); + s.tickets.publish(.{ .layer = layer, .cb = cb }); lane.cb = s.queue.commandBuffer(); lane.enc = lane.cb.compute(.concurrent); lane.fence.wait(lane.enc); + return 2 * t + 2; } /// The lane's later work waits for `value` (the program's output), between its encoders. @@ -124,29 +133,82 @@ pub const Share = struct { /// After a prompt chunk: the handoff buffers finished without error and the programs ran. pub fn drain(s: *Share) !void { - defer { - for (s.committed.items) |cb| mtl.objc.release(cb.id); - s.committed.clearRetainingCapacity(); - } - for (s.committed.items) |cb| { - cb.wait(); - if (@import("decode_round.zig").gpuFailed(cb, "a prompt chunk's Neural Engine handoff")) return error.GpuFailed; - s.gpu_seconds += cb.gpuSeconds(); + var batch = Batch{ .share = s }; + if (handoff.drain(&batch)) { + s.poisoned.store(true, .release); + return error.GpuFailed; } + if (s.poisoned.load(.acquire)) return error.AnePoisoned; if (s.failed.swap(false, .acq_rel)) return error.AneFailed; // that chunk fails; the next one runs again } + const Batch = struct { + share: *Share, + pub fn count(b: *Batch) usize { + return b.share.committed.items.len; + } + pub fn waitBuffer(b: *Batch, i: usize) void { + b.share.committed.items[i].wait(); + } + pub fn bufferFailed(b: *Batch, i: usize) bool { + const cb = b.share.committed.items[i]; + if (@import("decode_round.zig").gpuFailed(cb, "a prompt chunk's Neural Engine handoff")) return true; + b.share.gpu_seconds += cb.gpuSeconds(); + return false; + } + pub fn waitWorker(b: *Batch) void { + const s = b.share; + while (s.tickets.completed.load(.acquire) < s.tickets.issued) _ = s.event.wait(2 * s.tickets.issued, 100); + } + pub fn releaseBuffer(b: *Batch, i: usize) void { + b.share.committed.items[i].release(); + } + pub fn clear(b: *Batch) void { + b.share.committed.clearRetainingCapacity(); + } + }; + + const Job = struct { + share: *Share, + ticket: *const Ticket, + t: u64, + spins: u64 = 0, + pub fn ready(j: *Job) bool { + return j.share.event.value() >= 2 * j.t + 1; + } + pub fn inputFailed(j: *Job) bool { + return j.ticket.cb.status() == .@"error"; + } + pub fn poison(j: *Job) void { + j.share.poisoned.store(true, .release); + } + pub fn poisoned(j: *Job) bool { + return j.share.poisoned.load(.acquire); + } + pub fn waitInput(j: *Job) void { + j.spins += 1; + if (j.spins < 2_000_000) std.atomic.spinLoopHint() else _ = j.share.event.wait(2 * j.t + 1, 100); + } + pub fn evaluate(j: *Job) void { + j.share.requests[j.ticket.layer].run() catch j.share.failed.store(true, .release); + } + pub fn complete(j: *Job) void { + j.share.event.set(2 * j.t + 2); + j.share.tickets.complete(j.t); + } + }; + fn work(s: *Share) void { var t: u64 = 0; while (true) : (t += 1) { - // spin on the counter while prompt chunks run: a sleeping wait wakes too late for a per-layer handoff var spins: u64 = 0; - while (s.event.value() < 2 * t + 1) : (spins += 1) { - if (spins < 2_000_000) std.atomic.spinLoopHint() else if (s.event.wait(2 * t + 1, 100)) break else if (s.stop.load(.acquire)) return; - } - const layer = s.ring[t % s.ring.len].load(.acquire); - s.requests[layer].run() catch s.failed.store(true, .release); - s.event.set(2 * t + 2); + const ticket = while (true) : (spins += 1) { + if (s.tickets.get(t)) |ticket| break ticket; + if (s.stop.load(.acquire)) return; + if (spins < 2_000_000) std.atomic.spinLoopHint() else _ = s.event.wait(2 * t + 1, 100); + }; + var job = Job{ .share = s, .ticket = ticket, .t = t }; + handoff.execute(&job); } } }; diff --git a/zig/src/families/qwen27/decode_round.zig b/zig/src/families/qwen27/decode_round.zig index 4a405c0aea..9c51a9f5d8 100644 --- a/zig/src/families/qwen27/decode_round.zig +++ b/zig/src/families/qwen27/decode_round.zig @@ -223,6 +223,9 @@ pub const Runner = struct { var stages = forward.Stages{ .model = r.model, .rows = @intCast(round.ids.len), .hooks = .{ .ptr = r, .gdn = gdnForward, .attention = attentionForward, .tap = if (r.capture_taps) tap else null }, .head = head, .images = placed.images }; r.last_gpu_seconds = @import("core").segments.runCommitting(r.model.device, &.{r.model.queue}, r.model.config.layers, .concurrent, &stages, commit_every) catch |err| { r.failed = true; + if (r.model.ane) |share| share.drain() catch |drain_err| { + std.log.warn("qwen27: ANE cleanup after {s}: {s}", .{ @errorName(err), @errorName(drain_err) }); + }; return err; }; if (r.model.ane) |share| share.drain() catch |err| { diff --git a/zig/src/families/qwen27/forward.zig b/zig/src/families/qwen27/forward.zig index c268b7dfd5..8c5425153d 100644 --- a/zig/src/families/qwen27/forward.zig +++ b/zig/src/families/qwen27/forward.zig @@ -84,18 +84,18 @@ pub const Stages = struct { const width: u32 = @intCast(m.config.intermediate); if (if (m.kernels.prompt) m.ane else null) |share| { // the Neural Engine runs MLP columns below share.columns; the GPU runs the rest, then adds its partial - const v = share.next(index); const planes = Ref{ .buffer = share.output.buffer }; const own = share.columns / 64; const all = width / 64; + try share.prepare(); try m.kernels.aneInput(e, f.get(.input), f.get(.dims), .{ .buffer = share.input.buffer }, h, width, share.columns); - try share.signal(lane, v[0]); + const finished = share.submit(lane, index); try m.kernels.quantSkip(lane.enc, layer.gu, f.get(.input), sums(f, .sums, s.rows), f.get(.dims), f.get(.mixed), s.rows, share.columns / 32); lane.enc.barrier(); try m.kernels.mlpAne(lane.enc, f.get(.mixed), up, f.get(.dims), f.get(.activated), f.get(.act_sums), planes, h, width, share.columns, s.rows, own, all - own); lane.enc.barrier(); try m.kernels.quantCut(lane.enc, layer.down, f.get(.activated), sums(f, .act_sums, s.rows), f.get(.dims), .{ .buffer = share.partial }, s.rows, .{ own, all - own }); - share.wait(lane, v[1]); + share.wait(lane, finished); try m.kernels.addAne(lane.enc, .{ .buffer = share.partial }, planes, f.get(.output), h, width, share.columns, s.rows); lane.enc.barrier(); s.pending = f.get(.output); diff --git a/zig/src/families/qwen27/hidden_rows.zig b/zig/src/families/qwen27/hidden_rows.zig new file mode 100644 index 0000000000..d06c38da5f --- /dev/null +++ b/zig/src/families/qwen27/hidden_rows.zig @@ -0,0 +1,64 @@ +//! Final normalized prompt rows, with their own frame and fresh recurrent/KV state for each call. +const std = @import("std"); +const mtl = @import("metal"); +const Model = @import("model.zig").Model; +const Frame = @import("gpu_frame.zig").Frame; +const Runner = @import("decode_round.zig").Runner; +const Round = @import("round_plan.zig").Round; +pub const Plan = @import("hidden_rows_plan.zig").Plan; + +pub const Options = struct { + context: u32, + chunk: u32 = 128, + cancel: ?*const std.atomic.Value(bool) = null, + + fn check(o: Options) !void { + if (o.cancel) |c| if (c.load(.acquire)) return error.Canceled; + } +}; + +/// Owned bf16 bits in token order, [count][hidden], independent of later forward passes. +pub const Rows = struct { + allocator: std.mem.Allocator, + values: []u16, + count: u32, + hidden: usize, + + pub fn deinit(r: *Rows) void { + r.allocator.free(r.values); + r.* = undefined; + } +}; + +/// Caller owns the model exclusively for the entire call (the engine thread, never an HTTP thread). +pub fn captureExclusive(a: std.mem.Allocator, m: *Model, ids: []const u32, o: Options) !Rows { + const p = try Plan.init(ids, m.config.hidden, m.config.vocab, m.frame.capacity, o.context, o.chunk); + try o.check(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const out = try a.alloc(u16, p.values); + errdefer a.free(out); + var frame = try Frame.init(m.device, m.config, m.frame.capacity); + defer frame.deinit(); + std.mem.swap(Frame, &m.frame, &frame); + defer std.mem.swap(Frame, &m.frame, &frame); + const prompt = m.kernels.prompt; + m.kernels.prompt = true; + defer m.kernels.prompt = prompt; + var runner = try Runner.init(a, m, 1, p.rows); + defer runner.deinit(); + var first: usize = 0; + while (first < ids.len) { + try o.check(); + const end = @min(ids.len, first + o.chunk); + var round = try Round.init(a, &.{.{ .slot = 0, .start = runner.offsets[0], .capacity = runner.capacity, .ids = ids[first..end] }}, @intCast(m.config.conv_kernel), 1, @intCast(m.config.vocab)); + defer round.deinit(); + try runner.advance(&round, .none); + const values = m.frame.get(.input).buffer.slice(u16, (end - first) * m.config.hidden); + for (values) |bits| if (bits & 0x7f80 == 0x7f80) return error.NonfiniteHiddenRows; + @memcpy(out[first * m.config.hidden .. end * m.config.hidden], values); + first = end; + } + try o.check(); + return .{ .allocator = a, .values = out, .count = p.rows, .hidden = m.config.hidden }; +} diff --git a/zig/src/families/qwen27/hidden_rows_plan.zig b/zig/src/families/qwen27/hidden_rows_plan.zig new file mode 100644 index 0000000000..458a5d13fe --- /dev/null +++ b/zig/src/families/qwen27/hidden_rows_plan.zig @@ -0,0 +1,67 @@ +//! Check a whole disposable prompt before it can allocate device state or replace the chat frame. +const std = @import("std"); + +pub const Plan = struct { + rows: u32, + values: usize, + + pub fn init(ids: []const u32, hidden: usize, vocab: usize, frame: u32, context: u32, chunk: u32) !Plan { + if (ids.len == 0 or ids.len > context or ids.len > std.math.maxInt(i32)) return error.BadHiddenPrompt; + if (hidden == 0 or vocab == 0 or vocab > std.math.maxInt(u32)) return error.BadHiddenGeometry; + if (frame == 0 or frame > 128 or frame % 16 != 0 or chunk == 0 or chunk > frame) return error.BadHiddenChunk; + for (ids) |id| if (id >= vocab) return error.BadTokenId; + const values = try std.math.mul(usize, ids.len, hidden); + _ = try std.math.mul(usize, values, @sizeOf(u16)); + return .{ .rows = @intCast(ids.len), .values = values }; + } + + /// The owned CPU rows and private KV cache must fit beside the fixed capture buffers and server reserve. + pub fn admit(p: Plan, available: usize, fixed: usize, kv_width: usize, attention_layers: usize) !void { + const output = try std.math.mul(usize, p.values, @sizeOf(u16)); + const kv = try std.math.mul(usize, try std.math.mul(usize, p.rows, kv_width), 4); + const caches = try std.math.mul(usize, kv, attention_layers); + const required = try std.math.add(usize, try std.math.add(usize, fixed, output), caches); + if (required > available) return error.HiddenRowsMemory; + } +}; + +test "all tokens including later chunks are checked before a disposable pass" { + try std.testing.expectError(error.BadTokenId, Plan.init(&.{ 1, 2, 3, 96 }, 512, 96, 128, 4, 1)); + const p = try Plan.init(&.{ 0, 95 }, 512, 96, 128, 2, 1); + try std.testing.expectEqual(@as(u32, 2), p.rows); + try std.testing.expectEqual(@as(usize, 1024), p.values); +} + +test "empty and oversized prompts and incompatible frame sizes fail before allocation" { + try std.testing.expectError(error.BadHiddenPrompt, Plan.init(&.{}, 512, 96, 128, 4, 1)); + try std.testing.expectError(error.BadHiddenPrompt, Plan.init(&.{ 1, 2 }, 512, 96, 128, 1, 1)); + for ([_]u32{ 0, 1, 127, 129 }) |frame| + try std.testing.expectError(error.BadHiddenChunk, Plan.init(&.{1}, 512, 96, frame, 1, 1)); + for ([_]u32{ 0, 129 }) |chunk| + try std.testing.expectError(error.BadHiddenChunk, Plan.init(&.{1}, 512, 96, 128, 1, chunk)); +} + +test "hidden row allocation checks both element and byte multiplication" { + try std.testing.expectError(error.Overflow, Plan.init(&.{ 0, 0 }, std.math.maxInt(usize), 96, 128, 2, 1)); + try std.testing.expectError(error.Overflow, Plan.init(&.{0}, std.math.maxInt(usize), 96, 128, 1, 1)); + try std.testing.expectError(error.BadHiddenGeometry, Plan.init(&.{0}, 0, 96, 128, 1, 1)); +} + +test "a valid long capture is refused when private state would consume the chat reserve" { + const p = Plan{ .rows = 196608, .values = 196608 * 5120 }; + const fixed = 256 << 20; + const required = fixed + (196608 * 5120 * 2) + (196608 * 1024 * 4 * 16); + try std.testing.expectError(error.HiddenRowsMemory, p.admit(8 << 30, fixed, 1024, 16)); + try std.testing.expectError(error.HiddenRowsMemory, p.admit(required - 1, fixed, 1024, 16)); + try p.admit(required, fixed, 1024, 16); + try std.testing.expectError(error.HiddenRowsMemory, (Plan{ .rows = 1, .values = 5120 }).admit(10240, 1, 0, 0)); +} + +test "capture admission refuses overflowing byte totals rather than wrapping into free room" { + const max = std.math.maxInt(usize); + try std.testing.expectError(error.Overflow, (Plan{ .rows = 1, .values = max }).admit(max, 0, 0, 0)); + const p = Plan{ .rows = 2, .values = 2 }; + try std.testing.expectError(error.Overflow, p.admit(max, 0, max, 1)); + try std.testing.expectError(error.Overflow, p.admit(max, 0, 1, max)); + try std.testing.expectError(error.Overflow, p.admit(max, max, 0, 0)); +} diff --git a/zig/src/families/qwen27/qwen27.zig b/zig/src/families/qwen27/qwen27.zig index d5e6124e8c..3d3437d624 100644 --- a/zig/src/families/qwen27/qwen27.zig +++ b/zig/src/families/qwen27/qwen27.zig @@ -20,6 +20,7 @@ pub const glue = @import("glue.zig"); pub const glue_gpu = @import("glue_gpu.zig"); pub const image_rows = @import("image_rows.zig"); pub const gpu_weights = @import("gpu_weights.zig"); +pub const hidden_rows = @import("hidden_rows.zig"); pub const gpu_frame = @import("gpu_frame.zig"); pub const model = @import("model.zig"); pub const projection = @import("projection.zig"); diff --git a/zig/src/native/qwen27_served.zig b/zig/src/native/qwen27_served.zig index 168d9304c8..b7846bc173 100644 --- a/zig/src/native/qwen27_served.zig +++ b/zig/src/native/qwen27_served.zig @@ -49,6 +49,47 @@ pub const Host = struct { return h.host.engine(); } + /// Capture on the owner thread, between chat rounds; output stays owned by the calling allocator. + pub fn hiddenRows(h: *Host, a: Allocator, ids: []const u32, chunk: u32, cancel: ?*const std.atomic.Value(bool)) !q.hidden_rows.Rows { + const Call = struct { + a: Allocator, + host: *Host, + ids: []const u32, + options: q.hidden_rows.Options, + rows: ?q.hidden_rows.Rows = null, + failure: ?anyerror = null, + fn run(ptr: *anyopaque) void { + const c: *@This() = @ptrCast(@alignCast(ptr)); + c.host.admitRows(c.ids, c.options) catch |err| { + c.failure = err; + return; + }; + c.rows = q.hidden_rows.captureExclusive(c.a, c.host.model, c.ids, c.options) catch |err| { + c.failure = err; + return; + }; + } + }; + var call = Call{ .a = a, .host = h, .ids = ids, .options = .{ .context = h.host.info_.context_window, .chunk = chunk, .cancel = cancel } }; + try h.host.callOwner(&call, Call.run); + if (call.failure) |err| return err; + return call.rows.?; + } + + /// Existing multi-slot buffers bound a private single-slot pass; refuse without evicting chat snapshots. + fn admitRows(h: *Host, ids: []const u32, options: q.hidden_rows.Options) !void { + const m = h.model; + const plan = try q.hidden_rows.Plan.init(ids, m.config.hidden, m.config.vocab, m.frame.capacity, options.context, options.chunk); + var fixed: usize = h.runner.taps.buffer.length(); + inline for (.{ m.frame.buffers, h.runner.gdn.buffers, h.runner.scratch.buffers }) |buffers| { + for (buffers) |buffer| fixed = try std.math.add(usize, fixed, buffer.length()); + } + for (h.runner.bindings) |binding| for (binding.buffers) |buffer| { + fixed = try std.math.add(usize, fixed, buffer.length()); + }; + try plan.admit(room(m), fixed, m.config.kvDim(), h.runner.bindings.len); + } + fn of(ptr: *anyopaque) *Host { return @ptrCast(@alignCast(ptr)); } diff --git a/zig/tests/fixtures/clef/joint_head.safetensors b/zig/tests/fixtures/clef/joint_head.safetensors new file mode 100644 index 0000000000..38146c7c56 Binary files /dev/null and b/zig/tests/fixtures/clef/joint_head.safetensors differ diff --git a/zig/tests/fixtures/clef/joint_head_config.json b/zig/tests/fixtures/clef/joint_head_config.json new file mode 100644 index 0000000000..08ae98e634 --- /dev/null +++ b/zig/tests/fixtures/clef/joint_head_config.json @@ -0,0 +1 @@ +{"hidden_size": 16, "width": 8, "routing_layers": 2, "layers": 4, "heads": 2, "feedforward": 16} diff --git a/zig/tests/fixtures/clef/source.json b/zig/tests/fixtures/clef/source.json new file mode 100644 index 0000000000..499e158381 --- /dev/null +++ b/zig/tests/fixtures/clef/source.json @@ -0,0 +1,8 @@ +{ + "kind": "synthetic head checkpoint, not the model agreement corpus", + "generator": "JointSchemaHead with torch.manual_seed(317), converted to bf16", + "release": "Cloudflare/clef-flash", + "release_revision": "8b2e5fd17c09fd49bc2880b805d5323ec7ff4ff3", + "release_sha256": "0e304cf7c6500e8bb59bef7e2afd2c6373f82596dfb3b57d1aa93c175e2dc3a3", + "prior_art": ["ashhart/TensorFold#241 by outcastofmusic", "ashhart/TensorFold#251 by outcastofmusic"] +} diff --git a/zig/tests/qwen27/hidden_rows.zig b/zig/tests/qwen27/hidden_rows.zig new file mode 100644 index 0000000000..4df48208e7 --- /dev/null +++ b/zig/tests/qwen27/hidden_rows.zig @@ -0,0 +1,136 @@ +//! Synthetic normalized-row, chat and kept-state checks; build-only until an exclusive Metal test window. +const std = @import("std"); +const mtl = @import("metal"); +const q = @import("tensorfold").qwen27; +const graph = @import("synthetic_graph.zig"); + +/// Normal chat prompt arithmetic, with the LM head projected, keeps every row's final normalized bf16 bits. +fn reference(a: std.mem.Allocator, m: *q.model.Model, ids: []const u32, chunk: usize) ![]u16 { + const out = try a.alloc(u16, ids.len * m.config.hidden); + errdefer a.free(out); + var runner = try q.decode_round.Runner.init(a, m, 1, @intCast(ids.len)); + defer runner.deinit(); + const prompt = m.kernels.prompt; + m.kernels.prompt = true; + defer m.kernels.prompt = prompt; + var first: usize = 0; + while (first < ids.len) { + const end = @min(ids.len, first + chunk); + var round = try q.round_plan.Round.init(a, &.{.{ .slot = 0, .start = runner.offsets[0], .capacity = runner.capacity, .ids = ids[first..end] }}, @intCast(m.config.conv_kernel), 1, @intCast(m.config.vocab)); + defer round.deinit(); + try runner.advance(&round, .all); + @memcpy(out[first * m.config.hidden .. end * m.config.hidden], m.frame.get(.input).buffer.slice(u16, (end - first) * m.config.hidden)); + first = end; + } + return out; +} + +fn unchanged(r: *q.decode_round.Runner, frame: q.gpu_frame.Frame, prompt: bool, fingerprint: [32]u8) !void { + for (r.model.frame.buffers, frame.buffers) |got, want| if (got.id != want.id) return error.ChatFrameChanged; + if (r.model.kernels.prompt != prompt) return error.PromptFlagChanged; + if (!std.mem.eql(u8, &fingerprint, &try (q.session.Session{ .runner = r }).fingerprint())) return error.ChatStateChanged; +} + +fn tokens(s: q.session.Session, out: []u32) !void { + for (out, 0..) |*id, i| { + id.* = try s.greedy(); + if (i + 1 < out.len) try s.step(id.*); + } +} + +fn restoreChat(a: std.mem.Allocator, r: *q.decode_round.Runner, saved: *q.snapshot.State, last: []const u32) !void { + try q.snapshot.restore(a, r, null, saved); + try (q.session.Session{ .runner = r }).promptChunk(last, true); +} + +pub fn main(init: std.process.Init) !void { + var arena = std.heap.ArenaAllocator.init(init.gpa); + defer arena.deinit(); + const a = arena.allocator(); + const args = try init.minimal.args.toSlice(a); + if (args.len > 2 or (args.len == 2 and !std.mem.eql(u8, args[1], "--ane"))) return error.Usage; + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const m = try graph.create(a); + defer m.deinit(); + if (args.len == 2) m.ane = try q.ane_mlp.Share.init(a, m.device, m.queue, &m.checkpoint, m.config, 64); + var ids: [130]u32 = undefined; + for (&ids, 0..) |*id, i| id.* = @intCast((i * 17 + 7) % 96); + const expected_rows = try reference(a, m, &ids, 16); + defer a.free(expected_rows); + for ([_]u32{ 1, 16, 128 }) |chunk| { + var rows = try q.hidden_rows.captureExclusive(a, m, &ids, .{ .context = 256, .chunk = chunk }); + defer rows.deinit(); + if (rows.count != ids.len or rows.hidden != m.config.hidden) return error.RowShapeDiffers; + if (!std.mem.eql(u16, rows.values, expected_rows)) return error.NormalizedRowsDiffer; + var later = try q.hidden_rows.captureExclusive(a, m, ids[0..7], .{ .context = 256, .chunk = chunk }); + defer later.deinit(); + if (!std.mem.eql(u16, rows.values, expected_rows)) return error.RowsAliasLaterPass; + } + + var runner = try q.decode_round.Runner.init(a, m, 1, 256); + defer runner.deinit(); + const session = q.session.Session{ .runner = &runner }; + const chat = ids[0..11]; + try session.prefill(chat[0 .. chat.len - 1], 16); + const saved = try q.snapshot.save(a, &runner, null, @intCast(chat.len - 1)); + defer q.snapshot.drop(a, saved); + try session.promptChunk(chat[chat.len - 1 ..], true); + var expected_tokens: [8]u32 = undefined; + try tokens(session, &expected_tokens); + try restoreChat(a, &runner, saved, chat[chat.len - 1 ..]); + const original_frame = m.frame; + const original_state = try session.fingerprint(); + + for ([_]bool{ false, true }) |prompt| { + m.kernels.prompt = prompt; + var rows = try q.hidden_rows.captureExclusive(a, m, &ids, .{ .context = 256, .chunk = 16 }); + defer rows.deinit(); + try unchanged(&runner, original_frame, prompt, original_state); + } + m.kernels.prompt = false; + var canceled: std.atomic.Value(bool) = .init(true); + if (q.hidden_rows.captureExclusive(a, m, &ids, .{ .context = 256, .cancel = &canceled })) |rows_| { + var rows = rows_; + rows.deinit(); + return error.ExpectedCancellation; + } else |err| if (err != error.Canceled) return err; + try unchanged(&runner, original_frame, false, original_state); + + try @import("hidden_rows_cancel.zig").run(a, m, &ids, 16); + try unchanged(&runner, original_frame, false, original_state); + // A completed forward whose final norm is nonfinite must fail without touching the chat frame or caches. + { + const norm = m.weights.norm; + const scales = norm.buffer.slice(u16, norm.offset / 2 + m.config.hidden)[norm.offset / 2 ..]; + const before = scales[0]; + scales[0] = 0x7fc0; + defer scales[0] = before; + if (q.hidden_rows.captureExclusive(a, m, &ids, .{ .context = 256, .chunk = 16 })) |rows_| { + var rows = rows_; + rows.deinit(); + return error.ExpectedNonfiniteRows; + } else |err| if (err != error.NonfiniteHiddenRows) return err; + } + try unchanged(&runner, original_frame, false, original_state); + + // Allocation failures after the private frame is installed must also restore the chat frame and flag. + for ([_]usize{ 0, 1, 2, 3, 8, 16 }) |at| { + var failing = std.testing.FailingAllocator.init(a, .{ .fail_index = at }); + if (q.hidden_rows.captureExclusive(failing.allocator(), m, &ids, .{ .context = 256, .chunk = 16 })) |rows_| { + var rows = rows_; + rows.deinit(); + return error.ExpectedAllocationFailure; + } else |err| if (err != error.OutOfMemory or !failing.has_induced_failure) return err; + if (failing.allocated_bytes != failing.freed_bytes) return error.CaptureHostLeak; + try unchanged(&runner, original_frame, false, original_state); + } + var actual_tokens: [8]u32 = undefined; + try tokens(session, &actual_tokens); + if (!std.mem.eql(u32, &actual_tokens, &expected_tokens)) return error.ChatReplyChanged; + try restoreChat(a, &runner, saved, chat[chat.len - 1 ..]); + try unchanged(&runner, original_frame, false, original_state); + try tokens(session, &actual_tokens); + if (!std.mem.eql(u32, &actual_tokens, &expected_tokens)) return error.ResumedReplyChanged; + std.debug.print("hidden rows: exact bits, owned rows, chat/resume, cancel, OOM and nonfinite isolation passed; ANE={}\n", .{m.ane != null}); +} diff --git a/zig/tests/qwen27/hidden_rows_cancel.zig b/zig/tests/qwen27/hidden_rows_cancel.zig new file mode 100644 index 0000000000..bb5f4005e0 --- /dev/null +++ b/zig/tests/qwen27/hidden_rows_cancel.zig @@ -0,0 +1,82 @@ +//! Cancel and allocation failure after the allocation budget of a complete first prompt chunk. +const std = @import("std"); +const q = @import("tensorfold").qwen27; + +const InterruptAllocator = struct { + backing: std.mem.Allocator, + cancel: *std.atomic.Value(bool), + oom: bool = false, + at: usize, + calls: usize = 0, + expected: []const u8, + output: ?[]u8 = null, + saw_completed_first_chunk: bool = false, + fired: bool = false, + + fn allocator(c: *InterruptAllocator) std.mem.Allocator { + return .{ .ptr = c, .vtable = &.{ .alloc = alloc, .resize = resize, .remap = remap, .free = free } }; + } + fn alloc(ptr: *anyopaque, len: usize, alignment: std.mem.Alignment, ra: usize) ?[*]u8 { + const c: *InterruptAllocator = @ptrCast(@alignCast(ptr)); + if (c.calls == c.at) { + // The CPU writes output only after advance/drain returns; never read GPU buffers during encoding. + if (c.output) |out| c.saw_completed_first_chunk = out.len >= c.expected.len and std.mem.eql(u8, out[0..c.expected.len], c.expected); + c.fired = true; + if (c.oom) return null; + c.cancel.store(true, .release); + } + const out = c.backing.rawAlloc(len, alignment, ra) orelse return null; + if (c.calls == 0) { + c.output = out[0..len]; + @memset(out[0..len], 0xff); + } + c.calls += 1; + return out; + } + fn resize(ptr: *anyopaque, memory: []u8, alignment: std.mem.Alignment, len: usize, ra: usize) bool { + const c: *InterruptAllocator = @ptrCast(@alignCast(ptr)); + return c.backing.rawResize(memory, alignment, len, ra); + } + fn remap(ptr: *anyopaque, memory: []u8, alignment: std.mem.Alignment, len: usize, ra: usize) ?[*]u8 { + const c: *InterruptAllocator = @ptrCast(@alignCast(ptr)); + return c.backing.rawRemap(memory, alignment, len, ra); + } + fn free(ptr: *anyopaque, memory: []u8, alignment: std.mem.Alignment, ra: usize) void { + const c: *InterruptAllocator = @ptrCast(@alignCast(ptr)); + c.backing.rawFree(memory, alignment, ra); + } +}; + +pub fn run(a: std.mem.Allocator, m: *q.model.Model, ids: []const u32, chunk: u32) !void { + if (ids.len <= chunk) return error.NeedMultipleChunks; + var count = std.testing.FailingAllocator.init(a, .{}); + var first = try q.hidden_rows.captureExclusive(count.allocator(), m, ids[0..chunk], .{ .context = 256, .chunk = chunk }); + const expected = a.dupe(u16, first.values) catch |err| { + first.deinit(); + return err; + }; + defer a.free(expected); + first.deinit(); + if (count.allocated_bytes != count.freed_bytes) return error.CalibrationLeak; + + var canceled: std.atomic.Value(bool) = .init(false); + var cancel_memory = std.testing.FailingAllocator.init(a, .{}); + var interrupt = InterruptAllocator{ .backing = cancel_memory.allocator(), .cancel = &canceled, .at = count.alloc_index, .expected = std.mem.sliceAsBytes(expected) }; + if (q.hidden_rows.captureExclusive(interrupt.allocator(), m, ids, .{ .context = 256, .chunk = chunk, .cancel = &canceled })) |rows_| { + var rows = rows_; + rows.deinit(); + return error.ExpectedMidPassCancellation; + } else |err| if (err != error.Canceled or !canceled.load(.acquire)) return err; + if (!interrupt.saw_completed_first_chunk) return error.CancellationWasTooEarly; + if (cancel_memory.allocated_bytes != cancel_memory.freed_bytes) return error.CanceledHostLeak; + + var failing = std.testing.FailingAllocator.init(a, .{}); + var oom = InterruptAllocator{ .backing = failing.allocator(), .cancel = &canceled, .at = count.alloc_index, .expected = std.mem.sliceAsBytes(expected), .oom = true }; + if (q.hidden_rows.captureExclusive(oom.allocator(), m, ids, .{ .context = 256, .chunk = chunk })) |rows_| { + var rows = rows_; + rows.deinit(); + return error.ExpectedMidPassAllocationFailure; + } else |err| if (err != error.OutOfMemory or !oom.fired) return err; + if (!oom.saw_completed_first_chunk) return error.AllocationFailureWasTooEarly; + if (failing.allocated_bytes != failing.freed_bytes) return error.MidPassHostLeak; +} diff --git a/zig/tests/qwen27/hidden_rows_served.zig b/zig/tests/qwen27/hidden_rows_served.zig new file mode 100644 index 0000000000..33d81d21a0 --- /dev/null +++ b/zig/tests/qwen27/hidden_rows_served.zig @@ -0,0 +1,143 @@ +//! Queued captures beside a real synthetic chat; build-only until an exclusive Metal window. +const std = @import("std"); +const mtl = @import("metal"); +const api = @import("engine_api"); +const q = @import("tensorfold").qwen27; +const served = @import("qwen27_served"); +const graph = @import("synthetic_graph.zig"); +const Flag = std.atomic.Value(bool); + +fn wait(io: std.Io, flag: *const Flag) !void { + for (0..30_000) |_| { + if (flag.load(.acquire)) return; + try std.Io.sleep(io, .fromMilliseconds(1), .awake); + } + return error.WaitTimedOut; +} + +const Capture = struct { + host: *served.Host, + a: std.mem.Allocator, + ids: []const u32, + chunk: u32, + rows: ?q.hidden_rows.Rows = null, + failure: ?anyerror = null, + fn run(c: *Capture) void { + c.rows = c.host.hiddenRows(c.a, c.ids, c.chunk, null) catch |err| { + c.failure = err; + return; + }; + } +}; + +const Box = struct { + a: std.mem.Allocator, + io: std.Io, + mutex: std.Io.Mutex = .init, + prefilled: Flag = .init(false), + release: Flag = .init(false), + done: Flag = .init(false), + bad: Flag = .init(false), + tokens: std.ArrayList(u32) = .empty, + reason: ?api.Reason = null, + + fn event(ptr: *anyopaque, _: api.Id, e: *const api.Event) void { + const b: *Box = @ptrCast(@alignCast(ptr)); + if (e.* == .prefilled) { + b.prefilled.store(true, .release); + wait(b.io, &b.release) catch b.bad.store(true, .release); + return; + } + b.mutex.lockUncancelable(b.io); + defer b.mutex.unlock(b.io); + switch (e.*) { + .tokens => |ids| b.tokens.appendSlice(b.a, ids) catch b.bad.store(true, .release), + .finished => |f| { + b.reason = f.reason; + b.done.store(true, .release); + }, + else => {}, + } + } +}; + +pub fn main(init: std.process.Init) !void { + var arena = std.heap.ArenaAllocator.init(init.gpa); + defer arena.deinit(); + const a = arena.allocator(); + const shared = std.heap.smp_allocator; + const args = try init.minimal.args.toSlice(a); + if (args.len > 2 or (args.len == 2 and !std.mem.eql(u8, args[1], "--ane"))) return error.Usage; + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const m = try graph.create(a); + defer m.deinit(); + if (args.len == 2) m.ane = try q.ane_mlp.Share.init(a, m.device, m.queue, &m.checkpoint, m.config, 64); + var ids: [130]u32 = undefined; + for (&ids, 0..) |*id, i| id.* = @intCast((i * 17 + 7) % 96); + const chat = ids[0..11]; + var expected: [64]u32 = undefined; + { + var runner = try q.decode_round.Runner.init(a, m, 1, 256); + defer runner.deinit(); + const session = q.session.Session{ .runner = &runner }; + try session.prefill(chat, 128); + for (&expected, 0..) |*id, i| { + id.* = try session.greedy(); + if (i + 1 < expected.len) try session.step(id.*); + } + } + var why: []const u8 = ""; + const h = try served.attach(shared, init.io, m, null, .{ .context = 256, .streams = 2, .cache_gib = 0 }, a, &why); + var host_open = true; + defer if (host_open) served.close(h); + var solo = try h.hiddenRows(shared, &ids, 16, null); + defer solo.deinit(); + var box = Box{ .a = shared, .io = init.io }; + defer box.tokens.deinit(shared); + var jobs: [3]Capture = undefined; + var threads: [3]?std.Thread = @splat(null); + defer { + box.release.store(true, .release); + for (&threads) |*thread| if (thread.*) |t| { + t.join(); + thread.* = null; + }; + served.close(h); + host_open = false; + for (&jobs) |*job| if (job.rows) |*rows| rows.deinit(); + } + for (&jobs, [_]u32{ 1, 16, 128 }) |*job, chunk| job.* = .{ .host = h, .a = shared, .ids = &ids, .chunk = chunk }; + const request = api.Request{ .prompt = chat, .max_tokens = expected.len, .drafts = false }; + try h.engine().submit(1, &request, .{ .ctx = &box, .event = Box.event }); + try wait(init.io, &box.prefilled); + for (&threads, &jobs) |*thread, *job| thread.* = try std.Thread.spawn(.{}, Capture.run, .{job}); + var queued = false; + for (0..30_000) |_| { + h.host.lock(); + var count: usize = 0; + var next = h.host.calls.first; + while (next) |c| : (next = c.next) count += 1; + h.host.unlock(); + if (count == jobs.len) { + queued = true; + break; + } + try std.Io.sleep(init.io, .fromMilliseconds(1), .awake); + } + if (!queued) return error.CapturesNotQueuedDuringChat; + box.release.store(true, .release); + for (&threads) |*thread| { + thread.*.?.join(); + thread.* = null; + } + for (jobs) |job| { + if (job.failure) |err| return err; + if (!std.mem.eql(u16, solo.values, job.rows.?.values)) return error.ConcurrentCaptureDiffers; + } + try wait(init.io, &box.done); + box.mutex.lockUncancelable(init.io); + defer box.mutex.unlock(init.io); + if (box.bad.load(.acquire) or box.reason != .length or !std.mem.eql(u32, &expected, box.tokens.items)) return error.ChatReplyChanged; + std.debug.print("queued hidden rows: 3 concurrent captures equal solo; 64 chat tokens equal CLI; ANE={}\n", .{m.ane != null}); +}