From 5d6a0abff2e3e1db90233ba6040aeac99ff0e1e1 Mon Sep 17 00:00:00 2001 From: chaog992 <338238273+chaog992@users.noreply.github.com> Date: Fri, 16 Oct 2026 08:08:45 +0800 Subject: [PATCH 1/8] fix(lanes): a pipelined stream adopted between rounds starts in verify A stream adopted between rounds holds neither a mode nor a queued step: detach drops a verify or exit stream's mode, and a stream another driver opened never had one. pipelinedRound read the missing mode as pipe and unwrapped a step that was never queued; it now starts such a stream as verify, from its pending token. --- zig/src/core/lanes/engine.zig | 3 +- zig/src/core/lanes/engine_test.zig | 95 +++++++++++++++++++++++++++++- 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/zig/src/core/lanes/engine.zig b/zig/src/core/lanes/engine.zig index cf0a97ea4a..e0bc43a1de 100644 --- a/zig/src/core/lanes/engine.zig +++ b/zig/src/core/lanes/engine.zig @@ -470,7 +470,8 @@ pub const Engine = struct { /// One token, the next forward queued first; a copied continuation ahead switches to verify windows. fn pipelinedRound(e: *Engine, s: *Stream) !Result { const before = s.mode; - const mode = s.mode orelse .pipe; + // an adopted stream holds neither a mode nor a queued step: it starts where a drain step leaves one + const mode: sm.Mode = s.mode orelse if (s.inflight == null) .verify else .pipe; var result: Result = .{ .rows = 1, .keep = 1, .got = &.{} }; if (mode == .verify or mode == .exit) { const proposal = if (mode == .verify and e.cfg.family_width >= 2) try win.copyProposal(e, s) else &.{}; diff --git a/zig/src/core/lanes/engine_test.zig b/zig/src/core/lanes/engine_test.zig index 2100cc3762..bb32189fa3 100644 --- a/zig/src/core/lanes/engine_test.zig +++ b/zig/src/core/lanes/engine_test.zig @@ -88,9 +88,14 @@ const Classes = struct { }; fn model() !Config { + return modelWith(true); +} + +/// The test family drafting with its head (`mtp`) or by copies alone, when drafted streams step pipelined. +fn modelWith(mtp: bool) !Config { var costs: [16]@import("config.zig").Cost = undefined; for (&costs, 1..) |*c, w| c.* = .{ .width = @intCast(w), .ms = 5.0 + 0.8 * @as(f64, @floatFromInt(w)) }; - return Config.init(gpa, .{ .exact_width = 16, .gpu_tokens = true, .mtp = true, .speculate = true, .speculate_early = false, .drafts = 4, .window_costs = &costs, .mtp_step_ms = 0.5, .hidden_rows = true, .batch_rows = 32, .max_streams = 8, .draft_streams = true }, 16, 15); + return Config.init(gpa, .{ .exact_width = 16, .gpu_tokens = true, .mtp = mtp, .speculate = true, .speculate_early = false, .drafts = 4, .window_costs = &costs, .mtp_step_ms = 0.5, .hidden_rows = true, .batch_rows = 32, .max_streams = 8, .draft_streams = true }, 16, 15); } /// Every case's emitted tokens, the cases admitted together and stepped until done. @@ -440,6 +445,94 @@ test "structured output: where the grammar spells its own text, its drafts fill } } +/// Each token the fake target draws one at a time after `prompt`, as `got` must hold them. +fn expectDecode(prompt: []const u32, got: []const u32) !void { + var history: std.ArrayList(u32) = .empty; + defer history.deinit(gpa); + try history.appendSlice(gpa, prompt); + for (got) |t| { + try std.testing.expectEqual(fake.next(history.items, null, history.items.len), t); + try history.append(gpa, t); + } +} + +test "a lone stream handed to another driver between rounds and back replies as if kept, pipelined or windowed" { + // a plain stream left in verify by a shared round, a windowed drafted one, a copy-drafted one left there by a drain + const Set = struct { cases: []const Case, mtp: bool }; + const sets = [_]Set{ + .{ .cases = &.{ .{ .prompt = &p1, .drafts = false }, .{ .prompt = &p2, .max_new = 6 } }, .mtp = true }, + .{ .cases = &.{.{ .prompt = &p2 }}, .mtp = true }, + .{ .cases = &.{.{ .prompt = &p2 }}, .mtp = false }, + }; + for (sets) |set| { + var cfg = try modelWith(set.mtp); + defer cfg.deinit(gpa); + var target: fake.Fake = .{ .gpa = gpa }; + defer target.deinit(); + var clock: fake.FixedClock = .{}; + var engine = Engine.init(gpa, &cfg, target.backend(), clock.clock()); + defer engine.deinit(); + var proposers: [2]SuffixLookup = undefined; + var streams: [2]sm.Stream = undefined; + for (set.cases, proposers[0..set.cases.len], streams[0..set.cases.len]) |c, *p, *s| { + p.* = try SuffixLookup.init(gpa, .{ .min_match = 4 }); + s.* = try sm.Stream.init(gpa, .{ + .id = "s", + .prompt = c.prompt, + .max_new = c.max_new, + .eos = c.eos, + .drafts = c.drafts, + .proposer = p.proposer(), + }); + } + defer for (proposers[0..set.cases.len], streams[0..set.cases.len]) |*p, *s| { + s.deinit(gpa); + p.deinit(); + }; + for (streams[0..set.cases.len]) |*s| try engine.addStream(s); + var handed: usize = 0; + while (engine.activeCount() > 0) { + try engine.step(); + if (engine.live.items.len != 1) continue; + const s = engine.live.items[0]; + if (!engine.detach(s)) continue; + handed += 1; + try engine.adopt(s); + } + try std.testing.expect(handed > 0); + for (set.cases, streams[0..set.cases.len]) |c, *s| try expectDecode(c.prompt, s.emitted()); + } +} + +test "a stream another driver opened and hands to the lane core replies as its one-token decode" { + for ([_]bool{ true, false }) |mtp| { + var cfg = try modelWith(mtp); + defer cfg.deinit(gpa); + var target: fake.Fake = .{ .gpa = gpa }; + defer target.deinit(); + var clock: fake.FixedClock = .{}; + var engine = Engine.init(gpa, &cfg, target.backend(), clock.clock()); + defer engine.deinit(); + var p = try SuffixLookup.init(gpa, .{ .min_match = 4 }); + defer p.deinit(); + var s = try sm.Stream.init(gpa, .{ + .id = "s", + .prompt = &p2, + .max_new = 40, + .eos = &.{96}, + .proposer = p.proposer(), + }); + defer s.deinit(gpa); + // as a lone driver hands a stream over after its first token (lane_host_lone_test.zig's Lone) + _ = try target.backend().opening(gpa, &s); + s.pending = s.context.items[s.context.items.len - 1]; + s.cache_len = s.prompt_len; + try engine.adopt(&s); + while (engine.activeCount() > 0) try engine.step(); + try expectDecode(&p2, s.emitted()); + } +} + test "a family that keeps copies to shared rounds proposes none for a lone stream" { for ([_]bool{ true, false }) |lone_copies| { var cfg = try Config.init(gpa, .{ .exact_width = 16, .mtp = true, .speculate = true, .drafts = 4, .lone_copies = lone_copies }, 16, 15); From 2bc79cbb613f31049aab305cbe0b23ea438d397e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schr=C3=B6der?= <1188270+cwschroeder@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:49:02 +0200 Subject: [PATCH 2/8] Validate the separate Clef head checkpoint against its backbone width --- THIRD_PARTY_NOTICES.md | 9 ++ build.zig | 9 ++ zig/clef_checkpoint_cases.zig | 4 + zig/src/families/clef/weights.zig | 114 ++++++++++++++++ zig/src/families/clef/weights_test.zig | 127 ++++++++++++++++++ .../fixtures/clef/joint_head.safetensors | Bin 0 -> 22840 bytes .../fixtures/clef/joint_head_config.json | 1 + zig/tests/fixtures/clef/source.json | 8 ++ 8 files changed, 272 insertions(+) create mode 100644 zig/clef_checkpoint_cases.zig create mode 100644 zig/src/families/clef/weights.zig create mode 100644 zig/src/families/clef/weights_test.zig create mode 100644 zig/tests/fixtures/clef/joint_head.safetensors create mode 100644 zig/tests/fixtures/clef/joint_head_config.json create mode 100644 zig/tests/fixtures/clef/source.json diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 74546d20cd..bad38d8ccb 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -232,3 +232,12 @@ TensorFold. What they follow or copy: - The Level Zero loader and the Intel compute runtime are loaded at run time and are not bundled; `ocloc` compiles the kernels at build time and is not distributed. +## Clef head checkpoint + +`zig/src/families/clef/weights.zig` reads the separate bf16 head format from Cloudflare's +[`joint_schema_model.py`](https://huggingface.co/Cloudflare/clef-flash/blob/8b2e5fd17c09fd49bc2880b805d5323ec7ff4ff3/joint_schema_model.py), +Apache License 2.0. The earlier TensorFold work by @outcastofmusic in +[#241](https://github.com/ashhart/TensorFold/pull/241) and +[#251](https://github.com/ashhart/TensorFold/pull/251) established the separate head and backbone interface. +`zig/tests/fixtures/clef/` contains a small synthetic head generated with the release code at seed 317. +Its source metadata identifies the release revision. It contains no trained model weights or agreement corpus. diff --git a/build.zig b/build.zig index 380a38b3de..c31e44b2cb 100644 --- a/build.zig +++ b/build.zig @@ -161,6 +161,15 @@ pub fn build(b: *std.Build) void { b.step("test-cost-rule", "CPU scripted window and interleaved head timing").dependOn(&cost_run.step); test_step.dependOn(&cost_run.step); merge_build.targets(b, target, optimize); // tf-merge-check and test-merge, host only on every OS + const clef_head = b.addTest(.{ .root_module = b.createModule(.{ + .root_source_file = b.path("zig/clef_checkpoint_cases.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + }) }); + const clef_head_run = b.addRunArtifact(clef_head); + b.step("test-clef-checkpoint", "CPU-only Clef head checkpoint admission").dependOn(&clef_head_run.step); + test_step.dependOn(&clef_head_run.step); switch (target.result.os.tag) { .macos => metalTargets(b, target, optimize, draft_ids, build_options, test_step), .linux => cuda_build.targets(b, target, optimize, draft_ids, build_options), diff --git a/zig/clef_checkpoint_cases.zig b/zig/clef_checkpoint_cases.zig new file mode 100644 index 0000000000..0fb3e5d777 --- /dev/null +++ b/zig/clef_checkpoint_cases.zig @@ -0,0 +1,4 @@ +//! CPU-only admission of the separate Clef head; no model, device or scoring arithmetic. +test { + _ = @import("src/families/clef/weights.zig"); +} diff --git a/zig/src/families/clef/weights.zig b/zig/src/families/clef/weights.zig new file mode 100644 index 0000000000..fa265d0e3c --- /dev/null +++ b/zig/src/families/clef/weights.zig @@ -0,0 +1,114 @@ +//! The joint head owns a read-only bf16 safetensors map and validates every trained tensor. +const std = @import("std"); +const safetensors = @import("../../core/safetensors.zig"); +const convert = @import("../../core/quant/convert.zig"); +const Tensor = safetensors.Tensor; +const A = std.mem.Allocator; + +pub const Config = struct { + hidden_size: usize, + width: usize, + routing_layers: usize, + layers: usize, + heads: usize, + feedforward: usize, + dropout: f32 = 0, + pub fn validate(c: Config) !void { + if (c.hidden_size == 0 or c.hidden_size > 16384 or c.width == 0 or c.width > 4096 or c.heads == 0 or c.width % c.heads != 0 or c.feedforward == 0 or c.feedforward > 16384 or c.routing_layers > 16 or c.layers > 16 or c.dropout != 0) return error.InvalidHeadConfig; + } +}; + +pub const Weights = struct { + config: Config, + file: safetensors.File, + pub fn load(a: A, io: std.Io, dir: []const u8, backbone_hidden: usize) !Weights { + const config_path = try std.fs.path.join(a, &.{ dir, "joint_head_config.json" }); + defer a.free(config_path); + const text = try std.Io.Dir.cwd().readFileAlloc(io, config_path, a, .limited(1 << 20)); + defer a.free(text); + const parsed = try std.json.parseFromSlice(Config, a, text, .{}); + defer parsed.deinit(); + try parsed.value.validate(); + if (parsed.value.hidden_size != backbone_hidden) return error.IncompatibleHeadBackbone; + const weight_path = try std.fs.path.join(a, &.{ dir, "joint_head.safetensors" }); + defer a.free(weight_path); + var out = Weights{ .config = parsed.value, .file = try safetensors.File.open(a, io, weight_path) }; + errdefer out.file.close(io); + try out.validate(); + return out; + } + pub fn deinit(w: *Weights, io: std.Io) void { + w.file.close(io); + } + pub fn get(w: *const Weights, name: []const u8) Tensor { + return w.file.get(name).?; + } + pub fn named(w: *const Weights, prefix: []const u8, suffix: []const u8) Tensor { + var buf: [192]u8 = undefined; + return w.get(std.fmt.bufPrint(&buf, "{s}.{s}", .{ prefix, suffix }) catch unreachable); + } + fn tensor(w: *const Weights, prefix: []const u8, suffix: []const u8, shape: []const usize) !void { + var buf: [192]u8 = undefined; + const t = w.file.get(try std.fmt.bufPrint(&buf, "{s}{s}{s}", .{ prefix, if (suffix.len == 0) "" else ".", suffix })) orelse return error.MissingHeadWeight; + if (!t.is(.bf16, shape)) return error.InvalidHeadWeight; + for (0..t.numel()) |i| if (!std.math.isFinite(convert.load(t.dtype, t.bytes, i))) return error.NonfiniteHeadWeight; + } + fn norm(w: *const Weights, prefix: []const u8, n: usize) !void { + try w.tensor(prefix, "weight", &.{n}); + try w.tensor(prefix, "bias", &.{n}); + } + fn linear(w: *const Weights, prefix: []const u8, n: usize, k: usize, bias: bool) !void { + try w.tensor(prefix, "weight", &.{ n, k }); + if (bias) try w.tensor(prefix, "bias", &.{n}); + } + fn attention(w: *const Weights, prefix: []const u8) !void { + const d = w.config.width; + try w.tensor(prefix, "in_proj_weight", &.{ 3 * d, d }); + try w.tensor(prefix, "in_proj_bias", &.{3 * d}); + var buf: [192]u8 = undefined; + try w.linear(try std.fmt.bufPrint(&buf, "{s}.out_proj", .{prefix}), d, d, true); + } + pub fn validate(w: *const Weights) !void { + const c = w.config; + try c.validate(); + try w.norm("hidden_norm", c.hidden_size); + for ([_][]const u8{ "memory_projection", "question_projection", "option_question_projection", "global_projection", "option_context_projection", "option_lexical_projection" }) |name| try w.linear(name, c.width, c.hidden_size, false); + try w.tensor("type_embedding", "weight", &.{ 3, c.width }); + var buf: [192]u8 = undefined; + for (0..c.routing_layers) |i| { + for ([_][]const u8{ "query_norm", "memory_norm", "feedforward_norm" }) |name| try w.norm(try std.fmt.bufPrint(&buf, "evidence_layers.{d}.{s}", .{ i, name }), c.width); + try w.attention(try std.fmt.bufPrint(&buf, "evidence_layers.{d}.attention", .{i})); + try w.linear(try std.fmt.bufPrint(&buf, "evidence_layers.{d}.feedforward.0", .{i}), c.feedforward, c.width, true); + try w.linear(try std.fmt.bufPrint(&buf, "evidence_layers.{d}.feedforward.3", .{i}), c.width, c.feedforward, true); + } + try w.norm("option_summary_norm", c.width); + for (0..c.layers) |i| { + for ([_][]const u8{ "self_attn", "multihead_attn" }) |name| try w.attention(try std.fmt.bufPrint(&buf, "layers.{d}.{s}", .{ i, name })); + for ([_][]const u8{ "norm1", "norm2", "norm3" }) |name| try w.norm(try std.fmt.bufPrint(&buf, "layers.{d}.{s}", .{ i, name }), c.width); + try w.linear(try std.fmt.bufPrint(&buf, "layers.{d}.linear1", .{i}), c.feedforward, c.width, true); + try w.linear(try std.fmt.bufPrint(&buf, "layers.{d}.linear2", .{i}), c.width, c.feedforward, true); + } + try w.norm("field_norm", c.width); + try w.norm("option_norm", c.width); + try w.linear("residual_scorer.0", c.width, 4 * c.width, true); + try w.linear("residual_scorer.3", 1, c.width, true); + for ([_][]const u8{ "prior_logit_scale", "joint_logit_scale", "residual_gate" }) |name| try w.tensor(name, "", &.{}); + const expected = 22 + c.routing_layers * 14 + c.layers * 18; + if (w.file.names.count() != expected) return error.UnexpectedHeadWeight; + } +}; + +test "head configuration refuses invalid attention geometry and training dropout" { + const c = Config{ .hidden_size = 16, .width = 8, .routing_layers = 2, .layers = 4, .heads = 2, .feedforward = 16 }; + try c.validate(); + var bad = c; + bad.heads = 3; + try std.testing.expectError(error.InvalidHeadConfig, bad.validate()); + bad = c; + bad.dropout = 0.1; + try std.testing.expectError(error.InvalidHeadConfig, bad.validate()); +} + +test { + _ = @import("weights_test.zig"); +} diff --git a/zig/src/families/clef/weights_test.zig b/zig/src/families/clef/weights_test.zig new file mode 100644 index 0000000000..276608c20e --- /dev/null +++ b/zig/src/families/clef/weights_test.zig @@ -0,0 +1,127 @@ +//! A small seeded release head tests the complete tensor graph and every load refusal without a device. +const std = @import("std"); +const Weights = @import("weights.zig").Weights; +const a = std.testing.allocator; +const io = std.testing.io; +const config = @embedFile("../../../tests/fixtures/clef/joint_head_config.json"); +const fixture = @embedFile("../../../tests/fixtures/clef/joint_head.safetensors"); + +fn write(tmp: std.testing.TmpDir, bytes: []const u8) !void { + try tmp.dir.writeFile(io, .{ .sub_path = "joint_head_config.json", .data = config }); + try tmp.dir.writeFile(io, .{ .sub_path = "joint_head.safetensors", .data = bytes }); +} + +fn changed(needle: []const u8, replacement: []const u8) ![]u8 { + const n: usize = @intCast(std.mem.readInt(u64, fixture[0..8], .little)); + try std.testing.expect(std.mem.indexOf(u8, fixture[8..][0..n], needle) != null); + const header = try std.mem.replaceOwned(u8, a, fixture[8..][0..n], needle, replacement); + defer a.free(header); + const padded = std.mem.alignForward(usize, header.len, 8); + const out = try a.alloc(u8, 8 + padded + fixture.len - 8 - n); + std.mem.writeInt(u64, out[0..8], padded, .little); + @memcpy(out[8..][0..header.len], header); + @memset(out[8 + header.len .. 8 + padded], ' '); + @memcpy(out[8 + padded ..], fixture[8 + n ..]); + return out; +} + +test "the separate bf16 head validates every tensor and matches its backbone width" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try write(tmp, fixture); + const dir = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(dir); + var w = try Weights.load(a, io, dir, 16); + defer w.deinit(io); + try std.testing.expectEqual(@as(usize, 122), w.file.names.count()); + try std.testing.expectEqual(@as(usize, 16), w.config.hidden_size); + try std.testing.expectError(error.IncompatibleHeadBackbone, Weights.load(a, io, dir, 5120)); +} + +test "the trained head refuses missing tensors, changed ranks and a changed storage dtype" { + for ([_]struct { from: []const u8, to: []const u8, err: anyerror }{ + .{ .from = "prior_logit_scale", .to = "other_logit_scale", .err = error.MissingHeadWeight }, + .{ .from = "\"shape\":[16]", .to = "\"shape\":[8,2]", .err = error.InvalidHeadWeight }, + .{ .from = "\"BF16\"", .to = "\"F16\"", .err = error.InvalidHeadWeight }, + }) |change| { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const bytes = try changed(change.from, change.to); + defer a.free(bytes); + try write(tmp, bytes); + const dir = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(dir); + try std.testing.expectError(change.err, Weights.load(a, io, dir, 16)); + } +} + +test "a nonfinite trained weight is refused before any head execution" { + for ([_]u16{ 0x7f80, 0xff80, 0x7fc0 }) |value| { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const bytes = try a.dupe(u8, fixture); + defer a.free(bytes); + const n: usize = @intCast(std.mem.readInt(u64, bytes[0..8], .little)); + std.mem.writeInt(u16, bytes[8 + n ..][0..2], value, .little); + try write(tmp, bytes); + const dir = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(dir); + try std.testing.expectError(error.NonfiniteHeadWeight, Weights.load(a, io, dir, 16)); + } +} + +test "an unrelated tensor is refused even when all required trained tensors are present" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const n: usize = @intCast(std.mem.readInt(u64, fixture[0..8], .little)); + const end = fixture.len - 8 - n; + const prefix = try std.fmt.allocPrint(a, "{{\"unused\":{{\"dtype\":\"BF16\",\"shape\":[1],\"data_offsets\":[{d},{d}]}},\"evidence_layers.0.attention.in_proj_bias\"", .{ end, end + 2 }); + defer a.free(prefix); + const header = try changed("{\"evidence_layers.0.attention.in_proj_bias\"", prefix); + defer a.free(header); + const bytes = try a.alloc(u8, header.len + 2); + defer a.free(bytes); + @memcpy(bytes[0..header.len], header); + std.mem.writeInt(u16, bytes[header.len..][0..2], 0x3f80, .little); + try write(tmp, bytes); + const dir = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(dir); + try std.testing.expectError(error.UnexpectedHeadWeight, Weights.load(a, io, dir, 16)); +} + +test "a truncated head payload is refused" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try write(tmp, fixture[0 .. fixture.len - 1]); + const dir = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(dir); + if (Weights.load(a, io, dir, 16)) |raw| { + var w = raw; + defer w.deinit(io); + return error.TruncatedHeadAccepted; + } else |_| {} +} + +test "head backbone compatibility is checked before opening its payload" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.writeFile(io, .{ .sub_path = "joint_head_config.json", .data = config }); + const dir = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(dir); + try std.testing.expectError(error.IncompatibleHeadBackbone, Weights.load(a, io, dir, 5120)); + try std.testing.expectError(error.FileNotFound, Weights.load(a, io, dir, 16)); +} + +fn loadClose(gpa: std.mem.Allocator, dir: []const u8) !void { + var w = try Weights.load(gpa, io, dir, 16); + w.deinit(io); +} + +test "a failed allocation during a head load leaves no live allocation behind" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try write(tmp, fixture); + const dir = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(dir); + try std.testing.checkAllAllocationFailures(a, loadClose, .{dir}); +} diff --git a/zig/tests/fixtures/clef/joint_head.safetensors b/zig/tests/fixtures/clef/joint_head.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..38146c7c5662440bda92ff21980e2eed29210ddf GIT binary patch literal 22840 zcmb_i1$b0v)5RT%OR)l_P*_6T_Rio?oMOcxK!6|#5!{0X2<{L_kPzH8duM`sahKAs z?q6NL_HX}lNlV#e>9(QmQ=TU5p8LM<%$##(-rJ>Cw#WaBs~S2otbb^DztG-8L&k(g zMSFO9gv7*zhR1|OgnNXA_Z}VANK8oYhyeqlLu2sS?w&T^z@8Iqe+zz8XxP9(F@O00TVUo#_y*d1eZ1X0 zBw|F&<4t(D*@UYH_;}lV-EZT`6Ws12(B(F;55E9^*ZUX{8rpwAMAWE|sQ!qIn=N?x zXFen#+x$JB5ejE?x6AkPceQ??+XDT4-LK!r-TGaG0Km4uK+nGe?soZsHdhA(beor# zkNflQ9Uc)iEDKw(!0`mXXJqPAF-U)c`+a1NvIUQ4_&+1g!$OBeM2*S%2K;e!UV+aD z@zda0TYI3d&CA>K89|O55&9fU2Q07=cb8Q36n2(Y4q%|W<{+pK`=It5eGIO1ukriIGkZs;xUM@)~BfQ%!xMo%h-sbJ&nv|TL zkyU20!0`mv%;e$;?l+RTFf4eRx36n@^743^{r-7HShxUp%}>tYSz0@g0skz5dW!zC zvUCuW_cQE~m&a4sSz9?=k+*L;9SsNz9oj$3IRX$MrIXH6NLgDIh{fACy(k9`jp!RP z^s%=L?f2L&yN7h?KrBF;cR;#97!=kYcKmN`)|H#FV1Sig5l>^eU7Kq(EB6D_({pe{ zSa?kDp%DYaVtPmS3mFRM|ICiSMHvCbf_m&wp0pZ6!@@&DqP*O>Dwk^m76APolW-=$ zzp-6cDrn)^e7rm#vu*}ZZ+EM7p?DS)@Vp<>Y$iOni*(Ja7M{(=C&2kAhm9B-6E-L` zq(g*L!$^?{*U| zZH5JJ^9k~M?2DfS*Xk17CbX+m0T9pvoKGYj&Hef^7n}tJyg=tDT6kGXWB}P9R?dl> zj+T|xSx~^sKnE6HR@{sQ1T>$_k?`@z+Uk6KftP_>qC5b}F}L z;=A30>qNHTZGQev1d-_(Pg3rmXR`$j{J-AA6MVOuaGBB;yfxZ;x){AY(kVR)D+UnI zoMitGH23SvED#nH@PeFEIjaI;0onYV1wDOr-dUnx3krA{=pd^CVF3ZnH*+MiCJ+`B z@G@}ApEUa%gL4bj<`?8-LA^XOL%R+6GwKS+Hh=FRXTtdt{z=OJ^8~d}ZT>#bT=^4d zw@Y_fO%}4v-`A;Py`M)QEGz)}KkbnI1a!Md*8*YT+57{X9RDAnSrG^e3V1$GtJt4- zZWrlVAS^tae~?q@d;g09VL{shJe|tznfPw^;5v~lcw2yTmy+ojPg3rmXR`$j{J-AA z6MVOuaGBB;ye+`b$@6<>Ss*MRpgGC@A878^msubzC|iKPljl#z%Ss|!KtOX6^gqzD zv^q<`1Dpjs9WN^#gargNC&&K>TGmzvn;+nxfm{Bh+2;@l3)L17;ABC)|2=`QkZpm^ z4a(E-Pg4G$mz0HS3-o&C%AY{HKXF&X8w=SM=lo_0uo0=iwKYk{!v zY=M4Gj{gtPtkjhS1w7xURqRhZw~KTw5Eh21v!uUe6lPM77);!Wd9E|_v_0n5Ehgz z$k%CzmyVZ}M7Dr{=H&SQK+DqVECCO47VvbutO$e!1T-hd{|8#uRtK9OljN;8`-^<@NiW z&+iY29x-fK$aB1QV!>IU>6z>bG)sqJ0a~!>-Qe)3u!yKEU&uX%vB1(_WB50Z)WQU^ zPx?J+RA_Wq{}IU4fgv&f?9T8p#ABex-R~2i=zbAVp;7MN-EeWsVnM?4c=@KA(9DqT z<6^^gdZyPMA0q-cC|$lX;<{BFFISKNdkpDv_kUw><>iV5tu>WC+ zZ)pGiVc`SakLrA$f$57kd%QrfJts_f3IC+OUHbR`TxFD_(MDt4#LIcVn0kN@@dGya zJTKG~3gR1Fn#XCBG}ni8lV|G7nxN}tfgYlU9M9V{#i*<$=!EWL=4KqiUr1qHsQV}% zk1;0eaq6VaHC0k|i`kXWh_{iKx2V5f+p(=?QG@(DS`leH>0&}-aIzLkU0lq>lkuO8tgnqO9ED%aLCTv)qsK7EmAb51R)&FOtQuZi?M)#n|& zNIUU7Jt1THOmJCxASZPlcf-%4be1mWT)az?DO7goVWXYYpQOuJ8(+@H;ZZte`%+m-lPNNafQwjZ(WVNCR$yGr2}>xtGq+N?MX<@h3Egx9cR?N3GS)XQ?kDk)-wX z9Btt;noL6dwHTMxbvlPvBgWl1mg9I3NAiB!&iN&eY}KuZ@NT^-+l>RQl*gRK$vjF|V3qmxBy~sB56fqUk8a>t zV$Q1#b-ga(1zLl5(los&t&tnGb&9m1s=AVGT3I`5A1$uCG>8{)X?-ZIbhF0l72|7K zsc|$>%j=Hd@8nZSGGnx@4(D9vJC1Q$9eeX6J7Up`&yW{#WvFr4=%e8pfPaf<4Bawk zP!;nY`HEe7@n(tP6vtT^N~JVZQ8h+(!wb(Xugy4=D{^adqdcH5r8wu;Dzt&_nz2ST z{nlvBb$O?FavSZ)bHrCmAm&lBf}(gfR@#a0k_T7R9MXr1>SW1*-;LxH`q><=^|%y2 zk|KOZYO~N8isn#VC0me3Lv*^lZ4|(I8gqX-M_-%ChQGv872ZKPr6_-g^|r>@jN#n; zkOtCkWW!F@(CeA3K{^WRRc_fP*m%~&A`2w5Bzm{BwzmfCy1suUdw=cuQj%0n=r$@8(t?B)obYMxSup3raT1&YL45A$aJkx-|U%)a!K z;{x_}PnYXuc7O-->#xQl@Z&6<%_q#6GRG(^iPD9Cl{8uaPWhE4^Fw`s=jvT`@Fks~ z6S2FK`jedC_IggM^GNBUd%*qIv?p>gV>}Xcps__gIh>Pau{O}n<~p3wDs8H*b(7Z7 zBea31^N-R-JDBw-FYnZ@yv!VrO83S8P3GeKI)6*Q80%yw_8g{FX%F8r--Wj96nJdXUXh2?>D+%DUhP;y($^@NB zg?TM%eL7Fq%CeQ$AmeBAMyO#!+0Q@IZV6W({+?ds`chx&ssr`dU4NlO`9h{Lb9LRQ zrFEiAw=Y4I2J#zV&wNHD9;;ooDK7?t6qXX)gGZ5%(T1z&0dpA55-^P}Vmo)I6p7?y z?VuZZ6ZSb=NApau*($B==!VnTr?Ha9*w0J38}|oCO~ihx(tK!YL8&b-;j?~Ho#%t8 zwrX}W7)%+#%ka!5nrQw)TcHRuqyxCVv&_Y33qhYZ=?wnLoJiTBeTKBv0#aU+WB`XD zuKC%nwX~R4m-+UOWwFtb^Kb#(V6TWzbmDB+OMD@AD= z8bCRCJU52x7^5ZiBkDu%k?B}PlejC_(f27vn$Qpp(IHxrj*u7dw#k1b5vm&oovERQ zz6{nK#jlv9wGxGMRc@+x%!>ReRiIIbR0;Y@ZW`^hiheBBand)awhrPx8i`Zu#glLb zv63-=pO>%x>G!{&4<(%UIx6u6I!r%PLG_~I`iN}&IptGdIHFh8TfeqnreYk(^>Ky) zw9GLRSvpyYNpEmm39v(U9xCsfJ^8Fri+ghpZc1;_WxGRG)29-xK{^7t8V%K{sS(_k zeYrWbx-FEqJx-=R6tJ*HN=wvCZ^y6HLkqHLf1j%9|EM!9mzmTNl^U<#K`J>^Ky*|c1*(yugB zHgFnFa-K$W1wCfYf_qESFJ!EK0G=tKGc_M}*d9JMJMw=%#~6F15PX{-7^eh0P&;_S z*YJ&5@;|e(9N<(+0M|A%%fY>zplK4Ld5t=7S?`5BcCtu6Eia|HvYs=m zf_VaYj+vV$@OrAH(^0SU=(E%ptyptQsP|^A0JmI(?@A^9Tw=jK!+9?61tZ292{N7r zJGx4kR@ePdHZN4)Jb296Mit#46(v|(>NKt(8=*#fWiuE6-pW20+!qQLxKQp$oHRzx zw5C#0MO8X!OMXO`=(_opl%k*H5N)IWMSGC5+$T5mLxen8U#&qxl=UA}8caRC!L0;xwZv&N|hcDt4`leFY<*Vt6$8 zq%UfFGMv~%eFYhwlWOqm#tpL%6rer$VId8I65atH?}U3vl$EeHEYAoXEvw8m_g#8~MS+jd?Cj(h@j>uPNEgsauU%MhAztt}%>Eij^_rGX5NE zj$wj2Bx(|LunL^=2`Jg;(1ihdTCamwFM&%ZYXKT(8usa02i}o6hwh=JVDPSbh=<5Y zxapbja1;47`kM8|QC(ry(G($aP0@UJ^-ocYkz(K5sCo3r_G}|5r zWlhzKI)!p*TRw%3}4aeR>nDIh9Mu z6gb92DW?bZFfXIK_M`fS+eT_=l>}37?rKm*kEnvp0OwSmal*zMwZWjjqAb z9i~ov9GQ6En5$>NXf@D7y=JVZe>hjK}b)nfkdAk9>~PTY6NNa5J-+ zq|#hn0gu-T{roxOBG={XZ1N4xCX?wEGYQ^nBwTwlc-hU!*)e*G=hKW#`7nv<vI|*ITbA=gy5$noY(--R8khZ@|f|dE5yFn6tPD^r#M(;49yO zyqtm(cGWvnOv~{$o&q<%g^QZy!Trn8`zB)|5X=wkE5%>y>v*k8$>6|fzG}yWL3?vS zUBzwXeWRZQ%iHq4WYhN0%`aswDsn26bGq)(X%eVw;qMwD%A0thxsG=PFQQ!{T4;IPgCA;>*xzD8L^Z~rxSVZ$z z>B%jqrXHkJ3gzpR2O64P-=VLl7S)F~7d9s_X+0jHX=W5}p+@Lu>dLEnM}BC zYT1ExzXKkh!z;ny2hH~Utu&EU<`wv9rf&gAdR zt^5!ZixzxQx9N|R%%$NHeijGx^Z*!R6S|l8p*hEOpB_gi90`5n8fQTGMP@%=_pv+;3y}hc$>{AT+17M8RX^lIlD`<8-c3NhhPSZeU7y zNKff4_B0k?h2I%fu)_k{h0-)p*BY(43eQGozMK{ylaA^^IqmSK-((sEfR9F)$1!tI z^x3QAh@OSEokIPdh7*0un5;qCNmIBe-7~kt^Yqu7+zc!h0ye&kx@<`~p^Ue6Gy0;- z;>~YC=b2A%9K}l+tZ+7Zgwi^dzeKiXj7|@!Br7-wv%Z@4I9&?%XZY0CsIQ4o%4J4T zJw`co7&q2MnAWWUZv|^^qYOIGh1{B7qFB00(`XSowmB592j~=-sX5}gL2rY%3}ZSt zAknxW3*`cwv^78bOzxwPYXCisbogfCzvtCEci+#r|BOFhvQO28;DdFD#zLbFnP9H- z!5#H0slsV+B#D@3oR-S+KsOqV!A!YzAy3w^6hZ50HKq*tF<-Om498qOirl(Qb9o+k zW;`E7hqg!mPx_#`8o)VSfHRBbH#OFATQ8t%PU4!>23P{8RhBmCT})ZJac=aOmknz^ z*^k#C>PeVP4Q0 zwQbY=x(OZoF?fQNsKCAEa^!Y0ZGv_$M&I5V+&&jmu>@X8!_2!zalOqm;mmTu89YF} z4X68#GdxYQ!F%1qbZ84s;XKcw4pJQaosWMsR%Y^R2{aQrH5nSSfG3!{Fu9nG{%3;B z2i_{$hi+%JZq#;Ql4Qp80kyQA3&;$x#wM*maj2RXFyrW=r8FSB zQ@I_OW)2v39vHM7@6s6@!-+aquh@zAadq0x6?K9!PDFOWqgJFbd=N9zcyP~t>BcEw zoyBk}T`^7S#h;NUYHS%gsQ*Ai>!B+d!3DA2O*~$Irdw2uvl*2sgdfr_*<{|7rMyEM z!d+HCJj=s(91H$Q7U*nai7taG&4u^OBgfHe%|WM-j3@4cQ!g%!9QR~Dy2Ym2L*8KY zWZeA^kb z41PZfJ(vx58Z|xGtv=E4l2%_<+525I@J&zWvB%4 zLaL=>u-kQZ1K#cqs&5K4Mn^uHJ5oQ)V{4_B=V0>~-K77dE|5U85obqtv{dd(gfRed z84t#LMGs3osi9}h9p(gilRU8Y%|<^eq)VlZensnLWa?~*2k-xE_R=w6#3JZzUgiO2 zBpW;w&m9J(uA@0K@mWTG$@ue14FE^?yh^f_-(ATQpvorS(&H2Xjy4w!*Z*}LL})%w{df&W_JzHAzN$Za~S_n>qY^kZs>y2Jc?)(dIuCiV8IkCO9`O)rB9IRt*JDxn zJJ31ogX*8dY1!Ztck4mEfJsCdT5M#;TzNAlpO?{vo<_%dTQ5?8smQpGf~_gkZDd{z z*(>Eyclk5LD+x{{0<*9M+Dk+A6Y9l-xexeh8s<0cxv6^aJYGW$dARn&9D5_5!p$Q? zEmoj^{V+97zmyanr>_F#9&Spf@rUTGf}n|&;ds~c7pUD`x)vEYSL4wau9R2wq%`7L zx`mr*3(l2kJ?`}EdH+6RK4$!Rg(h(dmy$FdqL+0yX6z@>FT4rv*^5r=Haz@7o{moY z3Od3A+7ma75BNLUhF)wjx{L($GN<66O6fzyev-~Mn}B;_aC@~>4sjhbkw+@JFsY9I zzPJ&=_n|&}^r6-@w;?yFA9uo()MVCs14f!=BwRLfI)8H2&IrMQcETwd2;bTr%4Uy=*M!ld=}vRSiA5AD<>uR_jOT z_qxhYm~ySrAB<#7X0vfs@C|{>o8WG^tS-iR&w-1wOE?_EMlA+DS}l!nzpYB4;j_HLcKcK#Ho8AYf=Hi|@1WxTe`3$p`LOcjNDUa!iKe)UGe{U|A5WQ^- zMb7zYD!rs%%m2}7D0df3Eozf5{tQC*+K+1@FI(ap9>y@L2KU%n?>iVb$4&GRI=>-! z%OM{hlHIA%`2RZKpz4|*{@`nh)Xmagmtg*!U%#>6qJDhDEUzccm+3ot6;q%>=3pM| zxQ}VzVB~o>%od_`EV`~y@|*bJDN#Cz^YF)JK@FsEysyxMyW@WIKSpT|r;PdQ?&tkF z!QKhGPk@iype@V>ItE%44{aC+-npnhO0q7|)7%C=)-bkUJ`u~F_{}7`q)T`c?lOmJ zm@L%ysG!3WI^PAe;rzN9Cm9J2xdsM5#yL@Sm2gYF)iDU1UQPYztob=~XsO=dQI6@d z7!0wPB2oXTyju&=HT=0)r#hC@wd^XPdRP4uE$GfHF1Z z$>K$)bsy$SaopPQ()ZxkuWM2BlB_`Y8HUgOiuqO&UEohqH%VNS6To`!$}qV2gSs1T zy^-$6T79{S=D@wpH9mzIL^KD{o7knrsh5zs_xJ?e)w7s@#B(9UA`Ou_j&IiI!g>hP zfi!&+H#0l+GyL}g55v!uTBbX|s=>xe9UwK;RdG=ZG}s>F|21 z;IaG>i?(3PdVC!B2qCDI8M;aOa|kps5xlw#+J^T+9oxAU6+<(TO z7r|XDr$TU0yWo?HX?F5Jb}i!T90re8nGQ*uuA~1^6d_J6G~3i`L=P@c75AoxQ}0lMl1wmVvY zQ_IRjdwP`e_}H{5z+U;p!ZyFGCOT0%M%Ef@7FMcDbyR*aJ65`Lz-Fa*(b>4qnH+ zud+0UZ%E~n{Ed;q#o!*YVP|uB8(ioI=sIR=UA`b&!Cd`u27CDfGh;kv>1sLW7^Gr+ zEITlH4Ccl%%UH>gyh`_?Qof@EtxxmK-fG>}kJTBnm0M_6DlfTY3a^(L*vmBa&;rQa z8K|QMaQwM2McJ=QsI-nTldzlaTE`Ks-$^uL+l99neT^t6#XRnY&bK3O(R!gb+(gxN z1z5KO^5YTw@aOO}BWS$NkrJ|j%7Gzsf|I;8Nbbrw9mcu1p1y4l=hrkvhC_>saesbE z+aR`A?f=EJZY;Q9fOfSnkY>o3O3K{FTufn7A1XUh#`0Pzg71vySai=R@X3|oRYss= z9fcZds5Q8jaUDEt-KTYj&+NxFFtOe%b)nO(xQ6jFeApP&aV4yEsnG^3NbqHiRZK6l zC8q5a_#+x4VPN%eeq>GtOO%2K9EeDF0Vj1r>_-~yxP{T563qb0%ggM)P*u2~rf>~k z;@nEYZ4J|oGzgs7QBT3|@4%T{l305cK1m&@8CNhX@?|+L=jeNxN;Pq6qjh(xmp0Hm zoI~Cv52$S(E`vV09(_oK;KcXP1#}XDJU#V*+@NJf8J=cTfwHwmzZNe;p-F>rml271 zL0jC`JTS-dOh+?r#U-JN)%Adkhx`5(^UDa`K}Gb8QBn(OGjwhfxEYu@zf{x;`XA_7 zSIp%4mzQg2{jjp1qz9^G3^&mUy3d#`2OV#jqtU%a>13+IWy}OA&F3Y-_>2xqlqNzA zMxeVMi3~kVQxS{X^p)v}d3Q;8tzK|RzL@V+M@EgrU2VpC%*d|=aQ5A_wiYu!G)Lof zYN^4Mp&$9=1sSa+w6M-JBf$z)_$sQTEqrhR#dJQk1$6w~;9SVcN3xJedr)%?1dDjl zI6foADHpGB+$8I5Qy(_zE6S_ylCNHq%G{aWKu=hh_S1L{#`{K9pesRq()>b@;kaAq zYEHs>2H`V@;A1RD07z?f}5D}B4$@i zhUdsj?8EK2D7Q7C(pX^+ym8V}&qy`?$x%?t%17pI<0G81yYtDGtp?38KRq&<12xW3 z$NsjNTl?x*?!;|)x%8zAMkPHa1TX7>_sjgGqdYR}Y5rg@*+OIBpG$Ku`5y&pGd(YD z`G7H&o1=fIqCZHWbRj?OV9XFh-$1P#lg_x`ds%XE9kBfCWFcOv(sBBw*#unj8BLbCsHn#9`Xy+b7T1^g7T&j+kIj^IgB zM=xW_GF&A`~EvjqkwdXTgaZDVNr< z7nVzoIQtE}8?cdT=xJ2>F8R_d!gksy-@_}Unl<2ZKNo{Ok(=fn)IcrtbjV8keDK2{ zs)p>Wt|_#OK9&QHZ>b9OwWiJh)@}Ay5hl!VB$RL%?`<{-awyep2IsUeJBnm zmqf>Cnp8H6>Rr31eY=#@SM;V7!rjghW4=t0_TtIE(2HP#I_6l~Y|fEh=7)5RuE`~O zo3;~UzIa=5@=>wzX~#b7?1tUMI};^Qmxm~tPD(lT)y0NkT5q=ca4U(&^NN`TaYn7| z=Zpu`pYCCI!A3QDXn$S$88hksBp02aU{v>gs*4_?uoTsnw8X40KZAh}QIgqSzN2@g zfTGg|_gAJ%h*EJrfSL9TN|1*#)A6P~FSw;KSfwyul Date: Fri, 16 Oct 2026 10:35:12 +0200 Subject: [PATCH 3/8] Add disposable normalized rows and complete ANE handoffs on failure --- build.zig | 2 +- zig/build/qwen27.zig | 1 + zig/clef_checkpoint_cases.zig | 4 +- zig/src/families/clef/README.md | 22 ++ zig/src/families/qwen27/ane_handoff.zig | 207 +++++++++++++++++++ zig/src/families/qwen27/ane_mlp.zig | 116 ++++++++--- zig/src/families/qwen27/decode_round.zig | 3 + zig/src/families/qwen27/forward.zig | 6 +- zig/src/families/qwen27/hidden_rows.zig | 61 ++++++ zig/src/families/qwen27/hidden_rows_plan.zig | 39 ++++ zig/src/families/qwen27/qwen27.zig | 1 + zig/tests/qwen27/hidden_rows.zig | 136 ++++++++++++ zig/tests/qwen27/hidden_rows_cancel.zig | 82 ++++++++ 13 files changed, 648 insertions(+), 32 deletions(-) create mode 100644 zig/src/families/clef/README.md create mode 100644 zig/src/families/qwen27/ane_handoff.zig create mode 100644 zig/src/families/qwen27/hidden_rows.zig create mode 100644 zig/src/families/qwen27/hidden_rows_plan.zig create mode 100644 zig/tests/qwen27/hidden_rows.zig create mode 100644 zig/tests/qwen27/hidden_rows_cancel.zig diff --git a/build.zig b/build.zig index c31e44b2cb..c5039618d7 100644 --- a/build.zig +++ b/build.zig @@ -168,7 +168,7 @@ pub fn build(b: *std.Build) void { .link_libc = true, }) }); const clef_head_run = b.addRunArtifact(clef_head); - b.step("test-clef-checkpoint", "CPU-only Clef head checkpoint admission").dependOn(&clef_head_run.step); + b.step("test-clef-checkpoint", "CPU Clef checkpoint and disposable-pass contracts").dependOn(&clef_head_run.step); test_step.dependOn(&clef_head_run.step); switch (target.result.os.tag) { .macos => metalTargets(b, target, optimize, draft_ids, build_options, test_step), diff --git a/zig/build/qwen27.zig b/zig/build/qwen27.zig index 842e0073b2..3c44529b61 100644 --- a/zig/build/qwen27.zig +++ b/zig/build/qwen27.zig @@ -111,6 +111,7 @@ pub fn targets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.bu _ = program(b, "tf-qwen27-dflash-run", "Build the explicit draft prototype and its plain comparison on real weights", "zig/tests/qwen27/dflash_run.zig", target, true, family); const engine_only: []const std.Build.Module.Import = &.{ .{ .name = "tensorfold", .module = engine }, .{ .name = "metal", .module = metal } }; _ = program(b, "tf-qwen27-synthetic", "Build bounded synthetic native hybrid chunk checks, no models", "zig/tests/qwen27/synthetic.zig", target, false, engine_only); + _ = program(b, "tf-qwen27-hidden-rows-check", "Build disposable normalized-row and chat/resume checks without running a GPU", "zig/tests/qwen27/hidden_rows.zig", target, false, engine_only); _ = program(b, "tf-qwen27-dflash-synthetic", "Build model-free native draft context/forward/tree check", "zig/tests/qwen27/dflash_runtime.zig", target, false, engine_only); _ = program(b, "tf-qwen27-widths", "Build the projections at every oQ width before tensor units: float64, row windows, mixed stacks, embedding", "zig/tests/qwen27/widths.zig", target, false, engine_only); _ = program(b, "tf-qwen27-dflash-refine", "Build the drafter refinement probe (top-1 by block position, masked versus given)", "zig/tests/qwen27/dflash_refine.zig", target, false, engine_only); diff --git a/zig/clef_checkpoint_cases.zig b/zig/clef_checkpoint_cases.zig index 0fb3e5d777..fc8bc57da6 100644 --- a/zig/clef_checkpoint_cases.zig +++ b/zig/clef_checkpoint_cases.zig @@ -1,4 +1,6 @@ -//! CPU-only admission of the separate Clef head; no model, device or scoring arithmetic. +//! CPU-only Clef head admission and disposable-pass contracts; no device or scoring arithmetic. test { _ = @import("src/families/clef/weights.zig"); + _ = @import("src/families/qwen27/ane_handoff.zig"); + _ = @import("src/families/qwen27/hidden_rows_plan.zig"); } diff --git a/zig/src/families/clef/README.md b/zig/src/families/clef/README.md new file mode 100644 index 0000000000..dd894c54ee --- /dev/null +++ b/zig/src/families/clef/README.md @@ -0,0 +1,22 @@ +# Clef first-slice work + +Local work for [issue 692](https://github.com/ashhart/TensorFold/issues/692), not ready for a PR or deployment. The bf16 head reader is separate from the qwen27 backbone. The original 45-record, 128-question layout gate is still required; the older 15-record prototype does not replace it. The encoding and head port retain the credit for outcastofmusic's #241 and #251. + +`qwen27.hidden_rows.captureExclusive` returns owned bf16 final normalized rows in token order. It runs the existing prompt kernels with a fresh frame and fresh GDN/KV runner. It copies `frame.input` after each completed `advance(.none)` and restores the caller's frame and previous prompt flag on return. The caller must own the model for the entire call. This function is not wired to HTTP requests or the serving queue yet. + +The ANE handoff reserves storage before input encoding, then registers and publishes each committed command buffer without a fallible allocation. The worker can release an even wait after a terminal input-CB failure even when the odd signal never arrived. Drain waits for every registered CB and the last worker completion before releasing retained handles. A poisoned share stays unavailable until explicitly rebuilt. Restoring the chat frame after such a device failure does not guarantee that the server can continue serving. + +CPU validation uses Zig 0.17.0: + +```sh +zig build test-clef-checkpoint +zig build test +zig build test-golden fuzz-cortheon +zig fmt --check build.zig zig +``` + +The targeted check passes 27 tests: head admission, hidden-row bounds and the actual ANE worker/drain protocol. Its two-thread ring test publishes 1,024 tickets through a ring of four entries. The whole host build passes 162 steps; some tests are cached, so no aggregate fresh-test count is claimed. Goldens have zero unexpected differences. Lean has 527 inherited findings and zero new. + +`zig build tf-qwen27-hidden-rows-check` builds the synthetic device harness without executing it. The harness compares capture against normal prompt arithmetic with the LM head projected; this proves capture parity, not independent Clef numerical accuracy. It also checks output ownership, chat/snapshot continuation, cancellation, allocation failure and nonfinite final rows. Late cancel/OOM triggers must observe the CPU output of an already completed first chunk. Its optional `--ane` argument exercises the ANE path. + +The device harness has only been compiled. Run it only with exclusive use of the Metal engine. Pending gates include ANE on/off runs, device-fault behavior, performance against the pinned base, queued concurrent-request isolation, real-model chat/resume parity, the original layout corpus and an agreed numerical head tolerance. No probability tolerance has been selected. diff --git a/zig/src/families/qwen27/ane_handoff.zig b/zig/src/families/qwen27/ane_handoff.zig new file mode 100644 index 0000000000..a5147ede52 --- /dev/null +++ b/zig/src/families/qwen27/ane_handoff.zig @@ -0,0 +1,207 @@ +//! A handoff completes even when its input command buffer fails, so later GPU waits can end. +const std = @import("std"); + +/// One producer and one consumer; completion makes both payload reuse and retained-CB release safe. +pub fn Ring(comptime Payload: type, comptime capacity: usize) type { + return struct { + slots: [capacity]Entry = @splat(.{}), + issued: u64 = 0, + completed: std.atomic.Value(u64) = .init(0), + const Self = @This(); + const Entry = struct { payload: Payload = undefined, published: std.atomic.Value(u64) = .init(0) }; + + pub fn prepare(q: *Self, a: std.mem.Allocator, retained: anytype) !void { + if (q.issued >= std.math.maxInt(u64) / 2) return error.AneHandoffOverflow; + if (q.issued - q.completed.load(.acquire) >= capacity) return error.AneHandoffFull; + try retained.ensureUnusedCapacity(a, 1); + } + + pub fn publish(q: *Self, payload: Payload) void { + const entry = &q.slots[q.issued % capacity]; + entry.payload = payload; + q.issued += 1; + entry.published.store(q.issued, .release); + } + + pub fn get(q: *Self, t: u64) ?*const Payload { + const entry = &q.slots[t % capacity]; + return if (entry.published.load(.acquire) == t + 1) &entry.payload else null; + } + + pub fn complete(q: *Self, t: u64) void { + q.completed.store(t + 1, .release); + } + }; +} + +/// `job` owns its retained input CB until the batch has also observed worker completion. +pub fn execute(job: anytype) void { + while (!job.ready()) { + if (job.inputFailed()) { + job.poison(); + break; + } + job.waitInput(); + } + if (!job.poisoned()) job.evaluate(); + job.complete(); +} + +/// Wait every input CB and the worker before releasing any CB, including on multiple failures. +pub fn drain(batch: anytype) bool { + var failed = false; + for (0..batch.count()) |i| { + batch.waitBuffer(i); + failed = batch.bufferFailed(i) or failed; + } + batch.waitWorker(); + for (0..batch.count()) |i| batch.releaseBuffer(i); + batch.clear(); + return failed; +} + +const FakeJob = struct { + input: enum { pending, ready, failed } = .pending, + fail_after_wait: bool = false, + poison_: bool = false, + ran: bool = false, + completed: bool = false, + waits: usize = 0, + pub fn ready(j: *FakeJob) bool { + return j.input == .ready; + } + pub fn inputFailed(j: *FakeJob) bool { + return j.input == .failed; + } + pub fn poison(j: *FakeJob) void { + j.poison_ = true; + } + pub fn poisoned(j: *FakeJob) bool { + return j.poison_; + } + pub fn waitInput(j: *FakeJob) void { + j.waits += 1; + j.input = if (j.fail_after_wait) .failed else .ready; + } + pub fn evaluate(j: *FakeJob) void { + std.debug.assert(j.input == .ready and !j.poison_); + j.ran = true; + } + pub fn complete(j: *FakeJob) void { + std.debug.assert(j.input != .pending); + std.debug.assert(j.poison_ or j.ran); + j.completed = true; + } +}; + +test "failed input with no odd signal releases its even wait without running ANE" { + var j: FakeJob = .{ .fail_after_wait = true }; + execute(&j); + try std.testing.expect(j.poison_ and j.completed and !j.ran); + try std.testing.expectEqual(@as(usize, 1), j.waits); +} + +test "a poisoned followup still waits its input CB before releasing its even wait" { + var j: FakeJob = .{ .poison_ = true }; + execute(&j); + try std.testing.expect(j.completed and !j.ran); + try std.testing.expectEqual(@as(usize, 1), j.waits); +} + +test "successful input runs before worker completion" { + var j: FakeJob = .{}; + execute(&j); + try std.testing.expect(j.ran and j.completed and !j.poison_); +} + +const FakeBatch = struct { + waited: [3]bool = @splat(false), + released: [3]bool = @splat(false), + worker_done: bool = false, + cleared: bool = false, + pub fn count(_: *FakeBatch) usize { + return 3; + } + pub fn waitBuffer(b: *FakeBatch, i: usize) void { + b.waited[i] = true; + } + pub fn bufferFailed(_: *FakeBatch, i: usize) bool { + return i != 1; + } + pub fn waitWorker(b: *FakeBatch) void { + std.debug.assert(std.mem.allEqual(bool, &b.waited, true)); + b.worker_done = true; + } + pub fn releaseBuffer(b: *FakeBatch, i: usize) void { + std.debug.assert(b.worker_done); + b.released[i] = true; + } + pub fn clear(b: *FakeBatch) void { + std.debug.assert(std.mem.allEqual(bool, &b.released, true)); + b.cleared = true; + } +}; + +test "drain waits all CBs and the worker despite first and last CB failures" { + var b: FakeBatch = .{}; + try std.testing.expect(drain(&b)); + try std.testing.expect(b.cleared and b.worker_done); +} + +test "failed reservation publishes no ticket and the next successful ticket is zero" { + const a = std.testing.allocator; + var q: Ring(u32, 2) = .{}; + var retained: std.ArrayList(u32) = .empty; + defer retained.deinit(a); + var failing = std.testing.FailingAllocator.init(a, .{ .fail_index = 0 }); + try std.testing.expectError(error.OutOfMemory, q.prepare(failing.allocator(), &retained)); + try std.testing.expectEqual(@as(u64, 0), q.issued); + try std.testing.expect(q.get(0) == null); + try q.prepare(a, &retained); + retained.appendAssumeCapacity(42); + q.publish(42); + try std.testing.expectEqual(@as(u32, 42), q.get(0).?.*); + try std.testing.expectEqual(@as(u64, 1), q.issued); + q.complete(0); +} + +test "real producer and consumer publish complete payloads through repeated ring wraparound" { + const Payload = struct { t: u64, checksum: u64 }; + const Queue = Ring(Payload, 4); + const Consumer = struct { + fn run(q: *Queue, bad: *std.atomic.Value(bool)) void { + for (0..1024) |t| { + while (true) { + if (q.get(t)) |p| { + if (p.t != t or p.checksum != t ^ 0xaabbccdd) bad.store(true, .release); + q.complete(t); + break; + } + std.atomic.spinLoopHint(); + } + } + } + }; + var q: Queue = .{}; + var bad: std.atomic.Value(bool) = .init(false); + var retained: std.ArrayList(u32) = .empty; + defer retained.deinit(std.testing.allocator); + // Reserve once so an allocation error cannot leave the consumer waiting for unpublished work. + try retained.ensureTotalCapacity(std.testing.allocator, 1024); + const thread = try std.Thread.spawn(.{}, Consumer.run, .{ &q, &bad }); + defer thread.join(); + for (0..1024) |t| { + while (true) { + q.prepare(std.testing.allocator, &retained) catch |err| { + std.debug.assert(err == error.AneHandoffFull); + std.atomic.spinLoopHint(); + continue; + }; + break; + } + retained.appendAssumeCapacity(@intCast(t)); + q.publish(.{ .t = t, .checksum = t ^ 0xaabbccdd }); + } + while (q.completed.load(.acquire) < 1024) std.atomic.spinLoopHint(); + try std.testing.expect(!bad.load(.acquire)); +} diff --git a/zig/src/families/qwen27/ane_mlp.zig b/zig/src/families/qwen27/ane_mlp.zig index 64a183e2c7..b0fb01c9ec 100644 --- a/zig/src/families/qwen27/ane_mlp.zig +++ b/zig/src/families/qwen27/ane_mlp.zig @@ -5,6 +5,7 @@ const ane = mtl.ane; const cp = @import("core").checkpoint_metal; const Lane = @import("core").segments.Lane; const Config = @import("config.zig").Config; +const handoff = @import("ane_handoff.zig"); /// Program rows: a prompt chunk of up to 128 rows, zero past its last (rows never depend on each other). pub const width = 128; @@ -24,12 +25,17 @@ pub const Share = struct { /// Command buffers committed at handoffs during the current prompt chunk; drain() checks them. committed: std.ArrayList(mtl.CommandBuffer) = .empty, gpu_seconds: f64 = 0, - calls: u64 = 0, - ring: [256]std.atomic.Value(u32) = @splat(.init(0)), + tickets: handoff.Ring(Ticket, 256) = .{}, + poisoned: std.atomic.Value(bool) = .init(false), stop: std.atomic.Value(bool) = .init(false), failed: std.atomic.Value(bool) = .init(false), thread: ?std.Thread = null, + const Ticket = struct { + layer: usize, + cb: mtl.CommandBuffer, + }; + /// The fp16 weights a share of `columns` holds: every layer's gate, up and down over those columns. pub fn bytes(c: Config, columns: u32) u64 { return 3 * @as(u64, columns) * c.hidden * 2 * c.layers; @@ -78,6 +84,7 @@ pub const Share = struct { } pub fn deinit(s: *Share) void { + s.drain() catch {}; s.stop.store(true, .release); if (s.thread) |t| t.join(); for (s.requests) |r| r.deinit(); @@ -93,24 +100,26 @@ pub const Share = struct { s.allocator.destroy(s); } - /// Queues `layer`'s program; the GPU signals [0] once inputs are written and waits for [1] before the partial. - pub fn next(s: *Share, layer: usize) [2]u64 { - const t = s.calls; - s.calls += 1; - s.ring[t % s.ring.len].store(@intCast(layer), .release); - return .{ 2 * t + 1, 2 * t + 2 }; + /// Reserve before encoding inputs: no ticket or committed CB exists if allocation fails here. + pub fn prepare(s: *Share) !void { + if (s.poisoned.load(.acquire)) return error.AnePoisoned; + try s.tickets.prepare(s.allocator, &s.committed); } - /// Signals `value` at the buffer's end and commits it, then opens the next behind the fence. - pub fn signal(s: *Share, lane: *Lane, value: u64) !void { + /// After prepare and successful input encoding: publish a retained CB, then return its worker's even signal. + pub fn submit(s: *Share, lane: *Lane, layer: usize) u64 { + const t = s.tickets.issued; + const cb = lane.cb.retain(); + s.committed.appendAssumeCapacity(cb); lane.fence.update(lane.enc); lane.enc.end(); - lane.cb.signal(s.event, value); + lane.cb.signal(s.event, 2 * t + 1); lane.cb.commit(); - try s.committed.append(s.allocator, .{ .id = mtl.objc.retain(lane.cb.id) }); + s.tickets.publish(.{ .layer = layer, .cb = cb }); lane.cb = s.queue.commandBuffer(); lane.enc = lane.cb.compute(.concurrent); lane.fence.wait(lane.enc); + return 2 * t + 2; } /// The lane's later work waits for `value` (the program's output), between its encoders. @@ -124,29 +133,82 @@ pub const Share = struct { /// After a prompt chunk: the handoff buffers finished without error and the programs ran. pub fn drain(s: *Share) !void { - defer { - for (s.committed.items) |cb| mtl.objc.release(cb.id); - s.committed.clearRetainingCapacity(); - } - for (s.committed.items) |cb| { - cb.wait(); - if (@import("decode_round.zig").gpuFailed(cb, "a prompt chunk's Neural Engine handoff")) return error.GpuFailed; - s.gpu_seconds += cb.gpuSeconds(); + var batch = Batch{ .share = s }; + if (handoff.drain(&batch)) { + s.poisoned.store(true, .release); + return error.GpuFailed; } + if (s.poisoned.load(.acquire)) return error.AnePoisoned; if (s.failed.swap(false, .acq_rel)) return error.AneFailed; // that chunk fails; the next one runs again } + const Batch = struct { + share: *Share, + pub fn count(b: *Batch) usize { + return b.share.committed.items.len; + } + pub fn waitBuffer(b: *Batch, i: usize) void { + b.share.committed.items[i].wait(); + } + pub fn bufferFailed(b: *Batch, i: usize) bool { + const cb = b.share.committed.items[i]; + if (@import("decode_round.zig").gpuFailed(cb, "a prompt chunk's Neural Engine handoff")) return true; + b.share.gpu_seconds += cb.gpuSeconds(); + return false; + } + pub fn waitWorker(b: *Batch) void { + const s = b.share; + while (s.tickets.completed.load(.acquire) < s.tickets.issued) _ = s.event.wait(2 * s.tickets.issued, 100); + } + pub fn releaseBuffer(b: *Batch, i: usize) void { + b.share.committed.items[i].release(); + } + pub fn clear(b: *Batch) void { + b.share.committed.clearRetainingCapacity(); + } + }; + + const Job = struct { + share: *Share, + ticket: *const Ticket, + t: u64, + spins: u64 = 0, + pub fn ready(j: *Job) bool { + return j.share.event.value() >= 2 * j.t + 1; + } + pub fn inputFailed(j: *Job) bool { + return j.ticket.cb.status() == .@"error"; + } + pub fn poison(j: *Job) void { + j.share.poisoned.store(true, .release); + } + pub fn poisoned(j: *Job) bool { + return j.share.poisoned.load(.acquire); + } + pub fn waitInput(j: *Job) void { + j.spins += 1; + if (j.spins < 2_000_000) std.atomic.spinLoopHint() else _ = j.share.event.wait(2 * j.t + 1, 100); + } + pub fn evaluate(j: *Job) void { + j.share.requests[j.ticket.layer].run() catch j.share.failed.store(true, .release); + } + pub fn complete(j: *Job) void { + j.share.event.set(2 * j.t + 2); + j.share.tickets.complete(j.t); + } + }; + fn work(s: *Share) void { var t: u64 = 0; while (true) : (t += 1) { - // spin on the counter while prompt chunks run: a sleeping wait wakes too late for a per-layer handoff var spins: u64 = 0; - while (s.event.value() < 2 * t + 1) : (spins += 1) { - if (spins < 2_000_000) std.atomic.spinLoopHint() else if (s.event.wait(2 * t + 1, 100)) break else if (s.stop.load(.acquire)) return; - } - const layer = s.ring[t % s.ring.len].load(.acquire); - s.requests[layer].run() catch s.failed.store(true, .release); - s.event.set(2 * t + 2); + const ticket = while (true) : (spins += 1) { + if (s.tickets.get(t)) |ticket| break ticket; + if (s.stop.load(.acquire)) return; + if (spins < 2_000_000) std.atomic.spinLoopHint() else _ = s.event.wait(2 * t + 1, 100); + }; + var job = Job{ .share = s, .ticket = ticket, .t = t }; + handoff.execute(&job); } } }; diff --git a/zig/src/families/qwen27/decode_round.zig b/zig/src/families/qwen27/decode_round.zig index 4a405c0aea..9c51a9f5d8 100644 --- a/zig/src/families/qwen27/decode_round.zig +++ b/zig/src/families/qwen27/decode_round.zig @@ -223,6 +223,9 @@ pub const Runner = struct { var stages = forward.Stages{ .model = r.model, .rows = @intCast(round.ids.len), .hooks = .{ .ptr = r, .gdn = gdnForward, .attention = attentionForward, .tap = if (r.capture_taps) tap else null }, .head = head, .images = placed.images }; r.last_gpu_seconds = @import("core").segments.runCommitting(r.model.device, &.{r.model.queue}, r.model.config.layers, .concurrent, &stages, commit_every) catch |err| { r.failed = true; + if (r.model.ane) |share| share.drain() catch |drain_err| { + std.log.warn("qwen27: ANE cleanup after {s}: {s}", .{ @errorName(err), @errorName(drain_err) }); + }; return err; }; if (r.model.ane) |share| share.drain() catch |err| { diff --git a/zig/src/families/qwen27/forward.zig b/zig/src/families/qwen27/forward.zig index c268b7dfd5..8c5425153d 100644 --- a/zig/src/families/qwen27/forward.zig +++ b/zig/src/families/qwen27/forward.zig @@ -84,18 +84,18 @@ pub const Stages = struct { const width: u32 = @intCast(m.config.intermediate); if (if (m.kernels.prompt) m.ane else null) |share| { // the Neural Engine runs MLP columns below share.columns; the GPU runs the rest, then adds its partial - const v = share.next(index); const planes = Ref{ .buffer = share.output.buffer }; const own = share.columns / 64; const all = width / 64; + try share.prepare(); try m.kernels.aneInput(e, f.get(.input), f.get(.dims), .{ .buffer = share.input.buffer }, h, width, share.columns); - try share.signal(lane, v[0]); + const finished = share.submit(lane, index); try m.kernels.quantSkip(lane.enc, layer.gu, f.get(.input), sums(f, .sums, s.rows), f.get(.dims), f.get(.mixed), s.rows, share.columns / 32); lane.enc.barrier(); try m.kernels.mlpAne(lane.enc, f.get(.mixed), up, f.get(.dims), f.get(.activated), f.get(.act_sums), planes, h, width, share.columns, s.rows, own, all - own); lane.enc.barrier(); try m.kernels.quantCut(lane.enc, layer.down, f.get(.activated), sums(f, .act_sums, s.rows), f.get(.dims), .{ .buffer = share.partial }, s.rows, .{ own, all - own }); - share.wait(lane, v[1]); + share.wait(lane, finished); try m.kernels.addAne(lane.enc, .{ .buffer = share.partial }, planes, f.get(.output), h, width, share.columns, s.rows); lane.enc.barrier(); s.pending = f.get(.output); diff --git a/zig/src/families/qwen27/hidden_rows.zig b/zig/src/families/qwen27/hidden_rows.zig new file mode 100644 index 0000000000..fc102be11d --- /dev/null +++ b/zig/src/families/qwen27/hidden_rows.zig @@ -0,0 +1,61 @@ +//! Final normalized prompt rows, with their own frame and fresh recurrent/KV state for each call. +const std = @import("std"); +const Model = @import("model.zig").Model; +const Frame = @import("gpu_frame.zig").Frame; +const Runner = @import("decode_round.zig").Runner; +const Round = @import("round_plan.zig").Round; +const Plan = @import("hidden_rows_plan.zig").Plan; + +pub const Options = struct { + context: u32, + chunk: u32 = 128, + cancel: ?*const std.atomic.Value(bool) = null, + + fn check(o: Options) !void { + if (o.cancel) |c| if (c.load(.acquire)) return error.Canceled; + } +}; + +/// Owned bf16 bits in token order, [count][hidden], independent of later forward passes. +pub const Rows = struct { + allocator: std.mem.Allocator, + values: []u16, + count: u32, + hidden: usize, + + pub fn deinit(r: *Rows) void { + r.allocator.free(r.values); + r.* = undefined; + } +}; + +/// Caller owns the model exclusively for the entire call (the engine thread, never an HTTP thread). +pub fn captureExclusive(a: std.mem.Allocator, m: *Model, ids: []const u32, o: Options) !Rows { + const p = try Plan.init(ids, m.config.hidden, m.config.vocab, m.frame.capacity, o.context, o.chunk); + try o.check(); + const out = try a.alloc(u16, p.values); + errdefer a.free(out); + var frame = try Frame.init(m.device, m.config, m.frame.capacity); + defer frame.deinit(); + std.mem.swap(Frame, &m.frame, &frame); + defer std.mem.swap(Frame, &m.frame, &frame); + const prompt = m.kernels.prompt; + m.kernels.prompt = true; + defer m.kernels.prompt = prompt; + var runner = try Runner.init(a, m, 1, p.rows); + defer runner.deinit(); + var first: usize = 0; + while (first < ids.len) { + try o.check(); + const end = @min(ids.len, first + o.chunk); + var round = try Round.init(a, &.{.{ .slot = 0, .start = runner.offsets[0], .capacity = runner.capacity, .ids = ids[first..end] }}, @intCast(m.config.conv_kernel), 1, @intCast(m.config.vocab)); + defer round.deinit(); + try runner.advance(&round, .none); + const values = m.frame.get(.input).buffer.slice(u16, (end - first) * m.config.hidden); + for (values) |bits| if (bits & 0x7f80 == 0x7f80) return error.NonfiniteHiddenRows; + @memcpy(out[first * m.config.hidden .. end * m.config.hidden], values); + first = end; + } + try o.check(); + return .{ .allocator = a, .values = out, .count = p.rows, .hidden = m.config.hidden }; +} diff --git a/zig/src/families/qwen27/hidden_rows_plan.zig b/zig/src/families/qwen27/hidden_rows_plan.zig new file mode 100644 index 0000000000..a700194ec6 --- /dev/null +++ b/zig/src/families/qwen27/hidden_rows_plan.zig @@ -0,0 +1,39 @@ +//! Check a whole disposable prompt before it can allocate device state or replace the chat frame. +const std = @import("std"); + +pub const Plan = struct { + rows: u32, + values: usize, + + pub fn init(ids: []const u32, hidden: usize, vocab: usize, frame: u32, context: u32, chunk: u32) !Plan { + if (ids.len == 0 or ids.len > context or ids.len > std.math.maxInt(i32)) return error.BadHiddenPrompt; + if (hidden == 0 or vocab == 0 or vocab > std.math.maxInt(u32)) return error.BadHiddenGeometry; + if (frame == 0 or frame > 128 or frame % 16 != 0 or chunk == 0 or chunk > frame) return error.BadHiddenChunk; + for (ids) |id| if (id >= vocab) return error.BadTokenId; + const values = try std.math.mul(usize, ids.len, hidden); + _ = try std.math.mul(usize, values, @sizeOf(u16)); + return .{ .rows = @intCast(ids.len), .values = values }; + } +}; + +test "all tokens including later chunks are checked before a disposable pass" { + try std.testing.expectError(error.BadTokenId, Plan.init(&.{ 1, 2, 3, 96 }, 512, 96, 128, 4, 1)); + const p = try Plan.init(&.{ 0, 95 }, 512, 96, 128, 2, 1); + try std.testing.expectEqual(@as(u32, 2), p.rows); + try std.testing.expectEqual(@as(usize, 1024), p.values); +} + +test "empty and oversized prompts and incompatible frame sizes fail before allocation" { + try std.testing.expectError(error.BadHiddenPrompt, Plan.init(&.{}, 512, 96, 128, 4, 1)); + try std.testing.expectError(error.BadHiddenPrompt, Plan.init(&.{ 1, 2 }, 512, 96, 128, 1, 1)); + for ([_]u32{ 0, 1, 127, 129 }) |frame| + try std.testing.expectError(error.BadHiddenChunk, Plan.init(&.{1}, 512, 96, frame, 1, 1)); + for ([_]u32{ 0, 129 }) |chunk| + try std.testing.expectError(error.BadHiddenChunk, Plan.init(&.{1}, 512, 96, 128, 1, chunk)); +} + +test "hidden row allocation checks both element and byte multiplication" { + try std.testing.expectError(error.Overflow, Plan.init(&.{ 0, 0 }, std.math.maxInt(usize), 96, 128, 2, 1)); + try std.testing.expectError(error.Overflow, Plan.init(&.{0}, std.math.maxInt(usize), 96, 128, 1, 1)); + try std.testing.expectError(error.BadHiddenGeometry, Plan.init(&.{0}, 0, 96, 128, 1, 1)); +} diff --git a/zig/src/families/qwen27/qwen27.zig b/zig/src/families/qwen27/qwen27.zig index d5e6124e8c..3d3437d624 100644 --- a/zig/src/families/qwen27/qwen27.zig +++ b/zig/src/families/qwen27/qwen27.zig @@ -20,6 +20,7 @@ pub const glue = @import("glue.zig"); pub const glue_gpu = @import("glue_gpu.zig"); pub const image_rows = @import("image_rows.zig"); pub const gpu_weights = @import("gpu_weights.zig"); +pub const hidden_rows = @import("hidden_rows.zig"); pub const gpu_frame = @import("gpu_frame.zig"); pub const model = @import("model.zig"); pub const projection = @import("projection.zig"); diff --git a/zig/tests/qwen27/hidden_rows.zig b/zig/tests/qwen27/hidden_rows.zig new file mode 100644 index 0000000000..4df48208e7 --- /dev/null +++ b/zig/tests/qwen27/hidden_rows.zig @@ -0,0 +1,136 @@ +//! Synthetic normalized-row, chat and kept-state checks; build-only until an exclusive Metal test window. +const std = @import("std"); +const mtl = @import("metal"); +const q = @import("tensorfold").qwen27; +const graph = @import("synthetic_graph.zig"); + +/// Normal chat prompt arithmetic, with the LM head projected, keeps every row's final normalized bf16 bits. +fn reference(a: std.mem.Allocator, m: *q.model.Model, ids: []const u32, chunk: usize) ![]u16 { + const out = try a.alloc(u16, ids.len * m.config.hidden); + errdefer a.free(out); + var runner = try q.decode_round.Runner.init(a, m, 1, @intCast(ids.len)); + defer runner.deinit(); + const prompt = m.kernels.prompt; + m.kernels.prompt = true; + defer m.kernels.prompt = prompt; + var first: usize = 0; + while (first < ids.len) { + const end = @min(ids.len, first + chunk); + var round = try q.round_plan.Round.init(a, &.{.{ .slot = 0, .start = runner.offsets[0], .capacity = runner.capacity, .ids = ids[first..end] }}, @intCast(m.config.conv_kernel), 1, @intCast(m.config.vocab)); + defer round.deinit(); + try runner.advance(&round, .all); + @memcpy(out[first * m.config.hidden .. end * m.config.hidden], m.frame.get(.input).buffer.slice(u16, (end - first) * m.config.hidden)); + first = end; + } + return out; +} + +fn unchanged(r: *q.decode_round.Runner, frame: q.gpu_frame.Frame, prompt: bool, fingerprint: [32]u8) !void { + for (r.model.frame.buffers, frame.buffers) |got, want| if (got.id != want.id) return error.ChatFrameChanged; + if (r.model.kernels.prompt != prompt) return error.PromptFlagChanged; + if (!std.mem.eql(u8, &fingerprint, &try (q.session.Session{ .runner = r }).fingerprint())) return error.ChatStateChanged; +} + +fn tokens(s: q.session.Session, out: []u32) !void { + for (out, 0..) |*id, i| { + id.* = try s.greedy(); + if (i + 1 < out.len) try s.step(id.*); + } +} + +fn restoreChat(a: std.mem.Allocator, r: *q.decode_round.Runner, saved: *q.snapshot.State, last: []const u32) !void { + try q.snapshot.restore(a, r, null, saved); + try (q.session.Session{ .runner = r }).promptChunk(last, true); +} + +pub fn main(init: std.process.Init) !void { + var arena = std.heap.ArenaAllocator.init(init.gpa); + defer arena.deinit(); + const a = arena.allocator(); + const args = try init.minimal.args.toSlice(a); + if (args.len > 2 or (args.len == 2 and !std.mem.eql(u8, args[1], "--ane"))) return error.Usage; + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const m = try graph.create(a); + defer m.deinit(); + if (args.len == 2) m.ane = try q.ane_mlp.Share.init(a, m.device, m.queue, &m.checkpoint, m.config, 64); + var ids: [130]u32 = undefined; + for (&ids, 0..) |*id, i| id.* = @intCast((i * 17 + 7) % 96); + const expected_rows = try reference(a, m, &ids, 16); + defer a.free(expected_rows); + for ([_]u32{ 1, 16, 128 }) |chunk| { + var rows = try q.hidden_rows.captureExclusive(a, m, &ids, .{ .context = 256, .chunk = chunk }); + defer rows.deinit(); + if (rows.count != ids.len or rows.hidden != m.config.hidden) return error.RowShapeDiffers; + if (!std.mem.eql(u16, rows.values, expected_rows)) return error.NormalizedRowsDiffer; + var later = try q.hidden_rows.captureExclusive(a, m, ids[0..7], .{ .context = 256, .chunk = chunk }); + defer later.deinit(); + if (!std.mem.eql(u16, rows.values, expected_rows)) return error.RowsAliasLaterPass; + } + + var runner = try q.decode_round.Runner.init(a, m, 1, 256); + defer runner.deinit(); + const session = q.session.Session{ .runner = &runner }; + const chat = ids[0..11]; + try session.prefill(chat[0 .. chat.len - 1], 16); + const saved = try q.snapshot.save(a, &runner, null, @intCast(chat.len - 1)); + defer q.snapshot.drop(a, saved); + try session.promptChunk(chat[chat.len - 1 ..], true); + var expected_tokens: [8]u32 = undefined; + try tokens(session, &expected_tokens); + try restoreChat(a, &runner, saved, chat[chat.len - 1 ..]); + const original_frame = m.frame; + const original_state = try session.fingerprint(); + + for ([_]bool{ false, true }) |prompt| { + m.kernels.prompt = prompt; + var rows = try q.hidden_rows.captureExclusive(a, m, &ids, .{ .context = 256, .chunk = 16 }); + defer rows.deinit(); + try unchanged(&runner, original_frame, prompt, original_state); + } + m.kernels.prompt = false; + var canceled: std.atomic.Value(bool) = .init(true); + if (q.hidden_rows.captureExclusive(a, m, &ids, .{ .context = 256, .cancel = &canceled })) |rows_| { + var rows = rows_; + rows.deinit(); + return error.ExpectedCancellation; + } else |err| if (err != error.Canceled) return err; + try unchanged(&runner, original_frame, false, original_state); + + try @import("hidden_rows_cancel.zig").run(a, m, &ids, 16); + try unchanged(&runner, original_frame, false, original_state); + // A completed forward whose final norm is nonfinite must fail without touching the chat frame or caches. + { + const norm = m.weights.norm; + const scales = norm.buffer.slice(u16, norm.offset / 2 + m.config.hidden)[norm.offset / 2 ..]; + const before = scales[0]; + scales[0] = 0x7fc0; + defer scales[0] = before; + if (q.hidden_rows.captureExclusive(a, m, &ids, .{ .context = 256, .chunk = 16 })) |rows_| { + var rows = rows_; + rows.deinit(); + return error.ExpectedNonfiniteRows; + } else |err| if (err != error.NonfiniteHiddenRows) return err; + } + try unchanged(&runner, original_frame, false, original_state); + + // Allocation failures after the private frame is installed must also restore the chat frame and flag. + for ([_]usize{ 0, 1, 2, 3, 8, 16 }) |at| { + var failing = std.testing.FailingAllocator.init(a, .{ .fail_index = at }); + if (q.hidden_rows.captureExclusive(failing.allocator(), m, &ids, .{ .context = 256, .chunk = 16 })) |rows_| { + var rows = rows_; + rows.deinit(); + return error.ExpectedAllocationFailure; + } else |err| if (err != error.OutOfMemory or !failing.has_induced_failure) return err; + if (failing.allocated_bytes != failing.freed_bytes) return error.CaptureHostLeak; + try unchanged(&runner, original_frame, false, original_state); + } + var actual_tokens: [8]u32 = undefined; + try tokens(session, &actual_tokens); + if (!std.mem.eql(u32, &actual_tokens, &expected_tokens)) return error.ChatReplyChanged; + try restoreChat(a, &runner, saved, chat[chat.len - 1 ..]); + try unchanged(&runner, original_frame, false, original_state); + try tokens(session, &actual_tokens); + if (!std.mem.eql(u32, &actual_tokens, &expected_tokens)) return error.ResumedReplyChanged; + std.debug.print("hidden rows: exact bits, owned rows, chat/resume, cancel, OOM and nonfinite isolation passed; ANE={}\n", .{m.ane != null}); +} diff --git a/zig/tests/qwen27/hidden_rows_cancel.zig b/zig/tests/qwen27/hidden_rows_cancel.zig new file mode 100644 index 0000000000..bb5f4005e0 --- /dev/null +++ b/zig/tests/qwen27/hidden_rows_cancel.zig @@ -0,0 +1,82 @@ +//! Cancel and allocation failure after the allocation budget of a complete first prompt chunk. +const std = @import("std"); +const q = @import("tensorfold").qwen27; + +const InterruptAllocator = struct { + backing: std.mem.Allocator, + cancel: *std.atomic.Value(bool), + oom: bool = false, + at: usize, + calls: usize = 0, + expected: []const u8, + output: ?[]u8 = null, + saw_completed_first_chunk: bool = false, + fired: bool = false, + + fn allocator(c: *InterruptAllocator) std.mem.Allocator { + return .{ .ptr = c, .vtable = &.{ .alloc = alloc, .resize = resize, .remap = remap, .free = free } }; + } + fn alloc(ptr: *anyopaque, len: usize, alignment: std.mem.Alignment, ra: usize) ?[*]u8 { + const c: *InterruptAllocator = @ptrCast(@alignCast(ptr)); + if (c.calls == c.at) { + // The CPU writes output only after advance/drain returns; never read GPU buffers during encoding. + if (c.output) |out| c.saw_completed_first_chunk = out.len >= c.expected.len and std.mem.eql(u8, out[0..c.expected.len], c.expected); + c.fired = true; + if (c.oom) return null; + c.cancel.store(true, .release); + } + const out = c.backing.rawAlloc(len, alignment, ra) orelse return null; + if (c.calls == 0) { + c.output = out[0..len]; + @memset(out[0..len], 0xff); + } + c.calls += 1; + return out; + } + fn resize(ptr: *anyopaque, memory: []u8, alignment: std.mem.Alignment, len: usize, ra: usize) bool { + const c: *InterruptAllocator = @ptrCast(@alignCast(ptr)); + return c.backing.rawResize(memory, alignment, len, ra); + } + fn remap(ptr: *anyopaque, memory: []u8, alignment: std.mem.Alignment, len: usize, ra: usize) ?[*]u8 { + const c: *InterruptAllocator = @ptrCast(@alignCast(ptr)); + return c.backing.rawRemap(memory, alignment, len, ra); + } + fn free(ptr: *anyopaque, memory: []u8, alignment: std.mem.Alignment, ra: usize) void { + const c: *InterruptAllocator = @ptrCast(@alignCast(ptr)); + c.backing.rawFree(memory, alignment, ra); + } +}; + +pub fn run(a: std.mem.Allocator, m: *q.model.Model, ids: []const u32, chunk: u32) !void { + if (ids.len <= chunk) return error.NeedMultipleChunks; + var count = std.testing.FailingAllocator.init(a, .{}); + var first = try q.hidden_rows.captureExclusive(count.allocator(), m, ids[0..chunk], .{ .context = 256, .chunk = chunk }); + const expected = a.dupe(u16, first.values) catch |err| { + first.deinit(); + return err; + }; + defer a.free(expected); + first.deinit(); + if (count.allocated_bytes != count.freed_bytes) return error.CalibrationLeak; + + var canceled: std.atomic.Value(bool) = .init(false); + var cancel_memory = std.testing.FailingAllocator.init(a, .{}); + var interrupt = InterruptAllocator{ .backing = cancel_memory.allocator(), .cancel = &canceled, .at = count.alloc_index, .expected = std.mem.sliceAsBytes(expected) }; + if (q.hidden_rows.captureExclusive(interrupt.allocator(), m, ids, .{ .context = 256, .chunk = chunk, .cancel = &canceled })) |rows_| { + var rows = rows_; + rows.deinit(); + return error.ExpectedMidPassCancellation; + } else |err| if (err != error.Canceled or !canceled.load(.acquire)) return err; + if (!interrupt.saw_completed_first_chunk) return error.CancellationWasTooEarly; + if (cancel_memory.allocated_bytes != cancel_memory.freed_bytes) return error.CanceledHostLeak; + + var failing = std.testing.FailingAllocator.init(a, .{}); + var oom = InterruptAllocator{ .backing = failing.allocator(), .cancel = &canceled, .at = count.alloc_index, .expected = std.mem.sliceAsBytes(expected), .oom = true }; + if (q.hidden_rows.captureExclusive(oom.allocator(), m, ids, .{ .context = 256, .chunk = chunk })) |rows_| { + var rows = rows_; + rows.deinit(); + return error.ExpectedMidPassAllocationFailure; + } else |err| if (err != error.OutOfMemory or !oom.fired) return err; + if (!oom.saw_completed_first_chunk) return error.AllocationFailureWasTooEarly; + if (failing.allocated_bytes != failing.freed_bytes) return error.MidPassHostLeak; +} From 551bad61ef3357e73fc786e1f23bb5bfc0b94f3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schr=C3=B6der?= <1188270+cwschroeder@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:54:40 +0200 Subject: [PATCH 4/8] Queue hidden-row captures on the engine owner thread --- zig/build/qwen27.zig | 1 + zig/src/core/lane_host.zig | 18 +- zig/src/core/lane_host_calls.zig | 111 +++++++ zig/src/core/lane_host_calls_test.zig | 393 ++++++++++++++++++++++++ zig/src/core/lane_host_learn.zig | 2 +- zig/src/core/lane_host_rows.zig | 5 +- zig/src/core/lane_host_test.zig | 1 + zig/src/families/clef/README.md | 12 +- zig/src/families/qwen27/hidden_rows.zig | 3 + zig/src/native/qwen27_served.zig | 23 ++ zig/tests/qwen27/hidden_rows_served.zig | 143 +++++++++ 11 files changed, 704 insertions(+), 8 deletions(-) create mode 100644 zig/src/core/lane_host_calls.zig create mode 100644 zig/src/core/lane_host_calls_test.zig create mode 100644 zig/tests/qwen27/hidden_rows_served.zig diff --git a/zig/build/qwen27.zig b/zig/build/qwen27.zig index 3c44529b61..4c19645d67 100644 --- a/zig/build/qwen27.zig +++ b/zig/build/qwen27.zig @@ -119,6 +119,7 @@ pub fn targets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.bu const api = b.createModule(.{ .root_source_file = b.path("zig/src/core/engine_api.zig"), .target = target, .optimize = .ReleaseSafe, .link_libc = true, .imports = &.{.{ .name = "lanes", .module = lanes }} }); const host = b.createModule(.{ .root_source_file = b.path("zig/src/native/qwen27_served.zig"), .target = target, .optimize = .ReleaseSafe, .link_libc = true, .imports = &.{ .{ .name = "engine_api", .module = api }, .{ .name = "metal", .module = metal }, .{ .name = "tensorfold", .module = engine } } }); const served: []const std.Build.Module.Import = &.{ .{ .name = "tensorfold", .module = engine }, .{ .name = "metal", .module = metal }, .{ .name = "engine_api", .module = api }, .{ .name = "qwen27_served", .module = host } }; + _ = program(b, "tf-qwen27-hidden-rows-served-check", "Build queued capture versus solo and concurrent chat checks without running a GPU", "zig/tests/qwen27/hidden_rows_served.zig", target, false, served); _ = program(b, "tf-qwen27-served-check", "Build synthetic CLI versus served-host token check", "zig/tests/qwen27/served.zig", target, false, served); _ = program(b, "tf-qwen27-dflash-served-check", "Build synthetic drafted CLI versus served-host token check", "zig/tests/qwen27/dflash_served.zig", target, false, served); _ = program(b, "tf-qwen27-lanes-check", "Build synthetic lane-core versus plain greedy token and state check", "zig/tests/qwen27/lanes_check.zig", target, false, served); diff --git a/zig/src/core/lane_host.zig b/zig/src/core/lane_host.zig index 4a8f7d3493..fee64f995c 100644 --- a/zig/src/core/lane_host.zig +++ b/zig/src/core/lane_host.zig @@ -11,6 +11,7 @@ const admit = @import("lane_host_admit.zig"); const recalls = @import("lane_host_recall.zig"); const ends = @import("lane_host_end.zig"); const rows_host = @import("lane_host_rows.zig"); +const calls = @import("lane_host_calls.zig"); const Allocator = std.mem.Allocator; const Id = api.Id; const Request = api.Request; @@ -36,6 +37,8 @@ pub const LaneHost = struct { closing: bool = false, stale: bool = false, // the learner's last step: a put-back has not reached the device, so replies are refused thread: ?std.Thread = null, + owner_thread: ?std.Thread.Id = null, + calls: calls.Queue = .{}, decoded: std.ArrayList(status.Mark) = .empty, // tokens a round landed, for the 2 s decode rate prefill_rate: f64 = 0, prefill_at: i96 = 0, @@ -141,6 +144,11 @@ pub const LaneHost = struct { } }; } + /// Family work waits for the engine thread; ctx must remain valid until this returns, including cancellation. + pub fn callOwner(h: *LaneHost, ctx: *anyopaque, operation: calls.Run) calls.Error!void { + return calls.submit(h, ctx, operation); + } + /// The family's queue as a keepalive target, when it set one. fn keepaliveFn(ctx: *anyopaque) ?api.keepalive.Target { return self(ctx).keepalive_target; @@ -207,9 +215,9 @@ pub const LaneHost = struct { return job.host.foregroundWaiting() or job.host.core.activeCount() > 0; } - /// A foreground request is queued (they queue ahead of every background one); the caller holds the lock. + /// A foreground request or family call waits; the caller holds the lock. pub fn foregroundWaiting(h: *const LaneHost) bool { - return h.queued.items.len > 0 and !h.queued.items[0].request.background; + return h.calls.first != null or (h.queued.items.len > 0 and !h.queued.items[0].request.background); } /// Waits `warm_quiet` for other work (a request, a decoding stream, a cancel): true if none came. @@ -495,10 +503,14 @@ pub const LaneHost = struct { } fn run(h: *LaneHost) void { + h.lock(); + h.owner_thread = std.Thread.getCurrentId(); + h.unlock(); while (true) { ends.takeCancels(h); rows_host.answer(h); learn.refuse(h); + calls.one(h); while (h.admitOne()) {} h.fillOne(); status.noteLive(h); @@ -525,7 +537,7 @@ pub const LaneHost = struct { continue; } if (rows_host.idle(h)) continue; - if (h.cancels.items.len == 0 and (h.queued.items.len == 0 or h.admitted.items.len >= h.info_.lanes)) + if (h.calls.first == null and h.cancels.items.len == 0 and (h.queued.items.len == 0 or h.admitted.items.len >= h.info_.lanes)) h.wake.waitTimeout(h.io, &h.mutex, .{ .duration = .{ .raw = .fromMilliseconds(100), .clock = .awake } }) catch {}; h.unlock(); continue; diff --git a/zig/src/core/lane_host_calls.zig b/zig/src/core/lane_host_calls.zig new file mode 100644 index 0000000000..3d21c0a219 --- /dev/null +++ b/zig/src/core/lane_host_calls.zig @@ -0,0 +1,111 @@ +//! Family operations run on the engine thread between rounds, with caller-owned completion storage. +const std = @import("std"); +const LaneHost = @import("lane_host.zig").LaneHost; + +pub const Error = error{ Closed, NotStarted, ReentrantOwnerCall, StaleModel }; +pub const Run = *const fn (*anyopaque) void; + +const Call = struct { + next: ?*Call = null, + io: std.Io, + ctx: *anyopaque, + run: Run, + mutex: std.Io.Mutex = .init, + wake: std.Io.Condition = .init, + done: bool = false, + failure: ?Error = null, + + fn finish(c: *Call, failure: ?Error) void { + const io = c.io; + c.mutex.lockUncancelable(io); + c.failure = failure; + c.done = true; + c.wake.signal(io); + c.mutex.unlock(io); + // The caller can now return and destroy this node. Never touch c again. + } + + fn wait(c: *Call) Error!void { + const io = c.io; + c.mutex.lockUncancelable(io); + defer c.mutex.unlock(io); + while (!c.done) c.wake.waitUncancelable(io, &c.mutex); + if (c.failure) |err| return err; + } +}; + +/// Intrusive nodes stay on their callers' stacks until their own completion is observed. +pub const Queue = struct { + first: ?*Call = null, + last: ?*Call = null, + + fn append(q: *Queue, c: *Call) void { + if (q.last) |last| last.next = c else q.first = c; + q.last = c; + } + + fn pop(q: *Queue) ?*Call { + const c = q.first orelse return null; + q.first = c.next; + if (q.first == null) q.last = null; + c.next = null; + return c; + } +}; + +/// Once accepted, even a canceled caller waits until the owner has finished using ctx. +pub fn submit(h: *LaneHost, ctx: *anyopaque, run: Run) Error!void { + var c = Call{ .io = h.io, .ctx = ctx, .run = run }; + h.lock(); + if (h.closing) { + h.unlock(); + return error.Closed; + } + if (h.stale) { + h.unlock(); + return error.StaleModel; + } + if (h.thread == null) { + h.unlock(); + return error.NotStarted; + } + if (h.owner_thread == std.Thread.getCurrentId()) { + h.unlock(); + return error.ReentrantOwnerCall; + } + h.calls.append(&c); + h.wake.signal(h.io); + h.unlock(); + // No host access after enqueue: stop may free it while this caller observes its own completion. + try c.wait(); +} + +/// At most one operation per ordinary loop iteration; a running lesson reaches its safe end first. +pub fn one(h: *LaneHost) void { + h.lock(); + if (h.stale) { + const pending = detach(&h.calls); + h.unlock(); + return reject(pending, error.StaleModel); + } + const c = if (h.closing or h.lesson_open) null else h.calls.pop(); + h.unlock(); + const call = c orelse return; + call.run(call.ctx); + call.finish(null); +} + +/// Detach pending work under the host lock, then finish it without that lock or host storage. +pub fn detach(q: *Queue) ?*Call { + const first = q.first; + q.* = .{}; + return first; +} + +pub fn reject(first: ?*Call, failure: Error) void { + var next = first; + while (next) |c| { + next = c.next; + c.finish(failure); + } +} diff --git a/zig/src/core/lane_host_calls_test.zig b/zig/src/core/lane_host_calls_test.zig new file mode 100644 index 0000000000..274c624488 --- /dev/null +++ b/zig/src/core/lane_host_calls_test.zig @@ -0,0 +1,393 @@ +//! Owner calls through a real host thread: serialization, shutdown, reentrancy and chat fairness. +const std = @import("std"); +const lanes = @import("lanes"); +const api = @import("engine_api.zig"); +const LaneHost = @import("lane_host.zig").LaneHost; +const a = std.testing.allocator; +const io = std.testing.io; + +const Rig = struct { + cfg: lanes.Config = undefined, + target: lanes.fake.Fake = .{ .gpa = a }, + clock: lanes.fake.FixedClock = .{}, + core: lanes.Engine = undefined, + host: LaneHost = undefined, + stopped: bool = false, + + fn init(r: *Rig) !void { + r.cfg = try lanes.Config.init(a, .{ .exact_width = 8, .gpu_tokens = true, .hidden_rows = true }, 8, 7); + r.core = lanes.Engine.init(a, &r.cfg, r.target.backend(), r.clock.clock()); + r.host = LaneHost.init(a, io, &r.core, .{ .lanes = 2 }); + } + fn deinit(r: *Rig) void { + if (!r.stopped) r.host.stop(); + r.core.deinit(); + r.target.deinit(); + r.cfg.deinit(a); + } + fn stop(r: *Rig) void { + r.host.stop(); + r.stopped = true; + } +}; + +const Flag = std.atomic.Value(bool); +fn wait(flag: *const Flag) !void { + for (0..5000) |_| { + if (flag.load(.acquire)) return; + std.Io.sleep(io, .fromMilliseconds(1), .awake) catch {}; + } + return error.TestTimeout; +} + +const Threads = struct { + threads: [32]?std.Thread = @splat(null), + count: usize = 0, + fn add(t: *Threads, comptime f: anytype, args: anytype) !void { + t.threads[t.count] = try std.Thread.spawn(.{}, f, args); + t.count += 1; + } + fn join(t: *Threads) void { + for (t.threads[0..t.count]) |thread| thread.?.join(); + t.count = 0; + } +}; + +const Job = struct { + host: *LaneHost, + gate: ?*const Flag = null, + cancel: ?*const Flag = null, + entered: Flag = .init(false), + returned: Flag = .init(false), + ran: bool = false, + saw_cancel: bool = false, + reenter: bool = false, + failure: ?anyerror = null, + nested_failure: ?anyerror = null, + owner: ?std.Thread.Id = null, + counter: ?*std.atomic.Value(usize) = null, + observed_count: usize = 0, + + fn exec(ptr: *anyopaque) void { + const j: *Job = @ptrCast(@alignCast(ptr)); + j.owner = std.Thread.getCurrentId(); + if (j.counter) |counter| j.observed_count = counter.load(.acquire); + j.entered.store(true, .release); + if (j.gate) |gate| wait(gate) catch @panic("owner-call gate timed out"); + if (j.cancel) |cancel| j.saw_cancel = cancel.load(.acquire); + if (j.reenter) j.host.callOwner(j, exec) catch |err| { + j.nested_failure = err; + }; + j.ran = true; + } + fn run(j: *Job) void { + j.host.callOwner(j, exec) catch |err| { + j.failure = err; + }; + j.returned.store(true, .release); + } +}; + +fn pending(h: *LaneHost, want: usize) !void { + for (0..5000) |_| { + h.lock(); + var count: usize = 0; + var next = h.calls.first; + while (next) |c| : (next = c.next) count += 1; + h.unlock(); + if (count >= want) return; + std.Io.sleep(io, .fromMilliseconds(1), .awake) catch {}; + } + return error.TestTimeout; +} + +test "owner call refuses an unstarted host and owner reentrancy" { + var rig: Rig = .{}; + try rig.init(); + defer rig.deinit(); + var job = Job{ .host = &rig.host, .reenter = true }; + try std.testing.expectError(error.NotStarted, rig.host.callOwner(&job, Job.exec)); + try rig.host.start(); + try rig.host.callOwner(&job, Job.exec); + try std.testing.expectEqual(error.ReentrantOwnerCall, job.nested_failure.?); + try std.testing.expect(job.owner.? != std.Thread.getCurrentId()); + rig.stop(); + try std.testing.expectError(error.Closed, rig.host.callOwner(&job, Job.exec)); +} + +test "owner calls serialize and cancellation never releases a caller while its callback uses the context" { + var rig: Rig = .{}; + try rig.init(); + defer rig.deinit(); + try rig.host.start(); + var gate: Flag = .init(false); + var cancel: Flag = .init(false); + var threads: Threads = .{}; + defer { + gate.store(true, .release); + threads.join(); + } + var first = Job{ .host = &rig.host, .gate = &gate, .cancel = &cancel }; + var second = Job{ .host = &rig.host }; + try threads.add(Job.run, .{&first}); + try wait(&first.entered); + try threads.add(Job.run, .{&second}); + try pending(&rig.host, 1); + cancel.store(true, .release); + try std.testing.expect(!first.returned.load(.acquire) and !second.entered.load(.acquire)); + gate.store(true, .release); + threads.join(); + try std.testing.expect(first.ran and first.saw_cancel and second.ran); + try std.testing.expect(first.failure == null and second.failure == null); + try std.testing.expectEqual(first.owner, second.owner); +} + +test "owner stop rejects pending calls while active callback finishes before shutdown returns" { + var rig: Rig = .{}; + try rig.init(); + defer rig.deinit(); + try rig.host.start(); + var gate: Flag = .init(false); + var threads: Threads = .{}; + defer { + gate.store(true, .release); + threads.join(); + } + var first = Job{ .host = &rig.host, .gate = &gate }; + var second = Job{ .host = &rig.host }; + try threads.add(Job.run, .{&first}); + try wait(&first.entered); + try threads.add(Job.run, .{&second}); + try pending(&rig.host, 1); + try threads.add(Rig.stop, .{&rig}); + try wait(&second.returned); + try std.testing.expectEqual(error.Closed, second.failure.?); + try std.testing.expect(!second.ran and !first.returned.load(.acquire)); + gate.store(true, .release); + threads.join(); + try std.testing.expect(first.ran and first.failure == null and rig.stopped); +} + +test "engine drain rejects pending calls before joining an active owner callback" { + var rig: Rig = .{}; + try rig.init(); + defer rig.deinit(); + try rig.host.start(); + var gate: Flag = .init(false); + var threads: Threads = .{}; + defer { + gate.store(true, .release); + threads.join(); + } + var first = Job{ .host = &rig.host, .gate = &gate }; + var second = Job{ .host = &rig.host }; + try threads.add(Job.run, .{&first}); + try wait(&first.entered); + try threads.add(Job.run, .{&second}); + try pending(&rig.host, 1); + try threads.add(api.Engine.drain, .{rig.host.engine()}); + try wait(&second.returned); + try std.testing.expectEqual(error.Closed, second.failure.?); + try std.testing.expect(!second.ran and !first.returned.load(.acquire)); + gate.store(true, .release); + threads.join(); + try std.testing.expect(first.ran and first.failure == null); + try std.testing.expect(rig.host.thread == null); +} + +test "owner enqueue racing stop completes every accepted or refused caller" { + var rig: Rig = .{}; + try rig.init(); + defer rig.deinit(); + try rig.host.start(); + var threads: Threads = .{}; + defer threads.join(); + var jobs: [24]Job = undefined; + for (&jobs, 0..) |*job, i| { + job.* = .{ .host = &rig.host }; + try threads.add(Job.run, .{job}); + if (i == 12) try threads.add(Rig.stop, .{&rig}); + } + threads.join(); + for (jobs) |job| { + try std.testing.expect(job.returned.load(.acquire)); + if (job.failure) |err| try std.testing.expect(err == error.Closed) else try std.testing.expect(job.ran); + } +} + +const Box = struct { + done: Flag = .init(false), + tokens: std.atomic.Value(usize) = .init(0), + fn event(ptr: *anyopaque, _: api.Id, e: *const api.Event) void { + const b: *Box = @ptrCast(@alignCast(ptr)); + if (e.* == .finished) b.done.store(true, .release); + if (e.* == .tokens) _ = b.tokens.fetchAdd(e.tokens.len, .release); + } +}; + +test "owner queue lets a chat round run between two waiting family calls" { + var rig: Rig = .{}; + try rig.init(); + defer rig.deinit(); + try rig.host.start(); + var gate: Flag = .init(false); + var threads: Threads = .{}; + defer { + gate.store(true, .release); + threads.join(); + } + var box: Box = .{}; + var first = Job{ .host = &rig.host, .gate = &gate, .counter = &box.tokens }; + var second = Job{ .host = &rig.host, .counter = &box.tokens }; + try threads.add(Job.run, .{&first}); + try wait(&first.entered); + const request = api.Request{ .prompt = &.{ 2, 7, 1, 8 }, .max_tokens = 64 }; + try rig.host.engine().submit(1, &request, .{ .ctx = &box, .event = Box.event }); + try threads.add(Job.run, .{&second}); + try pending(&rig.host, 1); + gate.store(true, .release); + threads.join(); + try std.testing.expect(second.observed_count > first.observed_count and second.observed_count >= 2); + try wait(&box.done); +} + +test "owner call makes a running lone driver yield before its long reply ends" { + const Lone = struct { + backend: lanes.backend.Backend, + entered: Flag = .init(false), + yielded: Flag = .init(false), + fn run(ptr: *anyopaque, s: *lanes.Stream, hooks: api.LoneHooks) !bool { + const l: *@This() = @ptrCast(@alignCast(ptr)); + _ = try l.backend.opening(a, s); + hooks.committed(hooks.ctx); + s.pending = s.context.items[s.context.items.len - 1]; + s.cache_len = s.prompt_len; + l.entered.store(true, .release); + for (0..5000) |_| { + if (hooks.yield(hooks.ctx)) { + l.yielded.store(true, .release); + return true; + } + std.Io.sleep(io, .fromMilliseconds(1), .awake) catch {}; + } + return error.TestTimeout; + } + }; + var rig: Rig = .{}; + try rig.init(); + defer rig.deinit(); + var lone = Lone{ .backend = rig.target.backend() }; + rig.host.lone = .{ .ctx = &lone, .run = Lone.run }; + try rig.host.start(); + var box: Box = .{}; + const request = api.Request{ .prompt = &.{ 2, 7, 1, 8 }, .max_tokens = 64 }; + try rig.host.engine().submit(1, &request, .{ .ctx = &box, .event = Box.event }); + try wait(&lone.entered); + var job = Job{ .host = &rig.host, .counter = &box.tokens }; + try rig.host.callOwner(&job, Job.exec); + try std.testing.expect(lone.yielded.load(.acquire)); + try std.testing.expect(job.observed_count > 0 and job.observed_count < request.max_tokens); + try wait(&box.done); +} + +test "owner call waits for an open lesson without preventing that lesson from completing" { + const Learner = struct { + gate: Flag = .init(false), + entered: Flag = .init(false), + steps: std.atomic.Value(usize) = .init(0), + fn begin(_: *anyopaque, _: *const api.LearnRequest, _: api.LearnSink) !void { + return error.UnexpectedBegin; + } + fn step(ptr: *anyopaque) api.Learner.Step { + const l: *@This() = @ptrCast(@alignCast(ptr)); + if (l.steps.fetchAdd(1, .acq_rel) == 0) { + l.entered.store(true, .release); + wait(&l.gate) catch @panic("lesson gate timed out"); + return .{ .done = false, .changed = false }; + } + return .{ .done = true, .changed = false }; + } + fn abort(_: *anyopaque) void {} + }; + var rig: Rig = .{}; + try rig.init(); + defer rig.deinit(); + var learner: Learner = .{}; + rig.host.learner = .{ .ctx = &learner, .begin = Learner.begin, .step = Learner.step, .abort = Learner.abort }; + rig.host.lesson_open = true; + try rig.host.start(); + var threads: Threads = .{}; + defer { + learner.gate.store(true, .release); + if (!rig.stopped) rig.stop(); + threads.join(); + } + try wait(&learner.entered); + var job = Job{ .host = &rig.host, .counter = &learner.steps }; + try threads.add(Job.run, .{&job}); + try pending(&rig.host, 1); + learner.gate.store(true, .release); + try wait(&job.returned); + threads.join(); + try std.testing.expect(job.failure == null and job.observed_count >= 2); +} + +test "owner calls queued before a failed weight put-back are refused until the model heals" { + const Learner = struct { + gate: Flag = .init(false), + entered: Flag = .init(false), + healing: bool = false, + fn begin(ptr: *anyopaque, _: *const api.LearnRequest, _: api.LearnSink) !void { + const l: *@This() = @ptrCast(@alignCast(ptr)); + l.healing = true; + } + fn step(ptr: *anyopaque) api.Learner.Step { + const l: *@This() = @ptrCast(@alignCast(ptr)); + if (l.healing) return .{ .done = true, .changed = false, .stale = false }; + l.entered.store(true, .release); + wait(&l.gate) catch @panic("stale lesson gate timed out"); + return .{ .done = true, .changed = false, .stale = true }; + } + fn abort(_: *anyopaque) void {} + fn event(_: *anyopaque, _: *const api.LearnEvent) void {} + }; + var rig: Rig = .{}; + try rig.init(); + defer rig.deinit(); + var learner: Learner = .{}; + rig.host.learner = .{ .ctx = &learner, .begin = Learner.begin, .step = Learner.step, .abort = Learner.abort }; + rig.host.lesson_open = true; + try rig.host.start(); + var threads: Threads = .{}; + defer { + learner.gate.store(true, .release); + if (!rig.stopped) rig.stop(); + threads.join(); + } + try wait(&learner.entered); + var jobs = [_]Job{ .{ .host = &rig.host }, .{ .host = &rig.host } }; + for (&jobs) |*job| try threads.add(Job.run, .{job}); + try pending(&rig.host, 2); + learner.gate.store(true, .release); + for (&jobs) |*job| try wait(&job.returned); + threads.join(); + for (jobs) |job| { + try std.testing.expectEqual(error.StaleModel, job.failure.?); + try std.testing.expect(!job.ran); + } + var healed = Job{ .host = &rig.host }; + try std.testing.expectError(error.StaleModel, rig.host.callOwner(&healed, Job.exec)); + const heal: api.LearnRequest = .{}; + try rig.host.engine().learn(&heal, .{ .ctx = &learner, .event = Learner.event }); + var still_stale = true; + for (0..5000) |_| { + rig.host.lock(); + still_stale = rig.host.stale; + rig.host.unlock(); + if (!still_stale) break; + std.Io.sleep(io, .fromMilliseconds(1), .awake) catch {}; + } + try std.testing.expect(!still_stale); + try rig.host.callOwner(&healed, Job.exec); + try std.testing.expect(healed.ran); +} diff --git a/zig/src/core/lane_host_learn.zig b/zig/src/core/lane_host_learn.zig index dde774bfbd..b9b7224d50 100644 --- a/zig/src/core/lane_host_learn.zig +++ b/zig/src/core/lane_host_learn.zig @@ -28,7 +28,7 @@ pub fn refuse(h: *LaneHost) void { /// Learning runs only when nothing else waits: no stream, queued request or cancel (called under the lock). pub fn ready(h: *const LaneHost) bool { const work = h.lesson_open or h.lessons.items.len > 0; - return h.learner != null and work and h.queued.items.len == 0 and h.admitted.items.len == 0 and h.cancels.items.len == 0; + return h.learner != null and work and (h.lesson_open or h.calls.first == null) and h.queued.items.len == 0 and h.admitted.items.len == 0 and h.cancels.items.len == 0; } /// One unit of learning: open the next lesson if none is open, then step it; moved weights drop every kept prompt. diff --git a/zig/src/core/lane_host_rows.zig b/zig/src/core/lane_host_rows.zig index a45b9cfd2f..360a8e4f71 100644 --- a/zig/src/core/lane_host_rows.zig +++ b/zig/src/core/lane_host_rows.zig @@ -3,6 +3,7 @@ const std = @import("std"); const api = @import("engine_api.zig"); const LaneHost = @import("lane_host.zig").LaneHost; const rows = @import("prompt_cache.zig").rows; +const calls = @import("lane_host_calls.zig"); /// A planned stop's time to finish writing what waits. pub const stop_ns: i128 = 20 * std.time.ns_per_s; @@ -17,8 +18,10 @@ pub fn drain(ctx: *anyopaque) void { pub fn drainThread(h: *LaneHost) void { h.lock(); h.closing = true; + const pending = calls.detach(&h.calls); h.wake.broadcast(h.io); h.unlock(); + calls.reject(pending, error.Closed); if (h.thread) |t| t.join(); h.thread = null; } @@ -52,7 +55,7 @@ pub fn ready(h: *LaneHost) bool { const store = h.cache orelse return false; const r = store.disk orelse return false; if (!rows.write.pending(r)) return false; - if (h.queued.items.len > 0 or h.admitted.items.len > 0 or h.cancels.items.len > 0 or h.closing) return false; + if (h.calls.first != null or h.queued.items.len > 0 or h.admitted.items.len > 0 or h.cancels.items.len > 0 or h.closing) return false; return h.disk.quiet(h.admissions, std.Io.Clock.awake.now(h.io).toNanoseconds()); } diff --git a/zig/src/core/lane_host_test.zig b/zig/src/core/lane_host_test.zig index 85a34c0e15..2a6836a6fe 100644 --- a/zig/src/core/lane_host_test.zig +++ b/zig/src/core/lane_host_test.zig @@ -570,4 +570,5 @@ test { _ = @import("lane_host_borrow_test.zig"); _ = @import("lane_host_warm_test.zig"); _ = @import("lane_host_rows_test.zig"); + _ = @import("lane_host_calls_test.zig"); } diff --git a/zig/src/families/clef/README.md b/zig/src/families/clef/README.md index dd894c54ee..85cef8a083 100644 --- a/zig/src/families/clef/README.md +++ b/zig/src/families/clef/README.md @@ -1,8 +1,10 @@ # Clef first-slice work -Local work for [issue 692](https://github.com/ashhart/TensorFold/issues/692), not ready for a PR or deployment. The bf16 head reader is separate from the qwen27 backbone. The original 45-record, 128-question layout gate is still required; the older 15-record prototype does not replace it. The encoding and head port retain the credit for outcastofmusic's #241 and #251. +Work for [issue 692](https://github.com/ashhart/TensorFold/issues/692), not ready for an upstream PR or deployment. The bf16 head reader is separate from the qwen27 backbone. The original 45-record, 128-question layout gate is still required; the older 15-record prototype does not replace it. The encoding and head port retain the credit for outcastofmusic's #241 and #251. -`qwen27.hidden_rows.captureExclusive` returns owned bf16 final normalized rows in token order. It runs the existing prompt kernels with a fresh frame and fresh GDN/KV runner. It copies `frame.input` after each completed `advance(.none)` and restores the caller's frame and previous prompt flag on return. The caller must own the model for the entire call. This function is not wired to HTTP requests or the serving queue yet. +`qwen27.hidden_rows.captureExclusive` returns owned bf16 final normalized rows in token order. It runs the existing prompt kernels with a fresh frame and fresh GDN/KV runner. It copies `frame.input` after each completed `advance(.none)` and restores the caller's frame and previous prompt flag on return. The caller must own the model for the entire call. + +`qwen27_served.Host.hiddenRows` queues this operation on the engine thread between chat rounds. Each caller keeps its own completion storage alive until the operation ends. Pending calls make lone and background drivers yield. An open learning transaction finishes before capture starts. Stop rejects queued calls and waits for an active call to finish safely. A complete capture can pause chat for its duration; the queue does not promise a short pause. HTTP decision scoring is not connected yet. The ANE handoff reserves storage before input encoding, then registers and publishes each committed command buffer without a fallible allocation. The worker can release an even wait after a terminal input-CB failure even when the odd signal never arrived. Drain waits for every registered CB and the last worker completion before releasing retained handles. A poisoned share stays unavailable until explicitly rebuilt. Restoring the chat frame after such a device failure does not guarantee that the server can continue serving. @@ -17,6 +19,10 @@ zig fmt --check build.zig zig The targeted check passes 27 tests: head admission, hidden-row bounds and the actual ANE worker/drain protocol. Its two-thread ring test publishes 1,024 tickets through a ring of four entries. The whole host build passes 162 steps; some tests are cached, so no aggregate fresh-test count is claimed. Goldens have zero unexpected differences. Lean has 527 inherited findings and zero new. +Seven additional owner-call cases run with the host tests: owner identity/reentrancy, serialization, cancellation lifetime, stop and enqueue races, chat progress between calls, lone-driver yield and completion of an open lesson. Mutants that remove lone yield, block an open lesson behind a call, or serve two calls before chat each fail their corresponding regression test. + `zig build tf-qwen27-hidden-rows-check` builds the synthetic device harness without executing it. The harness compares capture against normal prompt arithmetic with the LM head projected; this proves capture parity, not independent Clef numerical accuracy. It also checks output ownership, chat/snapshot continuation, cancellation, allocation failure and nonfinite final rows. Late cancel/OOM triggers must observe the CPU output of an already completed first chunk. Its optional `--ane` argument exercises the ANE path. -The device harness has only been compiled. Run it only with exclusive use of the Metal engine. Pending gates include ANE on/off runs, device-fault behavior, performance against the pinned base, queued concurrent-request isolation, real-model chat/resume parity, the original layout corpus and an agreed numerical head tolerance. No probability tolerance has been selected. +`zig build tf-qwen27-hidden-rows-served-check` builds a second device harness. It queues three captures while a synthetic chat is open, compares their rows with a solo capture and compares all 64 chat tokens with CLI generation. Its optional `--ane` argument keeps the same capture path with the Neural Engine enabled. + +Both device harnesses have only been compiled. Run them only with exclusive use of the Metal engine. Pending gates include ANE on/off runs, device-fault behavior, performance against the pinned base, queued concurrent-request isolation on a device, real-model chat/resume parity, the original layout corpus and an agreed numerical head tolerance. No probability tolerance has been selected. diff --git a/zig/src/families/qwen27/hidden_rows.zig b/zig/src/families/qwen27/hidden_rows.zig index fc102be11d..1143d2f32a 100644 --- a/zig/src/families/qwen27/hidden_rows.zig +++ b/zig/src/families/qwen27/hidden_rows.zig @@ -1,5 +1,6 @@ //! Final normalized prompt rows, with their own frame and fresh recurrent/KV state for each call. const std = @import("std"); +const mtl = @import("metal"); const Model = @import("model.zig").Model; const Frame = @import("gpu_frame.zig").Frame; const Runner = @import("decode_round.zig").Runner; @@ -33,6 +34,8 @@ pub const Rows = struct { pub fn captureExclusive(a: std.mem.Allocator, m: *Model, ids: []const u32, o: Options) !Rows { const p = try Plan.init(ids, m.config.hidden, m.config.vocab, m.frame.capacity, o.context, o.chunk); try o.check(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); const out = try a.alloc(u16, p.values); errdefer a.free(out); var frame = try Frame.init(m.device, m.config, m.frame.capacity); diff --git a/zig/src/native/qwen27_served.zig b/zig/src/native/qwen27_served.zig index 168d9304c8..0ce69baded 100644 --- a/zig/src/native/qwen27_served.zig +++ b/zig/src/native/qwen27_served.zig @@ -49,6 +49,29 @@ pub const Host = struct { return h.host.engine(); } + /// Capture on the owner thread, between chat rounds; output stays owned by the calling allocator. + pub fn hiddenRows(h: *Host, a: Allocator, ids: []const u32, chunk: u32, cancel: ?*const std.atomic.Value(bool)) !q.hidden_rows.Rows { + const Call = struct { + a: Allocator, + model: *q.model.Model, + ids: []const u32, + options: q.hidden_rows.Options, + rows: ?q.hidden_rows.Rows = null, + failure: ?anyerror = null, + fn run(ptr: *anyopaque) void { + const c: *@This() = @ptrCast(@alignCast(ptr)); + c.rows = q.hidden_rows.captureExclusive(c.a, c.model, c.ids, c.options) catch |err| { + c.failure = err; + return; + }; + } + }; + var call = Call{ .a = a, .model = h.model, .ids = ids, .options = .{ .context = h.host.info_.context_window, .chunk = chunk, .cancel = cancel } }; + try h.host.callOwner(&call, Call.run); + if (call.failure) |err| return err; + return call.rows.?; + } + fn of(ptr: *anyopaque) *Host { return @ptrCast(@alignCast(ptr)); } diff --git a/zig/tests/qwen27/hidden_rows_served.zig b/zig/tests/qwen27/hidden_rows_served.zig new file mode 100644 index 0000000000..33d81d21a0 --- /dev/null +++ b/zig/tests/qwen27/hidden_rows_served.zig @@ -0,0 +1,143 @@ +//! Queued captures beside a real synthetic chat; build-only until an exclusive Metal window. +const std = @import("std"); +const mtl = @import("metal"); +const api = @import("engine_api"); +const q = @import("tensorfold").qwen27; +const served = @import("qwen27_served"); +const graph = @import("synthetic_graph.zig"); +const Flag = std.atomic.Value(bool); + +fn wait(io: std.Io, flag: *const Flag) !void { + for (0..30_000) |_| { + if (flag.load(.acquire)) return; + try std.Io.sleep(io, .fromMilliseconds(1), .awake); + } + return error.WaitTimedOut; +} + +const Capture = struct { + host: *served.Host, + a: std.mem.Allocator, + ids: []const u32, + chunk: u32, + rows: ?q.hidden_rows.Rows = null, + failure: ?anyerror = null, + fn run(c: *Capture) void { + c.rows = c.host.hiddenRows(c.a, c.ids, c.chunk, null) catch |err| { + c.failure = err; + return; + }; + } +}; + +const Box = struct { + a: std.mem.Allocator, + io: std.Io, + mutex: std.Io.Mutex = .init, + prefilled: Flag = .init(false), + release: Flag = .init(false), + done: Flag = .init(false), + bad: Flag = .init(false), + tokens: std.ArrayList(u32) = .empty, + reason: ?api.Reason = null, + + fn event(ptr: *anyopaque, _: api.Id, e: *const api.Event) void { + const b: *Box = @ptrCast(@alignCast(ptr)); + if (e.* == .prefilled) { + b.prefilled.store(true, .release); + wait(b.io, &b.release) catch b.bad.store(true, .release); + return; + } + b.mutex.lockUncancelable(b.io); + defer b.mutex.unlock(b.io); + switch (e.*) { + .tokens => |ids| b.tokens.appendSlice(b.a, ids) catch b.bad.store(true, .release), + .finished => |f| { + b.reason = f.reason; + b.done.store(true, .release); + }, + else => {}, + } + } +}; + +pub fn main(init: std.process.Init) !void { + var arena = std.heap.ArenaAllocator.init(init.gpa); + defer arena.deinit(); + const a = arena.allocator(); + const shared = std.heap.smp_allocator; + const args = try init.minimal.args.toSlice(a); + if (args.len > 2 or (args.len == 2 and !std.mem.eql(u8, args[1], "--ane"))) return error.Usage; + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const m = try graph.create(a); + defer m.deinit(); + if (args.len == 2) m.ane = try q.ane_mlp.Share.init(a, m.device, m.queue, &m.checkpoint, m.config, 64); + var ids: [130]u32 = undefined; + for (&ids, 0..) |*id, i| id.* = @intCast((i * 17 + 7) % 96); + const chat = ids[0..11]; + var expected: [64]u32 = undefined; + { + var runner = try q.decode_round.Runner.init(a, m, 1, 256); + defer runner.deinit(); + const session = q.session.Session{ .runner = &runner }; + try session.prefill(chat, 128); + for (&expected, 0..) |*id, i| { + id.* = try session.greedy(); + if (i + 1 < expected.len) try session.step(id.*); + } + } + var why: []const u8 = ""; + const h = try served.attach(shared, init.io, m, null, .{ .context = 256, .streams = 2, .cache_gib = 0 }, a, &why); + var host_open = true; + defer if (host_open) served.close(h); + var solo = try h.hiddenRows(shared, &ids, 16, null); + defer solo.deinit(); + var box = Box{ .a = shared, .io = init.io }; + defer box.tokens.deinit(shared); + var jobs: [3]Capture = undefined; + var threads: [3]?std.Thread = @splat(null); + defer { + box.release.store(true, .release); + for (&threads) |*thread| if (thread.*) |t| { + t.join(); + thread.* = null; + }; + served.close(h); + host_open = false; + for (&jobs) |*job| if (job.rows) |*rows| rows.deinit(); + } + for (&jobs, [_]u32{ 1, 16, 128 }) |*job, chunk| job.* = .{ .host = h, .a = shared, .ids = &ids, .chunk = chunk }; + const request = api.Request{ .prompt = chat, .max_tokens = expected.len, .drafts = false }; + try h.engine().submit(1, &request, .{ .ctx = &box, .event = Box.event }); + try wait(init.io, &box.prefilled); + for (&threads, &jobs) |*thread, *job| thread.* = try std.Thread.spawn(.{}, Capture.run, .{job}); + var queued = false; + for (0..30_000) |_| { + h.host.lock(); + var count: usize = 0; + var next = h.host.calls.first; + while (next) |c| : (next = c.next) count += 1; + h.host.unlock(); + if (count == jobs.len) { + queued = true; + break; + } + try std.Io.sleep(init.io, .fromMilliseconds(1), .awake); + } + if (!queued) return error.CapturesNotQueuedDuringChat; + box.release.store(true, .release); + for (&threads) |*thread| { + thread.*.?.join(); + thread.* = null; + } + for (jobs) |job| { + if (job.failure) |err| return err; + if (!std.mem.eql(u16, solo.values, job.rows.?.values)) return error.ConcurrentCaptureDiffers; + } + try wait(init.io, &box.done); + box.mutex.lockUncancelable(init.io); + defer box.mutex.unlock(init.io); + if (box.bad.load(.acquire) or box.reason != .length or !std.mem.eql(u32, &expected, box.tokens.items)) return error.ChatReplyChanged; + std.debug.print("queued hidden rows: 3 concurrent captures equal solo; 64 chat tokens equal CLI; ANE={}\n", .{m.ane != null}); +} From 812fc83e04cb53c7be0938d522287bdcb0a1fc27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schr=C3=B6der?= <1188270+cwschroeder@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:15:09 +0200 Subject: [PATCH 5/8] Inspect optional Clef heads without opening the backbone --- .github/workflows/no-gpu.yml | 4 + build.zig | 7 ++ tools/zig/test_clef_inspect.py | 79 ++++++++++++++++++++ zig/clef_checkpoint_cases.zig | 1 + zig/clef_inspect.zig | 44 +++++++++++ zig/src/families/clef/README.md | 20 ++++- zig/src/families/clef/checkpoint.zig | 28 +++++++ zig/src/families/clef/checkpoint_test.zig | 89 +++++++++++++++++++++++ 8 files changed, 269 insertions(+), 3 deletions(-) create mode 100644 tools/zig/test_clef_inspect.py create mode 100644 zig/clef_inspect.zig create mode 100644 zig/src/families/clef/checkpoint.zig create mode 100644 zig/src/families/clef/checkpoint_test.zig diff --git a/.github/workflows/no-gpu.yml b/.github/workflows/no-gpu.yml index 597ff6b522..a4c558e9dc 100644 --- a/.github/workflows/no-gpu.yml +++ b/.github/workflows/no-gpu.yml @@ -24,6 +24,10 @@ jobs: # The kernel contract's limits gate: every registered launch against M1-M5 and sm_80/86/89/120/121, controls red. - name: zig build test-kernel-contract run: zig build test-kernel-contract + - name: Clef head inspection CLI (CPU only) + run: | + zig build tf-clef-inspect + python3 tools/zig/test_clef_inspect.py zig-out/bin/tf-clef-inspect # The frozen server goldens: request parsing, rendering and streaming against recorded answers. - name: zig build test-golden run: zig build test-golden diff --git a/build.zig b/build.zig index c5039618d7..085f30381a 100644 --- a/build.zig +++ b/build.zig @@ -170,6 +170,13 @@ pub fn build(b: *std.Build) void { const clef_head_run = b.addRunArtifact(clef_head); b.step("test-clef-checkpoint", "CPU Clef checkpoint and disposable-pass contracts").dependOn(&clef_head_run.step); test_step.dependOn(&clef_head_run.step); + const clef_inspect = b.addExecutable(.{ .name = "tf-clef-inspect", .root_module = b.createModule(.{ + .root_source_file = b.path("zig/clef_inspect.zig"), + .target = target, + .optimize = .safe, + .link_libc = true, + }) }); + b.step("tf-clef-inspect", "Build CPU-only Clef head and backbone-config inspection").dependOn(&b.addInstallArtifact(clef_inspect, .{}).step); switch (target.result.os.tag) { .macos => metalTargets(b, target, optimize, draft_ids, build_options, test_step), .linux => cuda_build.targets(b, target, optimize, draft_ids, build_options), diff --git a/tools/zig/test_clef_inspect.py b/tools/zig/test_clef_inspect.py new file mode 100644 index 0000000000..06111a8b0c --- /dev/null +++ b/tools/zig/test_clef_inspect.py @@ -0,0 +1,79 @@ +"""CPU CLI receipts distinguish a validated head from a plain or damaged checkpoint.""" +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +BINARY = Path(sys.argv.pop(1)).resolve() if len(sys.argv) > 1 and not sys.argv[1].startswith("-") else ROOT / "zig-out/bin/tf-clef-inspect" +FIXTURE = ROOT / "zig/tests/fixtures/clef" +CONFIG = { + "model_type": "qwen3_5_text", "hidden_size": 16, "intermediate_size": 32, + "num_hidden_layers": 4, "num_attention_heads": 2, "num_key_value_heads": 1, + "head_dim": 8, "vocab_size": 97, "linear_num_key_heads": 2, + "linear_num_value_heads": 4, "linear_key_head_dim": 4, + "linear_value_head_dim": 4, "linear_conv_kernel_dim": 4, "eos_token_id": 2, +} + + +class InspectTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.path = Path(self.tmp.name) + (self.path / "config.json").write_text(json.dumps(CONFIG)) + + def pair(self): + for name in ("joint_head_config.json", "joint_head.safetensors"): + shutil.copyfile(FIXTURE / name, self.path / name) + + def run_cli(self): + return subprocess.run([str(BINARY), str(self.path), "--check"], text=True, capture_output=True, timeout=30) + + def rejected(self): + result = self.run_cli() + self.assertNotEqual(0, result.returncode) + self.assertEqual("", result.stdout, "a failed check must not emit a success receipt") + + def test_complete_pair_has_verified_payload_hash_and_no_model_readiness_claim(self): + self.pair() + result = self.run_cli() + self.assertEqual(0, result.returncode, result.stderr) + record = json.loads(result.stdout) + self.assertTrue(record["head_present"] and record["head_validated"]) + self.assertEqual(122, record["head_tensors"]) + self.assertEqual(16, record["backbone_hidden"]) + self.assertEqual(8, record["head_config"]["width"]) + self.assertEqual(hashlib.sha256((self.path / "joint_head.safetensors").read_bytes()).hexdigest(), record["head_file_sha256"]) + self.assertFalse(record["backbone_weights_checked"] or record["gpu"] or record["model_ready"]) + + def test_plain_pack_has_no_validated_head_or_hash(self): + result = self.run_cli() + self.assertEqual(0, result.returncode, result.stderr) + record = json.loads(result.stdout) + self.assertFalse(record["head_present"] or record["head_validated"] or record["model_ready"]) + self.assertEqual(0, record["head_tensors"]) + self.assertIsNone(record["head_config"]) + self.assertIsNone(record["head_file_sha256"]) + + def test_incomplete_pair_and_truncated_payload_are_errors(self): + shutil.copyfile(FIXTURE / "joint_head_config.json", self.path / "joint_head_config.json") + self.rejected() + self.pair() + payload = self.path / "joint_head.safetensors" + payload.write_bytes(payload.read_bytes()[:-2]) + self.rejected() + + def test_incompatible_backbone_is_an_error(self): + self.pair() + (self.path / "config.json").write_text(json.dumps({**CONFIG, "hidden_size": 32})) + self.rejected() + + +if __name__ == "__main__": + unittest.main() diff --git a/zig/clef_checkpoint_cases.zig b/zig/clef_checkpoint_cases.zig index fc8bc57da6..652325692a 100644 --- a/zig/clef_checkpoint_cases.zig +++ b/zig/clef_checkpoint_cases.zig @@ -1,6 +1,7 @@ //! CPU-only Clef head admission and disposable-pass contracts; no device or scoring arithmetic. test { _ = @import("src/families/clef/weights.zig"); + _ = @import("src/families/clef/checkpoint.zig"); _ = @import("src/families/qwen27/ane_handoff.zig"); _ = @import("src/families/qwen27/hidden_rows_plan.zig"); } diff --git a/zig/clef_inspect.zig b/zig/clef_inspect.zig new file mode 100644 index 0000000000..4ea2763254 --- /dev/null +++ b/zig/clef_inspect.zig @@ -0,0 +1,44 @@ +//! CPU inspection of a separate Clef head against its backbone config; no backbone payload or device is opened. +const std = @import("std"); +const checkpoint = @import("src/families/clef/checkpoint.zig"); +const qwen = @import("src/families/qwen27/config.zig"); + +fn digest(bytes: []const u8) [64]u8 { + var sum: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(bytes, &sum, .{}); + return std.fmt.bytesToHex(sum, .lower); +} + +pub fn main(init: std.process.Init) !void { + const a = init.arena.allocator(); + const io = init.io; + const args = try init.minimal.args.toSlice(a); + if (args.len != 3 or !std.mem.eql(u8, args[2], "--check")) return error.CheckModeRequired; + const path = try std.fs.path.join(a, &.{ args[1], "config.json" }); + const text = try std.Io.Dir.cwd().readFileAlloc(io, path, a, .limited(1 << 20)); + const c = try qwen.parse(a, text, null); + var head = try checkpoint.loadOptional(init.gpa, io, args[1], c.hidden); + defer if (head) |*h| h.deinit(io); + const config_sha = digest(text); + var head_sha: [64]u8 = undefined; + const sha: ?[]const u8 = if (head) |h| blk: { + head_sha = digest(h.file.map.memory); + break :blk &head_sha; + } else null; + const record = try std.json.Stringify.valueAlloc(a, .{ + .kind = "clef_head_checkpoint", + .version = 1, + .head_present = head != null, + .head_validated = head != null, + .head_tensors = if (head) |h| h.file.names.count() else @as(usize, 0), + .head_config = if (head) |h| h.config else null, + .head_file_sha256 = sha, + .backbone_hidden = c.hidden, + .backbone_config_sha256 = @as([]const u8, &config_sha), + .backbone_weights_checked = false, + .gpu = false, + .model_ready = false, + }, .{}); + try std.Io.File.stdout().writeStreamingAll(io, record); + try std.Io.File.stdout().writeStreamingAll(io, "\n"); +} diff --git a/zig/src/families/clef/README.md b/zig/src/families/clef/README.md index 85cef8a083..df7e24724b 100644 --- a/zig/src/families/clef/README.md +++ b/zig/src/families/clef/README.md @@ -2,9 +2,23 @@ Work for [issue 692](https://github.com/ashhart/TensorFold/issues/692), not ready for an upstream PR or deployment. The bf16 head reader is separate from the qwen27 backbone. The original 45-record, 128-question layout gate is still required; the older 15-record prototype does not replace it. The encoding and head port retain the credit for outcastofmusic's #241 and #251. +`checkpoint.loadOptional` returns no head only when both head sidecars are absent. A missing model directory, incomplete pair, broken symlink, wrong file type or invalid head remains an error. Valid Hugging Face cache symlinks are accepted. The caller owns a returned head and must call `deinit`. + +The CPU inspector checks the separate head against the hidden width in the backbone's `config.json`: + +```sh +zig build tf-clef-inspect +zig-out/bin/tf-clef-inspect /path/to/checkpoint --check +python3 tools/zig/test_clef_inspect.py zig-out/bin/tf-clef-inspect +``` + +Its JSON includes the head's tensor count, geometry and file SHA256, plus the backbone config SHA256. A plain pack reports no head. A failed check exits with an error and emits no success JSON. The inspector opens no backbone weights or GPU; `backbone_weights_checked`, `gpu` and `model_ready` remain false. Head detection is not connected to the serving path yet. + `qwen27.hidden_rows.captureExclusive` returns owned bf16 final normalized rows in token order. It runs the existing prompt kernels with a fresh frame and fresh GDN/KV runner. It copies `frame.input` after each completed `advance(.none)` and restores the caller's frame and previous prompt flag on return. The caller must own the model for the entire call. -`qwen27_served.Host.hiddenRows` queues this operation on the engine thread between chat rounds. Each caller keeps its own completion storage alive until the operation ends. Pending calls make lone and background drivers yield. An open learning transaction finishes before capture starts. Stop rejects queued calls and waits for an active call to finish safely. A complete capture can pause chat for its duration; the queue does not promise a short pause. HTTP decision scoring is not connected yet. +Capture currently accepts text token IDs. It has no image input. Isolation after a real vision-chat prefill requires its own device test. + +`qwen27_served.Host.hiddenRows` queues this operation on the engine thread between chat rounds. Each caller keeps its own completion storage alive until the operation ends. Pending calls make lone and background drivers yield. An open learning transaction finishes before capture starts. Both drain and stop reject queued calls and wait for an active call to finish safely. A failed learning put-back refuses calls with `StaleModel` until the model heals. A complete capture can pause chat for its duration; the queue does not promise a short pause. HTTP decision scoring is not connected yet. The ANE handoff reserves storage before input encoding, then registers and publishes each committed command buffer without a fallible allocation. The worker can release an even wait after a terminal input-CB failure even when the odd signal never arrived. Drain waits for every registered CB and the last worker completion before releasing retained handles. A poisoned share stays unavailable until explicitly rebuilt. Restoring the chat frame after such a device failure does not guarantee that the server can continue serving. @@ -17,9 +31,9 @@ zig build test-golden fuzz-cortheon zig fmt --check build.zig zig ``` -The targeted check passes 27 tests: head admission, hidden-row bounds and the actual ANE worker/drain protocol. Its two-thread ring test publishes 1,024 tickets through a ring of four entries. The whole host build passes 162 steps; some tests are cached, so no aggregate fresh-test count is claimed. Goldens have zero unexpected differences. Lean has 527 inherited findings and zero new. +The targeted check passes 34 tests: head admission, optional-head detection, hidden-row bounds and the actual ANE worker/drain protocol. Its two-thread ring test publishes 1,024 tickets through a ring of four entries. The CLI has four additional CPU tests. Whole-tree validation must be repeated after each upstream rebase. -Seven additional owner-call cases run with the host tests: owner identity/reentrancy, serialization, cancellation lifetime, stop and enqueue races, chat progress between calls, lone-driver yield and completion of an open lesson. Mutants that remove lone yield, block an open lesson behind a call, or serve two calls before chat each fail their corresponding regression test. +Nine additional owner-call cases run with the host tests: owner identity/reentrancy, serialization, cancellation lifetime, stop and enqueue races, chat progress between calls, lone-driver yield, completion of an open lesson, direct engine drain and stale-weight refusal. Mutants that remove lone yield, block an open lesson behind a call, or serve two calls before chat each fail their corresponding regression test. `zig build tf-qwen27-hidden-rows-check` builds the synthetic device harness without executing it. The harness compares capture against normal prompt arithmetic with the LM head projected; this proves capture parity, not independent Clef numerical accuracy. It also checks output ownership, chat/snapshot continuation, cancellation, allocation failure and nonfinite final rows. Late cancel/OOM triggers must observe the CPU output of an already completed first chunk. Its optional `--ane` argument exercises the ANE path. diff --git a/zig/src/families/clef/checkpoint.zig b/zig/src/families/clef/checkpoint.zig new file mode 100644 index 0000000000..16167160b8 --- /dev/null +++ b/zig/src/families/clef/checkpoint.zig @@ -0,0 +1,28 @@ +//! A joint head is optional only when both sidecars are absent; damaged decision packs never become chat packs. +const std = @import("std"); +const Weights = @import("weights.zig").Weights; + +fn present(dir: std.Io.Dir, io: std.Io, name: []const u8) !bool { + const entry = dir.statFile(io, name, .{ .follow_symlinks = false }) catch |err| switch (err) { + error.FileNotFound => return false, + else => return err, + }; + const file = if (entry.kind == .sym_link) try dir.statFile(io, name, .{}) else entry; + if (file.kind != .file) return error.InvalidHeadSidecar; + return true; +} + +/// Plain chat packs return null; a complete decision pack owns a fully validated separate bf16 head. +pub fn loadOptional(a: std.mem.Allocator, io: std.Io, path: []const u8, backbone_hidden: usize) !?Weights { + var dir = try std.Io.Dir.cwd().openDir(io, path, .{}); + defer dir.close(io); + const config = try present(dir, io, "joint_head_config.json"); + const weights = try present(dir, io, "joint_head.safetensors"); + if (!config and !weights) return null; + if (!config or !weights) return error.IncompleteJointHead; + return try Weights.load(a, io, path, backbone_hidden); +} + +test { + _ = @import("checkpoint_test.zig"); +} diff --git a/zig/src/families/clef/checkpoint_test.zig b/zig/src/families/clef/checkpoint_test.zig new file mode 100644 index 0000000000..fe2c3501d8 --- /dev/null +++ b/zig/src/families/clef/checkpoint_test.zig @@ -0,0 +1,89 @@ +//! Presence, corruption and cache-symlink checks keep decision checkpoints from silently taking the chat path. +const std = @import("std"); +const checkpoint = @import("checkpoint.zig"); +const a = std.testing.allocator; +const io = std.testing.io; +const config = @embedFile("../../../tests/fixtures/clef/joint_head_config.json"); +const weights = @embedFile("../../../tests/fixtures/clef/joint_head.safetensors"); + +fn loadClose(gpa: std.mem.Allocator, path: []const u8) !void { + var head = (try checkpoint.loadOptional(gpa, io, path, 16)) orelse return error.ExpectedHead; + defer head.deinit(io); +} + +test "a missing model directory is not a plain checkpoint" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const path = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(path); + const missing = try std.fs.path.join(a, &.{ path, "missing-model" }); + defer a.free(missing); + try std.testing.expectError(error.FileNotFound, checkpoint.loadOptional(a, io, missing, 16)); +} + +test "a plain pack has neither head sidecar, not an incomplete pair" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const path = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(path); + try std.testing.expect((try checkpoint.loadOptional(a, io, path, 16)) == null); + try tmp.dir.writeFile(io, .{ .sub_path = "joint_head_config.json", .data = config }); + try std.testing.expectError(error.IncompleteJointHead, checkpoint.loadOptional(a, io, path, 16)); + try tmp.dir.deleteFile(io, "joint_head_config.json"); + try tmp.dir.writeFile(io, .{ .sub_path = "joint_head.safetensors", .data = weights }); + try std.testing.expectError(error.IncompleteJointHead, checkpoint.loadOptional(a, io, path, 16)); +} + +test "a complete pair uses the same geometry and finite tensor checks as the strict reader" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const path = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(path); + try tmp.dir.writeFile(io, .{ .sub_path = "joint_head_config.json", .data = config }); + try tmp.dir.writeFile(io, .{ .sub_path = "joint_head.safetensors", .data = weights }); + try loadClose(a, path); + try std.testing.expectError(error.IncompatibleHeadBackbone, checkpoint.loadOptional(a, io, path, 5120)); + try tmp.dir.writeFile(io, .{ .sub_path = "joint_head.safetensors", .data = weights[0 .. weights.len - 2] }); + if (checkpoint.loadOptional(a, io, path, 16)) |head_| { + if (head_) |head__| { + var head = head__; + head.deinit(io); + } + return error.CorruptPackAccepted; + } else |_| {} +} + +test "head sidecar directories fail, while valid cache symlinks remain supported" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const path = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(path); + try tmp.dir.createDir(io, "joint_head_config.json", .default_dir); + try std.testing.expectError(error.InvalidHeadSidecar, checkpoint.loadOptional(a, io, path, 16)); + try tmp.dir.deleteDir(io, "joint_head_config.json"); + try tmp.dir.writeFile(io, .{ .sub_path = "config-blob", .data = config }); + try tmp.dir.writeFile(io, .{ .sub_path = "weights-blob", .data = weights }); + try tmp.dir.symLink(io, "config-blob", "joint_head_config.json", .{}); + try tmp.dir.symLink(io, "weights-blob", "joint_head.safetensors", .{}); + try loadClose(a, path); +} + +test "a dangling head symlink is a broken pack even when neither target exists" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const path = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(path); + try tmp.dir.symLink(io, "missing-config", "joint_head_config.json", .{}); + try tmp.dir.symLink(io, "missing-weights", "joint_head.safetensors", .{}); + try std.testing.expectError(error.FileNotFound, checkpoint.loadOptional(a, io, path, 16)); +} + +test "optional head loading preserves allocation failures and releases its map" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const path = try tmp.dir.realPathFileAlloc(io, ".", a); + defer a.free(path); + try tmp.dir.writeFile(io, .{ .sub_path = "joint_head_config.json", .data = config }); + try tmp.dir.writeFile(io, .{ .sub_path = "joint_head.safetensors", .data = weights }); + try std.testing.checkAllAllocationFailures(a, loadClose, .{path}); +} From b0b630f2617b432ddc9f833d843c44850c5bb851 Mon Sep 17 00:00:00 2001 From: chaog992 <338238273+chaog992@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:21:23 +0800 Subject: [PATCH 6/8] test(lanes): the adoption tests check each whole reply and its finish reason expectDecode compared only the tokens a reply held with the fake target's decode, so a reply that ended early passed. It now builds the decode to the case's max_new or first eos token and compares the whole reply and the stream's reason; like expectRows, it takes the case. --- zig/src/core/lanes/engine_test.zig | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/zig/src/core/lanes/engine_test.zig b/zig/src/core/lanes/engine_test.zig index bb32189fa3..d5415bf5c8 100644 --- a/zig/src/core/lanes/engine_test.zig +++ b/zig/src/core/lanes/engine_test.zig @@ -445,15 +445,22 @@ test "structured output: where the grammar spells its own text, its drafts fill } } -/// Each token the fake target draws one at a time after `prompt`, as `got` must hold them. -fn expectDecode(prompt: []const u32, got: []const u32) !void { +/// The fake target's one-token decode after `c.prompt`, to `c.max_new` tokens or an eos one: `s`'s reply and reason. +fn expectDecode(c: Case, s: *const sm.Stream) !void { var history: std.ArrayList(u32) = .empty; defer history.deinit(gpa); - try history.appendSlice(gpa, prompt); - for (got) |t| { - try std.testing.expectEqual(fake.next(history.items, null, history.items.len), t); + try history.appendSlice(gpa, c.prompt); + var reason: sm.Reason = .length; + while (history.items.len - c.prompt.len < c.max_new) { + const t = fake.next(history.items, c.sampling, history.items.len); try history.append(gpa, t); + if (std.mem.indexOfScalar(u32, c.eos, t) != null) { + reason = .stop; + break; + } } + try std.testing.expectEqualSlices(u32, history.items[c.prompt.len..], s.emitted()); + try std.testing.expectEqual(reason, s.reason); } test "a lone stream handed to another driver between rounds and back replies as if kept, pipelined or windowed" { @@ -500,11 +507,12 @@ test "a lone stream handed to another driver between rounds and back replies as try engine.adopt(s); } try std.testing.expect(handed > 0); - for (set.cases, streams[0..set.cases.len]) |c, *s| try expectDecode(c.prompt, s.emitted()); + for (set.cases, streams[0..set.cases.len]) |c, *s| try expectDecode(c, s); } } test "a stream another driver opened and hands to the lane core replies as its one-token decode" { + const c: Case = .{ .prompt = &p2 }; for ([_]bool{ true, false }) |mtp| { var cfg = try modelWith(mtp); defer cfg.deinit(gpa); @@ -517,9 +525,9 @@ test "a stream another driver opened and hands to the lane core replies as its o defer p.deinit(); var s = try sm.Stream.init(gpa, .{ .id = "s", - .prompt = &p2, - .max_new = 40, - .eos = &.{96}, + .prompt = c.prompt, + .max_new = c.max_new, + .eos = c.eos, .proposer = p.proposer(), }); defer s.deinit(gpa); @@ -529,7 +537,7 @@ test "a stream another driver opened and hands to the lane core replies as its o s.cache_len = s.prompt_len; try engine.adopt(&s); while (engine.activeCount() > 0) try engine.step(); - try expectDecode(&p2, s.emitted()); + try expectDecode(c, &s); } } From 721c29b6b0f08e302e44b0079d04a1d854a8e767 Mon Sep 17 00:00:00 2001 From: chaog992 <338238273+chaog992@users.noreply.github.com> Date: Fri, 16 Oct 2026 20:50:02 +0800 Subject: [PATCH 7/8] test(lanes): the hand-back test runs a round on what an adopted round leaves The hand-back test detached the lone stream again at every chance, so when an adopted round left no step queued, no round ran on what it left: after a copy window that keeps more than one token, the stream still holds neither a mode nor a queued step and starts in verify again. It now skips the hand-back in the step right after one, and the copy-drafted set runs 60 tokens, so that round comes before its reply ends. --- zig/src/core/lanes/engine_test.zig | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/zig/src/core/lanes/engine_test.zig b/zig/src/core/lanes/engine_test.zig index d5415bf5c8..570e8b03ea 100644 --- a/zig/src/core/lanes/engine_test.zig +++ b/zig/src/core/lanes/engine_test.zig @@ -469,7 +469,7 @@ test "a lone stream handed to another driver between rounds and back replies as const sets = [_]Set{ .{ .cases = &.{ .{ .prompt = &p1, .drafts = false }, .{ .prompt = &p2, .max_new = 6 } }, .mtp = true }, .{ .cases = &.{.{ .prompt = &p2 }}, .mtp = true }, - .{ .cases = &.{.{ .prompt = &p2 }}, .mtp = false }, + .{ .cases = &.{.{ .prompt = &p2, .max_new = 60 }}, .mtp = false }, }; for (sets) |set| { var cfg = try modelWith(set.mtp); @@ -498,13 +498,19 @@ test "a lone stream handed to another driver between rounds and back replies as }; for (streams[0..set.cases.len]) |*s| try engine.addStream(s); var handed: usize = 0; + var settle = false; // a round runs on what the adopted round leaves before the next hand-back while (engine.activeCount() > 0) { try engine.step(); if (engine.live.items.len != 1) continue; + if (settle) { + settle = false; + continue; + } const s = engine.live.items[0]; if (!engine.detach(s)) continue; handed += 1; try engine.adopt(s); + settle = true; } try std.testing.expect(handed > 0); for (set.cases, streams[0..set.cases.len]) |c, *s| try expectDecode(c, s); From 4c685d07b00c66a585c1ea5332b85f8b947c46b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schr=C3=B6der?= <1188270+cwschroeder@users.noreply.github.com> Date: Sat, 17 Oct 2026 11:14:45 +0200 Subject: [PATCH 8/8] fix(clef): admit capture buffers before touching chat state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian Schröder <1188270+cwschroeder@users.noreply.github.com> --- zig/src/families/clef/README.md | 4 ++- zig/src/families/qwen27/hidden_rows.zig | 2 +- zig/src/families/qwen27/hidden_rows_plan.zig | 28 ++++++++++++++++++++ zig/src/native/qwen27_served.zig | 24 ++++++++++++++--- 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/zig/src/families/clef/README.md b/zig/src/families/clef/README.md index df7e24724b..a7ff04e510 100644 --- a/zig/src/families/clef/README.md +++ b/zig/src/families/clef/README.md @@ -20,6 +20,8 @@ Capture currently accepts text token IDs. It has no image input. Isolation after `qwen27_served.Host.hiddenRows` queues this operation on the engine thread between chat rounds. Each caller keeps its own completion storage alive until the operation ends. Pending calls make lone and background drivers yield. An open learning transaction finishes before capture starts. Both drain and stop reject queued calls and wait for an active call to finish safely. A failed learning put-back refuses calls with `StaleModel` until the model heals. A complete capture can pause chat for its duration; the queue does not promise a short pause. HTTP decision scoring is not connected yet. +Before allocating capture buffers, the owner thread checks output rows, private KV state and a conservative fixed-buffer bound against the server's remaining device working set, preserving its 8 GiB reserve. Insufficient room returns `HiddenRowsMemory` without evicting chat snapshots. The current host's larger buffers bound the private single-slot pass; this can refuse a capture that a tighter estimate would admit. The bound covers buffers, not measured peak process memory or compiler/pipeline overhead. + The ANE handoff reserves storage before input encoding, then registers and publishes each committed command buffer without a fallible allocation. The worker can release an even wait after a terminal input-CB failure even when the odd signal never arrived. Drain waits for every registered CB and the last worker completion before releasing retained handles. A poisoned share stays unavailable until explicitly rebuilt. Restoring the chat frame after such a device failure does not guarantee that the server can continue serving. CPU validation uses Zig 0.17.0: @@ -31,7 +33,7 @@ zig build test-golden fuzz-cortheon zig fmt --check build.zig zig ``` -The targeted check passes 34 tests: head admission, optional-head detection, hidden-row bounds and the actual ANE worker/drain protocol. Its two-thread ring test publishes 1,024 tickets through a ring of four entries. The CLI has four additional CPU tests. Whole-tree validation must be repeated after each upstream rebase. +The targeted check covers head admission, optional-head detection, hidden-row bounds and the actual ANE worker/drain protocol. Its two-thread ring test publishes 1,024 tickets through a ring of four entries. Capture-budget cases cover the exact boundary, insufficient room and overflowing totals. The CLI has four additional CPU tests. Whole-tree validation must be repeated after each upstream rebase. Nine additional owner-call cases run with the host tests: owner identity/reentrancy, serialization, cancellation lifetime, stop and enqueue races, chat progress between calls, lone-driver yield, completion of an open lesson, direct engine drain and stale-weight refusal. Mutants that remove lone yield, block an open lesson behind a call, or serve two calls before chat each fail their corresponding regression test. diff --git a/zig/src/families/qwen27/hidden_rows.zig b/zig/src/families/qwen27/hidden_rows.zig index 1143d2f32a..d06c38da5f 100644 --- a/zig/src/families/qwen27/hidden_rows.zig +++ b/zig/src/families/qwen27/hidden_rows.zig @@ -5,7 +5,7 @@ const Model = @import("model.zig").Model; const Frame = @import("gpu_frame.zig").Frame; const Runner = @import("decode_round.zig").Runner; const Round = @import("round_plan.zig").Round; -const Plan = @import("hidden_rows_plan.zig").Plan; +pub const Plan = @import("hidden_rows_plan.zig").Plan; pub const Options = struct { context: u32, diff --git a/zig/src/families/qwen27/hidden_rows_plan.zig b/zig/src/families/qwen27/hidden_rows_plan.zig index a700194ec6..458a5d13fe 100644 --- a/zig/src/families/qwen27/hidden_rows_plan.zig +++ b/zig/src/families/qwen27/hidden_rows_plan.zig @@ -14,6 +14,15 @@ pub const Plan = struct { _ = try std.math.mul(usize, values, @sizeOf(u16)); return .{ .rows = @intCast(ids.len), .values = values }; } + + /// The owned CPU rows and private KV cache must fit beside the fixed capture buffers and server reserve. + pub fn admit(p: Plan, available: usize, fixed: usize, kv_width: usize, attention_layers: usize) !void { + const output = try std.math.mul(usize, p.values, @sizeOf(u16)); + const kv = try std.math.mul(usize, try std.math.mul(usize, p.rows, kv_width), 4); + const caches = try std.math.mul(usize, kv, attention_layers); + const required = try std.math.add(usize, try std.math.add(usize, fixed, output), caches); + if (required > available) return error.HiddenRowsMemory; + } }; test "all tokens including later chunks are checked before a disposable pass" { @@ -37,3 +46,22 @@ test "hidden row allocation checks both element and byte multiplication" { try std.testing.expectError(error.Overflow, Plan.init(&.{0}, std.math.maxInt(usize), 96, 128, 1, 1)); try std.testing.expectError(error.BadHiddenGeometry, Plan.init(&.{0}, 0, 96, 128, 1, 1)); } + +test "a valid long capture is refused when private state would consume the chat reserve" { + const p = Plan{ .rows = 196608, .values = 196608 * 5120 }; + const fixed = 256 << 20; + const required = fixed + (196608 * 5120 * 2) + (196608 * 1024 * 4 * 16); + try std.testing.expectError(error.HiddenRowsMemory, p.admit(8 << 30, fixed, 1024, 16)); + try std.testing.expectError(error.HiddenRowsMemory, p.admit(required - 1, fixed, 1024, 16)); + try p.admit(required, fixed, 1024, 16); + try std.testing.expectError(error.HiddenRowsMemory, (Plan{ .rows = 1, .values = 5120 }).admit(10240, 1, 0, 0)); +} + +test "capture admission refuses overflowing byte totals rather than wrapping into free room" { + const max = std.math.maxInt(usize); + try std.testing.expectError(error.Overflow, (Plan{ .rows = 1, .values = max }).admit(max, 0, 0, 0)); + const p = Plan{ .rows = 2, .values = 2 }; + try std.testing.expectError(error.Overflow, p.admit(max, 0, max, 1)); + try std.testing.expectError(error.Overflow, p.admit(max, 0, 1, max)); + try std.testing.expectError(error.Overflow, p.admit(max, max, 0, 0)); +} diff --git a/zig/src/native/qwen27_served.zig b/zig/src/native/qwen27_served.zig index 0ce69baded..b7846bc173 100644 --- a/zig/src/native/qwen27_served.zig +++ b/zig/src/native/qwen27_served.zig @@ -53,25 +53,43 @@ pub const Host = struct { pub fn hiddenRows(h: *Host, a: Allocator, ids: []const u32, chunk: u32, cancel: ?*const std.atomic.Value(bool)) !q.hidden_rows.Rows { const Call = struct { a: Allocator, - model: *q.model.Model, + host: *Host, ids: []const u32, options: q.hidden_rows.Options, rows: ?q.hidden_rows.Rows = null, failure: ?anyerror = null, fn run(ptr: *anyopaque) void { const c: *@This() = @ptrCast(@alignCast(ptr)); - c.rows = q.hidden_rows.captureExclusive(c.a, c.model, c.ids, c.options) catch |err| { + c.host.admitRows(c.ids, c.options) catch |err| { + c.failure = err; + return; + }; + c.rows = q.hidden_rows.captureExclusive(c.a, c.host.model, c.ids, c.options) catch |err| { c.failure = err; return; }; } }; - var call = Call{ .a = a, .model = h.model, .ids = ids, .options = .{ .context = h.host.info_.context_window, .chunk = chunk, .cancel = cancel } }; + var call = Call{ .a = a, .host = h, .ids = ids, .options = .{ .context = h.host.info_.context_window, .chunk = chunk, .cancel = cancel } }; try h.host.callOwner(&call, Call.run); if (call.failure) |err| return err; return call.rows.?; } + /// Existing multi-slot buffers bound a private single-slot pass; refuse without evicting chat snapshots. + fn admitRows(h: *Host, ids: []const u32, options: q.hidden_rows.Options) !void { + const m = h.model; + const plan = try q.hidden_rows.Plan.init(ids, m.config.hidden, m.config.vocab, m.frame.capacity, options.context, options.chunk); + var fixed: usize = h.runner.taps.buffer.length(); + inline for (.{ m.frame.buffers, h.runner.gdn.buffers, h.runner.scratch.buffers }) |buffers| { + for (buffers) |buffer| fixed = try std.math.add(usize, fixed, buffer.length()); + } + for (h.runner.bindings) |binding| for (binding.buffers) |buffer| { + fixed = try std.math.add(usize, fixed, buffer.length()); + }; + try plan.admit(room(m), fixed, m.config.kvDim(), h.runner.bindings.len); + } + fn of(ptr: *anyopaque) *Host { return @ptrCast(@alignCast(ptr)); }