diff --git a/zig/src/core/engine_api.zig b/zig/src/core/engine_api.zig index 1e98911bfd..8055b59e4a 100644 --- a/zig/src/core/engine_api.zig +++ b/zig/src/core/engine_api.zig @@ -261,6 +261,10 @@ pub const Info = struct { /// A lane-vision family's image input (LaneVision: each image part its placeholder, the family's prepare its span); /// null: none (upstream's image input is `vision`). lane_vision: ?LaneVision = null, + /// An added-token spelling in request text is plain text, not its token (server/plain_text.zig). + plain_message_text: bool = false, + /// With plain_message_text: the spellings that stay tokens in an assistant's content and reasoning. + plain_assistant_keep: []const []const u8 = &.{}, /// A prompt-only request (max_tokens 0) keeps its end state: the server prefills each reply for the next turn. warm_turns: bool = false, /// The engine decodes plain whatever a request asks: no drafter loaded, or this chip is faster plain. diff --git a/zig/src/core/tokenizer/steps.zig b/zig/src/core/tokenizer/steps.zig index d78e9cec29..ee3f04e3d3 100644 --- a/zig/src/core/tokenizer/steps.zig +++ b/zig/src/core/tokenizer/steps.zig @@ -121,6 +121,16 @@ pub const Normalizer = union(enum) { return error.UnsupportedNormalizer; } + /// Whether the normalizer changes no text: a sequence of no steps (DeepSeek's). + pub fn isIdentity(n: *const Normalizer) bool { + return switch (n.*) { + .sequence => |steps| for (steps) |*step| { + if (!step.isIdentity()) break false; + } else true, + else => false, + }; + } + pub fn hasForms(n: *const Normalizer) bool { return switch (n.*) { .sequence => |steps| for (steps) |*step| { diff --git a/zig/src/core/tokenizer/tokenizer.zig b/zig/src/core/tokenizer/tokenizer.zig index 785ed87f66..fb317e4229 100644 --- a/zig/src/core/tokenizer/tokenizer.zig +++ b/zig/src/core/tokenizer/tokenizer.zig @@ -161,6 +161,62 @@ pub const Tokenizer = struct { return a.dupe(u32, ids.items); } + /// What encodeCut does to the text between two cuts before it encodes it (a shielded request's markers spelled). + pub const Restore = struct { + ctx: *const anyopaque, + spelled: *const fn (ctx: *const anyopaque, a: Allocator, text: []const u8) Allocator.Error![]const u8, + }; + + /// Python's PlainText.encode: exact added-token spellings are cuts; the restored text between has no added token. + pub fn encodeCut(t: *const Tokenizer, a: Allocator, text: []const u8, restore: Restore) Error![]u32 { + var arena = std.heap.ArenaAllocator.init(a); + defer arena.deinit(); + var ctx = steps.Context{ .a = arena.allocator(), .matcher = .{ .a = arena.allocator() } }; + var valid = text; + if (!std.unicode.utf8ValidateSlice(text)) { + var fixed: std.ArrayList(u8) = .empty; + try unicode.appendLossy(&fixed, ctx.a, text); + valid = fixed.items; + } + var ids: std.ArrayList(u32) = .empty; + var at: usize = 0; + // the raw trie's tokens, then the normalized trie's in the text between them, both on the text as written + for (try t.splitAdded(&ctx, &t.raw, valid)) |seg| { + if (seg.id) |id| { + if (!t.spells(valid[seg.start..seg.end], id)) continue; + try t.encodeRegion(&ctx, valid[at..seg.start], restore, &ids); + try ids.append(ctx.a, id); + at = seg.end; + continue; + } + for (try t.splitAdded(&ctx, &t.normalized, valid[seg.start..seg.end])) |sub| { + const id = sub.id orelse continue; + if (!t.spells(valid[seg.start + sub.start .. seg.start + sub.end], id)) continue; // found through a strip: plain text + try t.encodeRegion(&ctx, valid[at .. seg.start + sub.start], restore, &ids); + try ids.append(ctx.a, id); + at = seg.start + sub.end; + } + } + try t.encodeRegion(&ctx, valid[at..], restore, &ids); + return a.dupe(u32, ids.items); + } + + /// Whether `span` is the added token `id`'s own text. + fn spells(t: *const Tokenizer, span: []const u8, id: u32) bool { + return std.mem.eql(u8, span, t.id_to_token.get(id) orelse return false); + } + + /// One region of encodeCut: restored, then encoded as its own text (Python's plain tokenizer on each region). + fn encodeRegion(t: *const Tokenizer, ctx: *steps.Context, region: []const u8, restore: Restore, ids: *std.ArrayList(u32)) Error!void { + if (region.len == 0) return; + const text = try restore.spelled(restore.ctx, ctx.a, region); + if (text.len == 0) return; + // text that spells no added token encodes the same with or without them: the memo's ids are good + const plain = t.raw.next(text, 0) == null and t.normalized.next(text, 0) == null and (t.normalizer == null or t.normalizer.?.isIdentity()); + if (plain and t.memo != null and text.len >= @import("memo.zig").min_bytes) return t.encodeSegment(ctx, text, true, ids); + return t.encodeTextWith(ctx, text, true, ids, false); + } + fn encodeText(t: *const Tokenizer, ctx: *steps.Context, text: []const u8, first: bool, ids: *std.ArrayList(u32)) Error!void { return t.encodeTextWith(ctx, text, first, ids, true); } @@ -386,86 +442,6 @@ test "byte-level BPE with merges, added tokens and decode" { try std.testing.expectEqualStrings("", unknown); } -test "with the memo, long segments encode as without it, and the second encode takes their kept ids" { - const json = - \\{"added_tokens":[{"id":7,"content":"<|end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}], - \\ "pre_tokenizer":{"type":"Sequence","pretokenizers":[{"type":"Split","pattern":{"Regex":" ?\\S+|\\s+"},"behavior":"Isolated","invert":false},{"type":"ByteLevel","add_prefix_space":false,"trim_offsets":false,"use_regex":false}]}, - \\ "decoder":{"type":"ByteLevel"}, - \\ "model":{"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"byte_fallback":false,"ignore_merges":false, - \\ "vocab":{"a":0,"b":1,"Ġ":2,"ab":3,"Ġa":4,"Ġab":5,"c":6},"merges":[["Ġ","a"],"a b","Ġa b"]}} - ; - const a = std.testing.allocator; - var t = try parse(a, json); - defer t.deinit(); - var text: std.ArrayList(u8) = .empty; - defer text.deinit(a); - for (0..300) |_| try text.appendSlice(a, "ab abc "); - try text.appendSlice(a, "<|end|>"); - for (0..300) |_| try text.appendSlice(a, " ab cab"); - const plain = try t.encode(a, text.items); - defer a.free(plain); - t.memo = try Memo.create(a, 1 << 20); - for (0..2) |_| { - const kept = try t.encode(a, text.items); - defer a.free(kept); - try std.testing.expectEqualSlices(u32, plain, kept); - } - try std.testing.expectEqual(@as(u64, 2), t.memo.?.hits); // both segments, the second time -} - -test "a kept segment's ids equal a fresh encode's wherever the segment sits" { - const added = - \\{"added_tokens":[{"id":20,"content":"<|end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}, - \\ {"id":21,"content":"<|l|>","single_word":false,"lstrip":true,"rstrip":false,"normalized":false,"special":true}, - \\ {"id":22,"content":"<|r|>","single_word":false,"lstrip":false,"rstrip":true,"normalized":false,"special":true}], - ; - const kinds = [_][]const u8{ - added ++ - \\ "pre_tokenizer":{"type":"Sequence","pretokenizers":[{"type":"Split","pattern":{"Regex":" ?\\S+|\\s+"},"behavior":"Isolated","invert":false},{"type":"ByteLevel","add_prefix_space":false,"trim_offsets":false,"use_regex":false}]}, - \\ "model":{"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"byte_fallback":false,"ignore_merges":false, - \\ "vocab":{"a":0,"b":1,"Ġ":2,"ab":3,"Ġa":4,"Ġab":5,"c":6,"x":7,"ï":8,"¿":9,"½":10},"merges":[["Ġ","a"],"a b","Ġa b"]}} - , - added ++ - \\ "pre_tokenizer":{"type":"Metaspace","replacement":"▁","prepend_scheme":"first","split":true}, - \\ "model":{"type":"BPE","unk_token":"","fuse_unk":true,"byte_fallback":true, - \\ "vocab":{"":0,"a":1,"b":2,"▁":3,"ab":4,"▁a":5,"▁ab":6,"c":7,"x":8,"<0xEF>":9,"<0xBF>":10,"<0xBD>":11},"merges":["a b","▁ a","▁a b"]}} - , - }; - const a = std.testing.allocator; - var body: std.ArrayList(u8) = .empty; - defer body.deinit(a); - for (0..200) |i| try body.appendSlice(a, if (i % 3 == 0) "ab abc " else "cab ab "); - const b = body.items; - // first or not, after lstrip and rstrip tokens, with a leading space, around a lossy byte, repeated in one prompt - const texts = [_][]const []const u8{ - &.{b}, &.{ "<|end|>", b }, &.{ b, "<|end|>", b }, &.{ "x <|l|>", b }, - &.{ "<|r|> ", b }, &.{ " ", b }, &.{ b, "\xff", b }, &.{ "<|end|> ", b, "<|r|>", b }, - }; - for (kinds, 0..) |json, kind| { - var t = try parse(a, json); - defer t.deinit(); - var fresh: [texts.len][]u32 = undefined; - for (texts, &fresh) |parts, *ids| { - const s = try std.mem.concat(a, u8, parts); - defer a.free(s); - ids.* = try t.encode(a, s); - } - defer for (fresh) |ids| a.free(ids); - // the Metaspace tokenizer prepends to the first segment alone, so a memo blind to `first` fails below - if (kind == 1) try std.testing.expect(!std.mem.eql(u32, fresh[0], fresh[1][1..])); - t.memo = try Memo.create(a, 1 << 20); - for (0..2) |round| for (0..texts.len) |k| { - const i = if (round == 0) k else texts.len - 1 - k; - const s = try std.mem.concat(a, u8, texts[i]); - defer a.free(s); - const kept = try t.encode(a, s); - defer a.free(kept); - try std.testing.expectEqualSlices(u32, fresh[i], kept); - }; - try std.testing.expect(t.memo.?.hits >= 10); // every long segment of the second round, at least - } -} - test "sentencepiece-style BPE with byte fallback" { const json = \\{"added_tokens":[],"normalizer":{"type":"Replace","pattern":{"String":" "},"content":"▁"}, @@ -592,3 +568,7 @@ test { test { _ = @import("identity.zig"); } + +test { + _ = @import("tokenizer_test.zig"); +} diff --git a/zig/src/core/tokenizer/tokenizer_test.zig b/zig/src/core/tokenizer/tokenizer_test.zig new file mode 100644 index 0000000000..8834e2b161 --- /dev/null +++ b/zig/src/core/tokenizer/tokenizer_test.zig @@ -0,0 +1,120 @@ +//! The tokenizer's cut encode and memo tests. + +const std = @import("std"); +const tokenizer = @import("tokenizer.zig"); +const Allocator = std.mem.Allocator; +const Tokenizer = tokenizer.Tokenizer; +const Memo = tokenizer.Memo; +const parse = tokenizer.parse; + +test "a cut encode: the text's own added tokens are tokens, the restored text between them is plain text" { + const json = + \\{"added_tokens":[{"id":10,"content":"<|end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}, + \\ {"id":11,"content":"","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false}], + \\ "normalizer":{"type":"Sequence","normalizers":[]}, + \\ "pre_tokenizer":{"type":"Sequence","pretokenizers":[{"type":"Split","pattern":{"Regex":" ?\\S+|\\s+"},"behavior":"Isolated","invert":false},{"type":"ByteLevel","add_prefix_space":false,"trim_offsets":false,"use_regex":false}]}, + \\ "decoder":{"type":"ByteLevel"}, + \\ "model":{"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"byte_fallback":false,"ignore_merges":false, + \\ "vocab":{"a":0,"b":1,"Ġ":2,"ab":3,"Ġa":4,"Ġab":5,"c":6,"<":7,"t":8,">":9},"merges":[["Ġ","a"],"a b","Ġa b"]}} + ; + const a = std.testing.allocator; + var t = try parse(a, json); + defer t.deinit(); + const Marks = struct { + fn spelled(_: *const anyopaque, al: Allocator, text: []const u8) Allocator.Error![]const u8 { + return std.mem.replaceOwned(u8, al, text, "@0@", ""); + } + }; + const restore: Tokenizer.Restore = .{ .ctx = &t, .spelled = Marks.spelled }; + const ids = try t.encodeCut(a, "ab<|end|>a@0@bc", restore); + defer a.free(ids); + // added tokens take ids past the model's 10 (tokenizers' rule): "<|end|>" 10, "" 11; the marker's "" is plain + try std.testing.expectEqualSlices(u32, &.{ 3, 10, 0, 7, 8, 9, 1, 11, 6 }, ids); + // no marker and no added token: the same ids as encode, through the memo too + t.memo = try Memo.create(a, 1 << 20); + const plain = try t.encode(a, "ab abc"); + defer a.free(plain); + const cut = try t.encodeCut(a, "ab abc", restore); + defer a.free(cut); + try std.testing.expectEqualSlices(u32, plain, cut); +} + +test "with the memo, long segments encode as without it, and the second encode takes their kept ids" { + const json = + \\{"added_tokens":[{"id":7,"content":"<|end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}], + \\ "pre_tokenizer":{"type":"Sequence","pretokenizers":[{"type":"Split","pattern":{"Regex":" ?\\S+|\\s+"},"behavior":"Isolated","invert":false},{"type":"ByteLevel","add_prefix_space":false,"trim_offsets":false,"use_regex":false}]}, + \\ "decoder":{"type":"ByteLevel"}, + \\ "model":{"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"byte_fallback":false,"ignore_merges":false, + \\ "vocab":{"a":0,"b":1,"Ġ":2,"ab":3,"Ġa":4,"Ġab":5,"c":6},"merges":[["Ġ","a"],"a b","Ġa b"]}} + ; + const a = std.testing.allocator; + var t = try parse(a, json); + defer t.deinit(); + var text: std.ArrayList(u8) = .empty; + defer text.deinit(a); + for (0..300) |_| try text.appendSlice(a, "ab abc "); + try text.appendSlice(a, "<|end|>"); + for (0..300) |_| try text.appendSlice(a, " ab cab"); + const plain = try t.encode(a, text.items); + defer a.free(plain); + t.memo = try Memo.create(a, 1 << 20); + for (0..2) |_| { + const kept = try t.encode(a, text.items); + defer a.free(kept); + try std.testing.expectEqualSlices(u32, plain, kept); + } + try std.testing.expectEqual(@as(u64, 2), t.memo.?.hits); // both segments, the second time +} + +test "a kept segment's ids equal a fresh encode's wherever the segment sits" { + const added = + \\{"added_tokens":[{"id":20,"content":"<|end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}, + \\ {"id":21,"content":"<|l|>","single_word":false,"lstrip":true,"rstrip":false,"normalized":false,"special":true}, + \\ {"id":22,"content":"<|r|>","single_word":false,"lstrip":false,"rstrip":true,"normalized":false,"special":true}], + ; + const kinds = [_][]const u8{ + added ++ + \\ "pre_tokenizer":{"type":"Sequence","pretokenizers":[{"type":"Split","pattern":{"Regex":" ?\\S+|\\s+"},"behavior":"Isolated","invert":false},{"type":"ByteLevel","add_prefix_space":false,"trim_offsets":false,"use_regex":false}]}, + \\ "model":{"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"byte_fallback":false,"ignore_merges":false, + \\ "vocab":{"a":0,"b":1,"Ġ":2,"ab":3,"Ġa":4,"Ġab":5,"c":6,"x":7,"ï":8,"¿":9,"½":10},"merges":[["Ġ","a"],"a b","Ġa b"]}} + , + added ++ + \\ "pre_tokenizer":{"type":"Metaspace","replacement":"▁","prepend_scheme":"first","split":true}, + \\ "model":{"type":"BPE","unk_token":"","fuse_unk":true,"byte_fallback":true, + \\ "vocab":{"":0,"a":1,"b":2,"▁":3,"ab":4,"▁a":5,"▁ab":6,"c":7,"x":8,"<0xEF>":9,"<0xBF>":10,"<0xBD>":11},"merges":["a b","▁ a","▁a b"]}} + , + }; + const a = std.testing.allocator; + var body: std.ArrayList(u8) = .empty; + defer body.deinit(a); + for (0..200) |i| try body.appendSlice(a, if (i % 3 == 0) "ab abc " else "cab ab "); + const b = body.items; + // first or not, after lstrip and rstrip tokens, with a leading space, around a lossy byte, repeated in one prompt + const texts = [_][]const []const u8{ + &.{b}, &.{ "<|end|>", b }, &.{ b, "<|end|>", b }, &.{ "x <|l|>", b }, + &.{ "<|r|> ", b }, &.{ " ", b }, &.{ b, "\xff", b }, &.{ "<|end|> ", b, "<|r|>", b }, + }; + for (kinds, 0..) |json, kind| { + var t = try parse(a, json); + defer t.deinit(); + var fresh: [texts.len][]u32 = undefined; + for (texts, &fresh) |parts, *ids| { + const s = try std.mem.concat(a, u8, parts); + defer a.free(s); + ids.* = try t.encode(a, s); + } + defer for (fresh) |ids| a.free(ids); + // the Metaspace tokenizer prepends to the first segment alone, so a memo blind to `first` fails below + if (kind == 1) try std.testing.expect(!std.mem.eql(u32, fresh[0], fresh[1][1..])); + t.memo = try Memo.create(a, 1 << 20); + for (0..2) |round| for (0..texts.len) |k| { + const i = if (round == 0) k else texts.len - 1 - k; + const s = try std.mem.concat(a, u8, texts[i]); + defer a.free(s); + const kept = try t.encode(a, s); + defer a.free(kept); + try std.testing.expectEqualSlices(u32, fresh[i], kept); + }; + try std.testing.expect(t.memo.?.hits >= 10); // every long segment of the second round, at least + } +} diff --git a/zig/src/families/dsv41/exl3_linear.zig b/zig/src/families/dsv41/exl3_linear.zig index 25a9241925..dcce543ad1 100644 --- a/zig/src/families/dsv41/exl3_linear.zig +++ b/zig/src/families/dsv41/exl3_linear.zig @@ -164,7 +164,7 @@ pub fn rotMany(k: *const Kernels, stream: cuda.Stream, calls: []const Call, m: u var a: RArgs = .{ .n = @intCast(calls.len), .M = @intCast(m) }; var warps: usize = 0; for (calls, 0..) |c, i| { - a.l[i] = .{ .x = c.x, .suh = c.layer.suh, .xh = c.xh, .ldx = c.ldx, .x_dtype = @intFromEnum(c.x_dtype), .K = @intCast(c.layer.k), .first = @intCast(warps) }; + a.l[i] = .{ .x = c.x, .suh = c.layer.suh, .xh = c.xh, .ldx = c.ldx, .x_dtype = @backingInt(c.x_dtype), .K = @intCast(c.layer.k), .first = @intCast(warps) }; warps += m * (c.layer.k / 128); } var args: cuda.Args = .{}; @@ -189,7 +189,7 @@ pub fn glinear(k: *const Kernels, stream: cuda.Stream, calls: []const Call, m: u done[j] = true; const s = strides(c.layer); const n: usize = @intCast(a.n); - a.l[n] = .{ .xh = c.xh, .ldx = @intCast(c.layer.k), .T = c.layer.words, .stride_k = s[0], .stride_nb = s[1], .svh = c.layer.svh, .y = c.y, .y_dtype = @intFromEnum(c.y_dtype), .ldy = c.ldy, .K = @intCast(c.layer.k), .N = @intCast(c.layer.n), .SK = @intCast(pj.sk), .counters = c.counters, .first = @intCast(blocks) }; + a.l[n] = .{ .xh = c.xh, .ldx = @intCast(c.layer.k), .T = c.layer.words, .stride_k = s[0], .stride_nb = s[1], .svh = c.layer.svh, .y = c.y, .y_dtype = @backingInt(c.y_dtype), .ldy = c.ldy, .K = @intCast(c.layer.k), .N = @intCast(c.layer.n), .SK = @intCast(pj.sk), .counters = c.counters, .first = @intCast(blocks) }; if (c.rot) |r| { a.l[n].rsuh = r.suh; a.l[n].rxh = r.xh; @@ -243,12 +243,12 @@ test "plans as linear.py makes them" { test "plans and strides equal every recorded glinear layer of the served build" { // (K, N, SK, WK, K2, stride_k, stride_nb) of each distinct layer the recording's glinear calls passed const seen = [_][7]u32{ - .{ 512, 128, 1, 4, 16, 512, 16384 }, .{ 1280, 4096, 2, 4, 10, 320, 25600 }, .{ 1280, 16384, 1, 8, 8, 256, 20480 }, - .{ 1280, 16384, 1, 8, 10, 320, 25600 }, .{ 3072, 25600, 1, 8, 8, 256, 49152 }, .{ 3072, 25600, 1, 8, 10, 320, 61440 }, - .{ 4096, 1024, 8, 4, 8, 256, 65536 }, .{ 4096, 1024, 8, 4, 10, 320, 81920 }, .{ 4096, 5120, 8, 4, 8, 256, 65536 }, - .{ 4096, 5120, 8, 4, 10, 320, 81920 }, .{ 5120, 512, 8, 4, 8, 256, 81920 }, .{ 5120, 512, 8, 4, 10, 320, 102400 }, - .{ 5120, 512, 8, 4, 12, 384, 122880 }, .{ 5120, 1280, 8, 4, 8, 256, 81920 }, .{ 5120, 1280, 8, 4, 10, 320, 102400 }, - .{ 5120, 1280, 8, 4, 12, 384, 122880 }, .{ 5120, 64640, 1, 8, 12, 384, 122880 }, .{ 15360, 5120, 8, 4, 8, 256, 245760 }, + .{ 512, 128, 1, 4, 16, 512, 16384 }, .{ 1280, 4096, 2, 4, 10, 320, 25600 }, .{ 1280, 16384, 1, 8, 8, 256, 20480 }, + .{ 1280, 16384, 1, 8, 10, 320, 25600 }, .{ 3072, 25600, 1, 8, 8, 256, 49152 }, .{ 3072, 25600, 1, 8, 10, 320, 61440 }, + .{ 4096, 1024, 8, 4, 8, 256, 65536 }, .{ 4096, 1024, 8, 4, 10, 320, 81920 }, .{ 4096, 5120, 8, 4, 8, 256, 65536 }, + .{ 4096, 5120, 8, 4, 10, 320, 81920 }, .{ 5120, 512, 8, 4, 8, 256, 81920 }, .{ 5120, 512, 8, 4, 10, 320, 102400 }, + .{ 5120, 512, 8, 4, 12, 384, 122880 }, .{ 5120, 1280, 8, 4, 8, 256, 81920 }, .{ 5120, 1280, 8, 4, 10, 320, 102400 }, + .{ 5120, 1280, 8, 4, 12, 384, 122880 }, .{ 5120, 64640, 1, 8, 12, 384, 122880 }, .{ 15360, 5120, 8, 4, 8, 256, 245760 }, }; for (seen) |r| { const p = plan(r[0], r[1]); diff --git a/zig/src/families/dsv41/native.zig b/zig/src/families/dsv41/native.zig index 07074e2fa4..194d43d13e 100644 --- a/zig/src/families/dsv41/native.zig +++ b/zig/src/families/dsv41/native.zig @@ -52,6 +52,8 @@ pub const max_segments: u32 = 1; pub const prompt_rows: u32 = 2048; /// The served lane's reasoning effort numbers (text.zig): the server passes each name and the template hears its number. pub const efforts = @import("text.zig").efforts; +/// An added-token spelling in request text is plain text; these grounding tokens stay tokens in an assistant's text. +pub const plain_assistant_keep = @import("text.zig").grounding; pub const Options = struct { context: usize, drafts: bool, segments: usize = 1, slide: bool = false }; // slide: no learner here diff --git a/zig/src/families/dsv41/round.zig b/zig/src/families/dsv41/round.zig index c674aa0c3f..59dac0c8de 100644 --- a/zig/src/families/dsv41/round.zig +++ b/zig/src/families/dsv41/round.zig @@ -446,9 +446,9 @@ pub const Round = struct { fn mark(rd: *Round, e: *const Engine, p: PhaseTimer.Phase) !void { if (rd.clk != 0 and rd.clk_marks and (rd.clk_ratio == null or rd.clk_ratio.? == rd.cur_ratio)) { const now = cuda.launch.count; - rd.launches[@intFromEnum(p)] += now -% rd.launch_mark; + rd.launches[@backingInt(p)] += now -% rd.launch_mark; rd.launch_mark = now; - try rd.clock(e, @intFromEnum(p)); + try rd.clock(e, @backingInt(p)); } if (rd.timer) |t| try t.mark(e.s, p); } @@ -801,7 +801,6 @@ fn linearBytes(l: weights.Linear) [2]u64 { return .{ l.words, @as(u64, l.k) * l.n * l.k2 / 16 }; } - /// A measurement's L2 prefetches left out (TF_DS_L2_SKIP, read at Model.open: bit 1 the "attn" fork after q, 2 the "moe" /// fork after wo_b, 4 the "next" fork after the experts); 0 in service. pub var l2_skip: u8 = 0; diff --git a/zig/src/families/dsv41/split_bench_main.zig b/zig/src/families/dsv41/split_bench_main.zig index fc8952dd2a..7980a7e5ef 100644 --- a/zig/src/families/dsv41/split_bench_main.zig +++ b/zig/src/families/dsv41/split_bench_main.zig @@ -320,10 +320,23 @@ pub fn main(init: std.process.Init) !u8 { try scales.upload(0, hs); } const run: Run = .{ - .t = t, .o = &o, .s = stream, .top = top, .n = n, .rb = rb, + .t = t, + .o = &o, + .s = stream, + .top = top, + .n = n, + .rb = rb, .kx = .{ .fp4 = .{ .codes = codes.ptr, .scales = scales.ptr } }, - .q = q.ptr, .w = w.ptr, .vis = vis.ptr, .nvis = nvis.ptr, .keys = keys.ptr, .tmax = tmax.ptr, .bufs = bufs, - .parts = parts.ptr, .merged = merged.ptr, .vblk = vblk.ptr, + .q = q.ptr, + .w = w.ptr, + .vis = vis.ptr, + .nvis = nvis.ptr, + .keys = keys.ptr, + .tmax = tmax.ptr, + .bufs = bufs, + .parts = parts.ptr, + .merged = merged.ptr, + .vblk = vblk.ptr, }; // every output byte, both splits, both patterns; the served keys past vis diff --git a/zig/src/families/dsv41/text.zig b/zig/src/families/dsv41/text.zig index 8fe1e86c68..1012e7e38a 100644 --- a/zig/src/families/dsv41/text.zig +++ b/zig/src/families/dsv41/text.zig @@ -6,6 +6,9 @@ //! without one hears the template's default ("high": 75), the number the lane sends; "none" turns thinking off. //! Reasoning split ("" closes the block the generation prompt opens) and DSML tool calls are the server's own. +/// Grounding tokens (app.py GROUNDING) stay tokens in an assistant's content and reasoning; other spellings are text. +pub const grounding: []const []const u8 = &.{ "<|ref|>", "<|/ref|>", "<|box|>", "<|/box|>", "<|point|>", "<|/point|>", "<|polygon|>", "<|/polygon|>" }; + /// A name and its number, as engine_api.Effort takes them. pub const Effort = struct { name: []const u8, value: i64 }; diff --git a/zig/src/families/dsv41/tri_basic.zig b/zig/src/families/dsv41/tri_basic.zig index 66cb3470a5..34acadee40 100644 --- a/zig/src/families/dsv41/tri_basic.zig +++ b/zig/src/families/dsv41/tri_basic.zig @@ -19,9 +19,9 @@ fn ci(name: []const u8, v: usize) aot.Const { pub fn hcPre(t: Tri, h: u64, fn_w: u64, scale: u64, base: u64, pre_in: u64, norm_w: u64, eps: f32, hc_eps: f32, iters: usize, out: u64, pre_out: u64, post: u64, comb: u64, part: u64, rows: usize, d: usize) !void { try t.run("_hc_partial", .{ u(rows), hc_blocks, 1 }, &.{ p("X", "*bf16", h), p("FN", "*fp32", fn_w), p("PART", "*fp32", part) }, &.{ ci("WIDE", 4 * d), ci("NB", hc_blocks), ci("SUB", 128) }); try t.run("_hc_finish", .{ u(rows), 1, 1 }, &.{ - p("X", "*bf16", h), p("PART", "*fp32", part), p("BASE", "*fp32", base), p("SCALE", "*fp32", scale), - p("PRE_IN", "*fp32", pre_in), p("NW", "*bf16", norm_w), p("OUT", "*bf16", out), p("PRE_OUT", "*fp32", pre_out), - p("POST", "*fp32", post), p("COMB", "*fp32", comb), aot.float("eps", eps), aot.float("hc_eps", hc_eps), + p("X", "*bf16", h), p("PART", "*fp32", part), p("BASE", "*fp32", base), p("SCALE", "*fp32", scale), + p("PRE_IN", "*fp32", pre_in), p("NW", "*bf16", norm_w), p("OUT", "*bf16", out), p("PRE_OUT", "*fp32", pre_out), + p("POST", "*fp32", post), p("COMB", "*fp32", comb), aot.float("eps", eps), aot.float("hc_eps", hc_eps), }, &.{ ci("D", d), ci("NB", hc_blocks), ci("ITERS", iters), ci("BLOCK", 1024) }); } diff --git a/zig/src/native/cuda.zig b/zig/src/native/cuda.zig index e3580135e7..df94ce6b88 100644 --- a/zig/src/native/cuda.zig +++ b/zig/src/native/cuda.zig @@ -477,6 +477,8 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O .vision = if (@hasField(@TypeOf(loaded), "vision")) vision_offer.offer(loaded.vision) else null, .lane_vision = if (@hasField(@TypeOf(loaded), "lane_vision")) loaded.lane_vision else null, .efforts = familyEfforts(F), + .plain_message_text = @hasDecl(F, "plain_assistant_keep"), + .plain_assistant_keep = if (@hasDecl(F, "plain_assistant_keep")) F.plain_assistant_keep else &.{}, }, .{ .ctx = loaded.ctx, .text = F.explain }); opened = true; return .{ .engine = h.host.engine(), .close = Host.close, .ctx = h }; diff --git a/zig/src/server/effort_values_test.zig b/zig/src/server/effort_values_test.zig index f8b01f8ae9..e3ec47b9ef 100644 --- a/zig/src/server/effort_values_test.zig +++ b/zig/src/server/effort_values_test.zig @@ -10,7 +10,7 @@ const Conn = @import("http_conn.zig").Conn; /// DeepSeek-V4.1's numbers (families/dsv41/text.zig; the served lane's EFFORT_VALUES). const numbers = [_]api.Effort{ - .{ .name = "minimal", .value = 25 }, .{ .name = "low", .value = 50 }, .{ .name = "medium", .value = 62 }, + .{ .name = "minimal", .value = 25 }, .{ .name = "low", .value = 50 }, .{ .name = "medium", .value = 62 }, .{ .name = "high", .value = 75 }, .{ .name = "xhigh", .value = 88 }, .{ .name = "max", .value = 100 }, }; diff --git a/zig/src/server/hf_text.zig b/zig/src/server/hf_text.zig index e4617f5c7c..99c9227c1a 100644 --- a/zig/src/server/hf_text.zig +++ b/zig/src/server/hf_text.zig @@ -212,6 +212,8 @@ pub const HfText = struct { .special = specialFn, .token_bytes = tokenBytesFn, .encode_plain = encodePlainFn, + .added_spellings = addedFn, + .encode_cut = encodeCutFn, } }; } @@ -236,6 +238,17 @@ pub const HfText = struct { return self(ctx).tok.encodePlain(a, s) catch |e| if (e == error.OutOfMemory) error.OutOfMemory else error.Template; } + fn addedFn(ctx: *anyopaque, a: Allocator) Allocator.Error![]const []const u8 { + const added = self(ctx).tok.added; + const out = try a.alloc([]const u8, added.len); + for (added, out) |t, *o| o.* = t.content; + return out; + } + + fn encodeCutFn(ctx: *anyopaque, a: Allocator, s: []const u8, restore: model_text.Text.Restore) model_text.Error![]u32 { + return self(ctx).tok.encodeCut(a, s, .{ .ctx = restore.ctx, .spelled = restore.spelled }) catch |e| if (e == error.OutOfMemory) error.OutOfMemory else error.Template; + } + fn decodeFn(ctx: *anyopaque, a: Allocator, ids: []const u32) Allocator.Error![]u8 { return self(ctx).tok.decode(a, ids, false) catch |e| if (e == error.OutOfMemory) error.OutOfMemory else a.dupe(u8, "\u{fffd}"); } diff --git a/zig/src/server/model_text.zig b/zig/src/server/model_text.zig index cd7ae359a6..683e3d09d4 100644 --- a/zig/src/server/model_text.zig +++ b/zig/src/server/model_text.zig @@ -42,6 +42,16 @@ pub const Text = struct { encode_plain: ?*const fn (ctx: *anyopaque, a: Allocator, text: []const u8) Error![]u32 = null, /// A token's own bytes (a partial UTF-8 sequence kept whole) for logprob rows; null: the tokenizer can't say. token_bytes: ?*const fn (ctx: *anyopaque, a: Allocator, id: u32) Allocator.Error!?[]u8 = null, + /// Every added token's text (plain_text.zig shields these in request text); null: none are known. + added_spellings: ?*const fn (ctx: *anyopaque, a: Allocator) Allocator.Error![]const []const u8 = null, + /// Ids of a shielded prompt: exact added-token spellings are tokens; the restored text between is plain. + encode_cut: ?*const fn (ctx: *anyopaque, a: Allocator, text: []const u8, restore: Restore) Error![]u32 = null, + }; + + /// What encode_cut does to the text between two cuts: a shielded request's markers spelled again. + pub const Restore = struct { + ctx: *const anyopaque, + spelled: *const fn (ctx: *const anyopaque, a: Allocator, text: []const u8) Allocator.Error![]const u8, }; pub fn encode(t: Text, a: Allocator, text: []const u8, add_special: bool) Error![]u32 { @@ -53,6 +63,18 @@ pub const Text = struct { return f(t.ctx, a, text); } + /// Every added token's text, or none when this text cannot list them. + pub fn addedSpellings(t: Text, a: Allocator) Allocator.Error![]const []const u8 { + const f = t.vtable.added_spellings orelse return &.{}; + return f(t.ctx, a); + } + + /// The ids of a shielded prompt (plain_text.zig); null when this text has no cut encode. + pub fn encodeCut(t: Text, a: Allocator, text: []const u8, restore: Restore) ?Error![]u32 { + const f = t.vtable.encode_cut orelse return null; + return f(t.ctx, a, text, restore); + } + pub fn decode(t: Text, a: Allocator, ids: []const u32) Allocator.Error![]u8 { return t.vtable.decode(t.ctx, a, ids); } diff --git a/zig/src/server/plain_text.zig b/zig/src/server/plain_text.zig new file mode 100644 index 0000000000..e32da10d3b --- /dev/null +++ b/zig/src/server/plain_text.zig @@ -0,0 +1,271 @@ +//! Request text stays plain text (server/plain_text.py): a client's `` or `<|User|>` cannot forge a token. +const std = @import("std"); +const json = @import("json"); +const model_text = @import("model_text.zig"); +const Value = json.Value; +const Allocator = std.mem.Allocator; + +/// Leftmost-longest matcher over the added-token spellings, as Python's prefix-tree pattern finds them. +pub const Spellings = struct { + first: [256]u32 = @splat(0), + edges: std.AutoHashMapUnmanaged(u64, u32) = .empty, + /// For each node: the spelling that ends there, else null. + ends: std.ArrayList(?[]const u8) = .empty, + + pub fn init(a: Allocator, words: []const []const u8) Allocator.Error!Spellings { + var m: Spellings = .{}; + try m.ends.append(a, null); + for (words) |w| { + if (w.len == 0) continue; + var node: u32 = 0; + for (w, 0..) |b, i| { + const slot = if (i == 0) &m.first[b] else (try m.edges.getOrPutValue(a, @as(u64, node) << 8 | b, 0)).value_ptr; + if (slot.* == 0) { + try m.ends.append(a, null); + slot.* = @intCast(m.ends.items.len - 1); + } + node = slot.*; + } + m.ends.items[node] = w; + } + return m; + } + + const Hit = struct { start: usize, end: usize, word: []const u8 }; + + fn next(m: *const Spellings, s: []const u8, from: usize) ?Hit { + for (from..s.len) |i| { + var node = m.first[s[i]]; + if (node == 0) continue; + var best: ?Hit = null; + var j = i + 1; + while (true) { + if (m.ends.items[node]) |w| best = .{ .start = i, .end = j, .word = w }; + if (j == s.len) break; + node = m.edges.get(@as(u64, node) << 8 | s[j]) orelse break; + j += 1; + } + if (best) |b| return b; + } + return null; + } +}; + +/// A request's messages and tools after `shield`: `table[n]` is the spelling that marker `prefix` n `@` replaced. +pub const Shielded = struct { + messages: Value, + tools: []const Value, + table: []const []const u8 = &.{}, + prefix: []const u8 = "", + + /// `text` with each marker turned back into its spelling. + pub fn spelled(s: *const Shielded, a: Allocator, text: []const u8) Allocator.Error![]const u8 { + if (s.table.len == 0 or std.mem.indexOf(u8, text, s.prefix) == null) return text; + var out: std.ArrayList(u8) = .empty; + var rest = text; + while (std.mem.indexOf(u8, rest, s.prefix)) |at| { + const after = rest[at + s.prefix.len ..]; + var k: usize = 0; + while (k < after.len and std.ascii.isDigit(after[k])) k += 1; + const n = if (k > 0 and k < after.len and after[k] == '@') std.fmt.parseInt(usize, after[0..k], 10) catch null else null; + if (n == null or n.? >= s.table.len) { // not one of ours: ordinary text + try out.appendSlice(a, rest[0 .. at + s.prefix.len]); + rest = after; + continue; + } + try out.appendSlice(a, rest[0..at]); + try out.appendSlice(a, s.table[n.?]); + rest = after[k + 1 ..]; + } + try out.appendSlice(a, rest); + return out.items; + } + + fn spelledFn(ctx: *const anyopaque, a: Allocator, text: []const u8) Allocator.Error![]const u8 { + const s: *const Shielded = @ptrCast(@alignCast(ctx)); + return s.spelled(a, text); + } +}; + +/// The shield for one tokenizer: its added-token spellings, and those an assistant's content and reasoning keep. +pub const PlainText = struct { + spellings: Spellings, + assistant_keep: []const []const u8, + + /// Null when the text cannot list its added tokens or cannot encode a shielded prompt. + pub fn init(a: Allocator, text: model_text.Text, assistant_keep: []const []const u8) Allocator.Error!?PlainText { + if (text.vtable.encode_cut == null) return null; + const words = try text.addedSpellings(a); + if (words.len == 0) return null; + return .{ .spellings = try Spellings.init(a, words), .assistant_keep = assistant_keep }; + } + + /// `messages` and `tools` with each spelling as a marker; `content_keep`: spellings the server wrote in content. + pub fn shield(p: *const PlainText, a: Allocator, messages: Value, tools: []const Value, content_keep: []const []const u8) Allocator.Error!Shielded { + var attempt: u64 = 0; + while (true) : (attempt += 1) { + var w: Walk = .{ .p = p, .a = a, .prefix = try std.fmt.allocPrint(a, "@TF{x:0>16}:", .{std.hash.Wyhash.hash(attempt, "tensorfold plain text")}) }; + const shielded_messages = try w.messages(messages, content_keep); + const shielded_tools = try a.alloc(Value, tools.len); + for (tools, shielded_tools) |t, *o| o.* = try w.value(t, &.{}); + if (w.table.items.len == 0) return .{ .messages = messages, .tools = tools }; + // a marker is unique: no string of the request holds the prefix + if (!holds(messages, w.prefix) and !holds(.{ .array = @constCast(tools) }, w.prefix)) + return .{ .messages = shielded_messages, .tools = shielded_tools, .table = w.table.items, .prefix = w.prefix }; + } + } + + /// The ids of rendered `text`: the template's spellings as tokens, the rest (markers spelled again) without them. + pub fn encode(p: *const PlainText, text: model_text.Text, a: Allocator, rendered: []const u8, s: *const Shielded) model_text.Error![]u32 { + _ = p; + // DeepSeek's normalizer changes no text, so a prompt without a marker is the tokenizer's own encode + if (s.table.len == 0 or std.mem.indexOf(u8, rendered, s.prefix) == null) return text.encode(a, rendered, false); + return text.encodeCut(a, rendered, .{ .ctx = s, .spelled = Shielded.spelledFn }) orelse text.encode(a, rendered, false); + } +}; + +const Walk = struct { + p: *const PlainText, + a: Allocator, + prefix: []const u8, + table: std.ArrayList([]const u8) = .empty, + index: std.StringHashMapUnmanaged(usize) = .empty, + + fn mark(w: *Walk, out: *std.ArrayList(u8), spelling: []const u8) Allocator.Error!void { + const slot = try w.index.getOrPut(w.a, spelling); + if (!slot.found_existing) { + slot.value_ptr.* = w.table.items.len; + try w.table.append(w.a, spelling); + } + try out.print(w.a, "{s}{d}@", .{ w.prefix, slot.value_ptr.* }); + } + + fn string(w: *Walk, s: []const u8, keep: []const []const u8) Allocator.Error![]const u8 { + var out: std.ArrayList(u8) = .empty; + var at: usize = 0; + var changed = false; + while (w.p.spellings.next(s, at)) |hit| { + if (contains(keep, hit.word)) { + try out.appendSlice(w.a, s[at..hit.end]); + } else { + try out.appendSlice(w.a, s[at..hit.start]); + try w.mark(&out, hit.word); + changed = true; + } + at = hit.end; + } + if (!changed) return s; + try out.appendSlice(w.a, s[at..]); + return out.items; + } + + /// Python's _walk: strings, object keys and values, list items, all with the same `keep`. + fn value(w: *Walk, v: Value, keep: []const []const u8) Allocator.Error!Value { + return switch (v) { + .string => |s| .{ .string = try w.string(s, keep) }, + .array => |items| blk: { + const out = try w.a.alloc(Value, items.len); + for (items, out) |x, *o| o.* = try w.value(x, keep); + break :blk .{ .array = out }; + }, + .object => |o| blk: { + const out = try json.newObject(w.a); + var it = o.iterator(); + while (it.next()) |kv| try out.put(w.a, try w.string(kv.key_ptr.*, keep), try w.value(kv.value_ptr.*, keep)); + break :blk .{ .object = out }; + }, + else => v, + }; + } + + /// Python's _messages: text parts joined first (a spelling can span two); assistant and server spellings kept. + fn messages(w: *Walk, v: Value, content_keep: []const []const u8) Allocator.Error!Value { + if (v != .array) return w.value(v, &.{}); + const out = try w.a.alloc(Value, v.array.len); + for (v.array, out) |m, *o| { + if (m != .object) { + o.* = try w.value(m, &.{}); + continue; + } + const role = m.get("role"); + const assistant = role != null and role.? == .string and std.mem.eql(u8, role.?.string, "assistant"); + const both = try std.mem.concat(w.a, []const u8, &.{ content_keep, if (assistant) w.p.assistant_keep else &.{} }); + const item = try json.newObject(w.a); + var it = m.object.iterator(); + while (it.next()) |kv| { + const key = kv.key_ptr.*; + var x = kv.value_ptr.*; + if (std.mem.eql(u8, key, "content")) x = try joined(w.a, x); + const keep: []const []const u8 = if (std.mem.eql(u8, key, "content")) both else if (std.mem.eql(u8, key, "reasoning_content") and assistant) w.p.assistant_keep else &.{}; + try item.put(w.a, try w.string(key, &.{}), try w.value(x, keep)); + } + o.* = .{ .object = item }; + } + return .{ .array = out }; + } +}; + +/// Text parts as one string (every part a "text" object with a string): Python's _joined. +fn joined(a: Allocator, content: Value) Allocator.Error!Value { + if (content != .array or content.array.len == 0) return content; + for (content.array) |part| { + if (part != .object) return content; + const t = part.get("type") orelse return content; + if (t != .string or !std.mem.eql(u8, t.string, "text")) return content; + const text = part.get("text") orelse return content; + if (text != .string) return content; + } + var out: std.ArrayList(u8) = .empty; + for (content.array) |part| try out.appendSlice(a, part.get("text").?.string); + return .{ .string = out.items }; +} + +fn contains(list: []const []const u8, word: []const u8) bool { + for (list) |x| if (std.mem.eql(u8, x, word)) return true; + return false; +} + +/// Whether a string of `v` (an object key included) holds `needle`. +fn holds(v: Value, needle: []const u8) bool { + return switch (v) { + .string => |s| std.mem.indexOf(u8, s, needle) != null, + .array => |items| for (items) |x| { + if (holds(x, needle)) break true; + } else false, + .object => |o| blk: { + var it = o.iterator(); + while (it.next()) |kv| if (std.mem.indexOf(u8, kv.key_ptr.*, needle) != null or holds(kv.value_ptr.*, needle)) break :blk true; + break :blk false; + }, + else => false, + }; +} + +test "a spelling in request text becomes a marker, the kept ones stay, and the markers spell back" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + const words = [_][]const u8{ "", "", "<|User|>", "<|ref|>", "<|deepseek_image|>" }; + const p: PlainText = .{ .spellings = try Spellings.init(a, &words), .assistant_keep = &.{"<|ref|>"} }; + const messages = (try json.parse(a, + \\[{"role":"user","content":[{"type":"text","text":"ab <|ref|> <|deepseek_image|>"}]}, + \\ {"role":"assistant","content":"<|ref|>x","reasoning_content":"<|ref|><|User|>"}] + )).ok; + const s = try p.shield(a, messages, &.{}, &.{"<|deepseek_image|>"}); + try std.testing.expect(s.table.len == 4); + const user = s.messages.array[0].strField("content").?; + const assistant = s.messages.array[1].strField("content").?; + const reasoning = s.messages.array[1].strField("reasoning_content").?; + // a spelling across two text parts is found; grounding in a user turn is plain; the server's placeholder stays + try std.testing.expect(std.mem.indexOf(u8, user, "") == null and std.mem.indexOf(u8, user, "<|ref|>") == null); + try std.testing.expect(std.mem.indexOf(u8, user, "<|deepseek_image|>") != null); + try std.testing.expect(std.mem.startsWith(u8, assistant, "<|ref|>x@TF") and std.mem.startsWith(u8, reasoning, "<|ref|>@TF")); + try std.testing.expectEqualStrings("ab <|ref|> <|deepseek_image|>", try s.spelled(a, user)); + try std.testing.expectEqualStrings("<|ref|><|User|>", try s.spelled(a, reasoning)); + // no spelling: the request's own values, unjoined + const plain = (try json.parse(a, + \\[{"role":"user","content":[{"type":"text","text":"hi"}]}] + )).ok; + const none = try p.shield(a, plain, &.{}, &.{}); + try std.testing.expect(none.table.len == 0 and none.messages.array[0].get("content").? == .array); +} diff --git a/zig/src/server/prompt.zig b/zig/src/server/prompt.zig index d77c4332f7..0cdbeaaa48 100644 --- a/zig/src/server/prompt.zig +++ b/zig/src/server/prompt.zig @@ -31,6 +31,11 @@ pub const isTitle = messages_mod.isTitleRequest; /// ``render_prompt_ids``: messages normalized for this template, rendered, and a dangling ```` closed. pub fn renderIds(srv: *Server, cx: *Cx, messages: Value, tools: []const Value, thinking: bool, effort: ?[]const u8, generation: bool) errors.Refused![]const u32 { + return renderKept(srv, cx, messages, tools, thinking, effort, generation, &.{}); +} + +/// renderIds; `kept`: spellings that the server itself wrote into the messages' content (a lane image's placeholder). +fn renderKept(srv: *Server, cx: *Cx, messages: Value, tools: []const Value, thinking: bool, effort: ?[]const u8, generation: bool, kept: []const []const u8) errors.Refused![]const u32 { const normalized = try messages_mod.toolArguments(cx, try messages_mod.normalizeWith(cx, messages, srv.late_system, srv.needs_user_after_tool, .of(srv.info.vision))); var problem: []const u8 = ""; const options: model_text.RenderOptions = .{ @@ -40,7 +45,7 @@ pub fn renderIds(srv: *Server, cx: *Cx, messages: Value, tools: []const Value, t .reasoning_effort = if (thinking) effort else null, .reasoning_effort_value = if (thinking) srv.effortValue(effort) else null, }; - var ids = render(srv, cx.a, normalized, options, &problem) catch |e| switch (e) { + var ids = render(srv, cx.a, normalized, options, kept, &problem) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Template => return cx.fail(.server, "{s}", .{problem}), }; @@ -57,8 +62,19 @@ pub fn renderIds(srv: *Server, cx: *Cx, messages: Value, tools: []const Value, t return ids; } -/// The template's ids for `messages`; with image input, a marker the text spells is ordinary text (images.zig). -fn render(srv: *Server, a: std.mem.Allocator, messages: Value, options: model_text.RenderOptions, problem: *[]const u8) model_text.Error![]u32 { +/// The template's ids for `messages`; image markers (images.zig) and plain_message_text spellings are plain text. +fn render(srv: *Server, a: std.mem.Allocator, messages: Value, options: model_text.RenderOptions, kept: []const []const u8, problem: *[]const u8) model_text.Error![]u32 { + if (srv.plain) |*p| { + const tools: []const Value = if (options.tools) |t| t.array else &.{}; + const s = try p.shield(a, messages, tools, kept); + var shielded = options; + if (options.tools != null) shielded.tools = .{ .array = @constCast(s.tools) }; + const text = srv.text.render(a, s.messages, shielded, problem) catch |e| { + if (e == error.Template) problem.* = try s.spelled(a, problem.*); + return e; + }; + return p.encode(srv.text, a, text, &s); + } const v = srv.info.vision orelse return srv.text.renderIds(a, messages, options, problem); const markers = try images.markerText(srv.text, a, v); const text = try srv.text.render(a, try images.shield(a, messages, markers), options, problem); @@ -109,7 +125,10 @@ pub fn prepare(srv: *Server, cx: *Cx, input: chat.Input, thinking: bool, effort: } }, .ids => |ids| return .{ .ids = ids }, }; - const prompt = try renderIds(srv, cx, input.messages, input.tools, thinking, effort, true); + // a lane image's placeholder in content is the server's own: normalizeImages refused each one the client wrote + const placeholder = [1][]const u8{if (srv.info.lane_vision) |v| v.placeholder else ""}; + const kept: []const []const u8 = if (input.lane_images.len > 0 and srv.info.lane_vision != null) &placeholder else &.{}; + const prompt = try renderKept(srv, cx, input.messages, input.tools, thinking, effort, true, kept); if (input.lane_images.len > 0) return laneSpans(srv, cx, prompt, input.lane_images); // (no kept history: the cache keys tokens) const history = try renderIds(srv, cx, input.messages, input.tools, thinking, effort, false); var history_len: usize = 0; diff --git a/zig/src/server/root.zig b/zig/src/server/root.zig index 06be1ffc75..5ba0a3346a 100644 --- a/zig/src/server/root.zig +++ b/zig/src/server/root.zig @@ -43,6 +43,7 @@ test { _ = @import("request_log.zig"); _ = @import("messages.zig"); _ = @import("images.zig"); + _ = @import("plain_text.zig"); _ = @import("image_key.zig"); _ = @import("media_fetch.zig"); _ = @import("videos.zig"); diff --git a/zig/src/server/server.zig b/zig/src/server/server.zig index d048ab0c17..fbd58ae794 100644 --- a/zig/src/server/server.zig +++ b/zig/src/server/server.zig @@ -7,6 +7,7 @@ const fields = @import("fields.zig"); const auth = @import("auth.zig"); const metrics = @import("metrics.zig"); const model_text = @import("model_text.zig"); +const plain_text = @import("plain_text.zig"); const reply_text = @import("reply_text.zig"); const http_conn = @import("http_conn.zig"); const listener_mod = @import("listener.zig"); @@ -88,6 +89,8 @@ pub const Server = struct { config: Config, keys: ?*auth.Store, late_system: []const u8 = "system", + /// Info.plain_message_text's shield; null: request text encodes as the template renders it. + plain: ?plain_text.PlainText = null, needs_user_after_tool: bool = false, markers: reply_text.Markers = reply_text.think_markers, chunks: chunk_plan.Plan = .{}, @@ -147,6 +150,7 @@ pub const Server = struct { srv.thinking_switch = std.mem.indexOf(u8, source, "enable_thinking") != null or std.mem.indexOf(u8, source, "thinking_mode") != null; srv.late_system = try lateSystem(a, text); + if (srv.info.plain_message_text) srv.plain = try plain_text.PlainText.init(a, text, srv.info.plain_assistant_keep); srv.needs_user_after_tool = needsUserAfterTool(source); if (text.tokenId("")) |end| { const lead = try text.encode(a, "\n", false);