diff --git a/changelog.d/thinking-binding-drop-block.added.md b/changelog.d/thinking-binding-drop-block.added.md new file mode 100644 index 000000000..3d73bdd98 --- /dev/null +++ b/changelog.d/thinking-binding-drop-block.added.md @@ -0,0 +1,30 @@ +- `thinkingBinding.dropBlock` in an agent's config asks the provider to drop + a signed thinking block whose earlier conversation has changed, as a fold + changes it, rather than refuse the request. An account that enforces + thinking-signature binding answers such a request with a 400; with the + setting, it drops the blocks that no longer match and answers. Where an + account doesn't enforce it, the provider decides for itself what to do with + such a block, and the setting has it drop them too. The setting is off by + default. + - **What's sent:** `thinking.block_binding.prefix_mismatch_behavior: + "drop_block"` with the `thinking-binding-controls-2026-08-01` beta, which + membrane's Anthropic and Bedrock adapters carry. It has no effect through + other adapters, or where the host sends a thinking config of its own, + through `request.extra` or `providerParams`, which replaces membrane's. + Any other model carries it only while thinking is on. A model whose + thinking is always on (Fable, Mythos) gets an adaptive thinking config + to carry it, with the agent's `thinking.display` when that's set, so on + such a model `display` takes effect only with this setting. membrane + notes that the Mythos family has refused thinking configs with a 400 + before, so on a Mythos model the setting may fail every request. + - **What's reported:** on a framework turn, each call's dropped blocks, and + any shown to the model though they failed the check, appear in its + `inference:usage` trace as `thinking`. A call that dropped blocks, and + every later call of its stream, isn't used to calibrate the context + strategy's token estimates, since the provider doesn't bill a block it + dropped and the strategy still counts it. Where the strategy still sends + the blocks a fold left bound to a conversation the request no longer + holds, the provider drops them on every call until a later fold takes + them out of the context, and while they're sent, no stream calibrates and + the estimates keep the value they had. The direct API carries the setting + too, but reports nothing of it. diff --git a/src/agent.ts b/src/agent.ts index da33f98dd..1f7f92a8d 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -100,15 +100,16 @@ const DEFAULT_SAME_ROUND_THINK_TEXT_POLICY: SameRoundThinkTextPolicy = 'public'; /** * Refuse an agent config whose `promptCaching`, `toolResultGuard`, - * `toolWrapperProseGuard`, `thinking.enabled` or `refusalHandling.autoRewind` - * holds anything but true or false (assertBooleanSetting), or whose `thinking` - * or `refusalHandling` is anything but an object, null or false - * (holdsSettings), or whose `autoContinue` resolveAutoContinue refuses. - * The constructor runs it first, so the rule holds wherever - * an Agent is built. `create()` also runs it for each of its agents before - * anything opens, and `createEphemeralAgent` before it reserves the name, so a - * refusal there leaves nothing behind. Membrane reads `thinking.enabled`, and - * the refusal is here because the setting is AgentConfig's. + * `toolWrapperProseGuard`, `thinking.enabled`, `thinkingBinding.dropBlock` or + * `refusalHandling.autoRewind` holds anything but true or false + * (assertBooleanSetting), or whose `thinking`, `thinkingBinding` or + * `refusalHandling` is anything but an object, null or false (holdsSettings), + * or whose `autoContinue` resolveAutoContinue refuses. The constructor runs it + * first, so the rule holds wherever an Agent is built. `create()` also runs it + * for each of its agents before anything opens, and `createEphemeralAgent` + * before it reserves the name, so a refusal there leaves nothing behind. + * Membrane reads `thinking.enabled`, and the refusal is here because the + * setting is AgentConfig's. */ export function assertAgentSwitches(config: AgentConfig): void { const agent = `Agent "${config.name}"`; @@ -124,6 +125,14 @@ export function assertAgentSwitches(config: AgentConfig): void { 'true asks the provider for extended thinking, and false or leaving it out doesn\'t', ); } + if (holdsSettings(`${agent} thinkingBinding`, config.thinkingBinding, '{ dropBlock: true }')) { + assertBooleanSetting( + `${agent} thinkingBinding.dropBlock`, + config.thinkingBinding.dropBlock, + 'true asks the provider to drop a signed thinking block whose earlier conversation changed rather than refuse the request, ' + + 'and false or leaving it out sends nothing, so the account\'s default applies', + ); + } if (holdsSettings(`${agent} refusalHandling`, config.refusalHandling, '{ autoRewind: true }')) { assertBooleanSetting( `${agent} refusalHandling.autoRewind`, @@ -208,6 +217,9 @@ export class Agent { * blocks are preserved through Chronicle. */ readonly thinking: AgentConfig['thinking']; + /** Whether requests ask the provider to drop signed thinking whose earlier + * conversation changed (see AgentConfig.thinkingBinding.dropBlock). */ + private readonly thinkingDropBlock: boolean; /** Refusal auto-rewind policy (see AgentConfig.refusalHandling). */ readonly refusalHandling: AgentConfig['refusalHandling']; readonly toolResultGuard: ToolResultGuard; @@ -347,6 +359,7 @@ export class Agent { this.maxTokens = config.maxTokens ?? 4096; this.temperature = config.temperature; this.thinking = config.thinking; + this.thinkingDropBlock = config.thinkingBinding?.dropBlock === true; this.refusalHandling = config.refusalHandling; this.toolResultGuard = new ToolResultGuard(config.name, contextManager, config.toolResultGuard === true); this.proseRouting = config.proseRouting ?? 'locus'; @@ -1498,20 +1511,21 @@ export class Agent { /** * The request fields that come from this agent's own settings: its model - * config, prompt caching and cache TTL, prefill message, provider - * parameters and participant name. Every request the agent sends takes - * them from here, a framework turn's (assembleActivationRequest) and the - * direct API's (runInferenceWithInjections), so a setting reaches both or - * neither. + * config, thinking binding, prompt caching and cache TTL, prefill message, + * provider parameters and participant name. Every request the agent sends + * takes them from here, a framework turn's (assembleActivationRequest) and + * the direct API's (runInferenceWithInjections), so a setting reaches both + * or neither. */ private requestSettings(): Pick { + const thinking = this.requestThinking(); return { config: { model: this.model, maxTokens: this.maxTokens, ...(this.temperature !== undefined && { temperature: this.temperature }), - ...(this.thinking !== undefined && { thinking: this.thinking }), + ...(thinking !== undefined && { thinking }), }, promptCaching: this.promptCaching, cacheTtl: this.cacheTtl, @@ -1521,6 +1535,20 @@ export class Agent { }; } + /** + * The thinking config a request carries: the agent's own, with the binding + * control added when `thinkingBinding.dropBlock` asks for it. A model whose + * thinking is always on takes the control whatever `enabled` says, so an + * agent with no thinking config of its own still sends one, holding only + * the control and `enabled: false`. Membrane sends nothing from it to any + * other model, since thinking is off there. + */ + private requestThinking(): NonNullable | undefined { + if (!this.thinkingDropBlock) return this.thinking; + const own = this.thinking !== null && typeof this.thinking === 'object' ? this.thinking : { enabled: false }; + return { ...own, blockBinding: { prefixMismatchBehavior: 'drop_block' } }; + } + /** The request `assembleActivationRequest` builds (see there), for preview * and debug tooling: what this agent's next activation would send. */ async buildActivationRequest( diff --git a/src/framework.ts b/src/framework.ts index 5c18f0ab7..5f81f1a9a 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -18826,6 +18826,9 @@ export class AgentFramework { cacheCreationTokens: 0, cacheReadTokens: 0, }; + // Whether a call of this stream reported signed thinking the provider + // dropped: from then on none of its calls is a calibration sample. + let thinkingDroppedInStream = false; // Output tokens billed since this turn's assistant blocks were last // persisted. The residual over the round's visible blocks is the price of // its hidden thinking, stamped on the signed blocks at persist time @@ -20493,11 +20496,28 @@ export class AgentFramework { // agent's samples can fall out of its band for good // (context-manager #323). A strategy from before then ignores // the second argument. + // A call whose report names thinking the provider dropped billed + // less than the request held: the strategy still counts those + // blocks, so the call's ratio would pull its calibration low. The + // strategy takes the first call it's given after a compile as + // that compile's one sample, and each later call of the stream + // carries what the stream appended, so once a call has dropped + // blocks, none of the stream's later calls is given either. A + // compile whose first call dropped blocks gives no sample, and the + // calibration keeps its value. If the provider drops an earlier + // model's blocks after a model switch (`model_binding_mismatch`), + // they drop on every call until they leave the context, and the + // calibration takes up the new model's tokenizer only after that. + const thinkingReport = event.round?.thinking; + const thinkingDropped = (thinkingReport?.dropped.length ?? 0) > 0; + if (thinkingDropped) thinkingDroppedInStream = true; try { - strat?.reportRealInputTokens?.( - realTotal, - compiledRequest ? { system: compiledRequest.system, tools: compiledRequest.tools } : undefined, - ); + if (!thinkingDroppedInStream) { + strat?.reportRealInputTokens?.( + realTotal, + compiledRequest ? { system: compiledRequest.system, tools: compiledRequest.tools } : undefined, + ); + } } catch { /* calibration is best-effort */ } const kvSubmission = takeKvSubmission?.(); @@ -20525,6 +20545,9 @@ export class AgentFramework { cacheCreation: event.usage.cacheCreationTokens, cacheRead: event.usage.cacheReadTokens, }, + ...(thinkingReport && (thinkingDropped || thinkingReport.mismatchAllowed.length > 0) + ? { thinking: { dropped: thinkingReport.dropped, mismatchAllowed: thinkingReport.mismatchAllowed } } + : {}), }); break; } diff --git a/src/types/agent.ts b/src/types/agent.ts index f8f77cfbb..40a495f4d 100644 --- a/src/types/agent.ts +++ b/src/types/agent.ts @@ -152,7 +152,9 @@ export interface AgentConfig { * Extended thinking config. When `enabled: true`, the agent runs with * Anthropic's native extended thinking; responses include `thinking` blocks * with cryptographic signatures, and the API enforces `temperature: 1` - * (Membrane handles this). Omit or set `enabled: false` to disable. + * (Membrane handles this). Omit or set `enabled: false` to disable, except + * on a model whose thinking is always on (Fable, Mythos), which thinks + * whatever this says. */ thinking?: { enabled: boolean; @@ -162,10 +164,64 @@ export interface AgentConfig { /** * How thinking content is returned: 'summarized' (readable summary) or * 'omitted' (empty thinking field, signature only). Fable 5 / Opus 4.7+ - * default to 'omitted' — set 'summarized' to receive thinking text. + * default to 'omitted' — set 'summarized' to receive thinking text. A + * model whose thinking is always on (Fable, Mythos) is sent a thinking + * config only to carry `thinkingBinding.dropBlock`, so this reaches it + * only with that set. */ display?: 'summarized' | 'omitted'; }; + /** + * How the provider binds this agent's signed thinking to the conversation + * before it. A host sets it from what it knows of the agent's model and + * route. + */ + thinkingBinding?: { + /** + * Ask the provider to drop a signed thinking block whose earlier + * conversation has changed, as a fold changes it, rather than refuse the + * request. An account that enforces the binding otherwise answers such a + * request with a 400. One that doesn't decides for itself: the four + * accounts tried on 2026-10-18 dropped such blocks silently, so there the + * model sees nothing different and the report is what's new, and one + * that would show such a block to the model is asked to drop it instead. + * + * It's sent as `thinking.block_binding.prefix_mismatch_behavior: + * "drop_block"` with the `thinking-binding-controls-2026-08-01` beta, + * which membrane's Anthropic and Bedrock adapters carry (Bedrock's + * acceptance of the beta hasn't been tried). On any other adapter it does + * nothing, so a host should refuse it there. A thinking config the host + * sends itself, through `request.extra` or as `thinking` in + * `providerParams`, replaces the one membrane builds, control included. + * + * On a model whose thinking is always on (Fable, Mythos), membrane + * carries it in an adaptive thinking config whatever `thinking.enabled` + * says, with `thinking.display` when that's set, so such an agent's + * `display` takes effect only with this setting. On any other model, it + * goes only in the thinking config membrane sends while thinking is on. + * Membrane tells the first kind by its id, which names `fable` or + * `mythos`. Opus 5.5, Haiku 5.5, Sonnet 5.5, Sonnet 5, Fable 5 and Fable + * 5.1 accepted the config on 2026-10-18. Mythos hasn't been tried, and + * membrane notes that the family has refused thinking configs with a 400 + * before, so for a Mythos model the setting may fail every request. + * + * On a framework turn, each call's report of the blocks the provider + * dropped goes to its `inference:usage` trace, and neither that call nor + * its stream's later calls feed the context strategy's calibration, since + * the provider doesn't bill a block it dropped and the strategy still + * counts it. After a fold, each block after the stretch it rewrote is + * bound to a conversation the request no longer holds. Where the context + * strategy still sends such a block, the provider drops it on every call + * until a later fold takes it out of the context, and while such blocks + * are sent, no stream gives a sample and the calibration keeps the value + * it had. A strategy that leaves that thinking out at the fold sends none + * to drop. The direct API carries the control too, but reads no report: it + * traces no usage and feeds no calibration. + * + * Default false: nothing is sent, and the account's default applies. + */ + dropBlock?: boolean; + }; /** * How to handle a model content-policy refusal (`stop_reason: refusal`). * When `autoRewind` is on, a refused turn triggers an automatic rewind: the diff --git a/src/types/trace.ts b/src/types/trace.ts index 1b48fb3f8..e754d8238 100644 --- a/src/types/trace.ts +++ b/src/types/trace.ts @@ -1,3 +1,4 @@ +import type { ThinkingBindingEntry } from '@animalabs/membrane'; import type { ProcessEvent } from './events.js'; import type { ModuleProcessResponse } from './framework.js'; import type { SessionUsage } from '../usage/types.js'; @@ -193,6 +194,20 @@ export type TraceEvent = type: 'inference:usage'; agentName: string; tokenUsage: { input: number; output: number; cacheCreation?: number; cacheRead?: number }; + /** + * The provider's report on the signed thinking of the call this event + * closes (membrane's `RoundReport.thinking`): blocks it dropped before + * the model saw them, and blocks it showed the model anyway because the + * check isn't enforced. `tokenUsage` beside it is the stream's running + * total, not this call's. The provider reports them for a request that + * carried the binding control, whoever asked for it: + * `thinkingBinding.dropBlock`, or a thinking config the host sends + * itself. Present only when the report names any block. + */ + thinking?: { + dropped: ThinkingBindingEntry[]; + mismatchAllowed: ThinkingBindingEntry[]; + }; }) | (TraceEventBase & { type: 'inference:stream_resumed'; diff --git a/test/agent-cache-ttl.test.ts b/test/agent-cache-ttl.test.ts index 0b9ab51bd..ee084de09 100644 --- a/test/agent-cache-ttl.test.ts +++ b/test/agent-cache-ttl.test.ts @@ -74,14 +74,17 @@ describe('the direct API sends the agent request settings', () => { it("carries every field the agent's settings give a request", async () => { const { agent, sent } = directAgent({ - maxTokens: 1234, temperature: 0.3, thinking: { enabled: true, budgetTokens: 2048 }, + maxTokens: 1234, temperature: 0.3, thinking: { enabled: true, budgetTokens: 2048 }, thinkingBinding: { dropBlock: true }, promptCaching: false, cacheTtl: '5m', prefillUserMessage: 'scaffold', providerParams: { reasoning: { effort: 'low' } }, }); await agent.runInference([]); assert.equal(sent.length, 1); const { messages: _m, system: _s, tools: _t, ...settings } = sent[0]; assert.deepEqual(settings, { - config: { model: 'test-model', maxTokens: 1234, temperature: 0.3, thinking: { enabled: true, budgetTokens: 2048 } }, + config: { + model: 'test-model', maxTokens: 1234, temperature: 0.3, + thinking: { enabled: true, budgetTokens: 2048, blockBinding: { prefixMismatchBehavior: 'drop_block' } }, + }, promptCaching: false, cacheTtl: '5m', prefillUserMessage: 'scaffold', @@ -94,7 +97,7 @@ describe('the direct API sends the agent request settings', () => { // too: an unset setting reaches both paths as the same resolved value. const parityCases: Array<[string, Partial]> = [ ['every setting set', { - maxTokens: 1234, temperature: 0.3, thinking: { enabled: true, budgetTokens: 2048 }, + maxTokens: 1234, temperature: 0.3, thinking: { enabled: true, budgetTokens: 2048 }, thinkingBinding: { dropBlock: true }, promptCaching: false, cacheTtl: '5m', prefillUserMessage: 'scaffold', providerParams: { reasoning: { effort: 'low' } }, }], ['no setting set', {}], diff --git a/test/helpers/mock-membrane.ts b/test/helpers/mock-membrane.ts index e3d282b3e..b91001ba8 100644 --- a/test/helpers/mock-membrane.ts +++ b/test/helpers/mock-membrane.ts @@ -94,7 +94,11 @@ export class MockYieldingStream implements YieldingStream { } if (response.usage) { - this.events.push({ type: 'usage', usage: response.usage } as StreamEvent); + // A response may carry the round report its usage event closes + // (`round`, a test-only field), as membrane 0.6.0's yielding streams + // send it. + const round = (response as { round?: unknown }).round; + this.events.push({ type: 'usage', usage: response.usage, ...(round ? { round } : {}) } as StreamEvent); } if (response.toolCalls.length > 0) { diff --git a/test/thinking-binding.test.ts b/test/thinking-binding.test.ts new file mode 100644 index 000000000..d7e4a0aef --- /dev/null +++ b/test/thinking-binding.test.ts @@ -0,0 +1,287 @@ +/** + * `thinkingBinding.dropBlock` asks the provider to drop a signed thinking + * block whose earlier conversation changed, as a fold changes it, rather than + * refuse the request. These pin what an agent sends for it, through + * membrane's Anthropic adapter to the wire; what a call whose report names + * dropped blocks does to the trace and to the context strategy's + * calibration; and the setting's refusals. + */ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { createServer, type IncomingHttpHeaders, type Server } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { existsSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { ContextManager } from '@animalabs/context-manager'; +import { AnthropicAdapter, Membrane } from '@animalabs/membrane'; +import type { DetailedUsage, Membrane as MembraneType, NormalizedResponse } from '@animalabs/membrane'; + +import { Agent, AgentFramework } from '../src/index.js'; +import type { EventResponse, Module, ProcessEvent, ToolCall, TraceEvent } from '../src/index.js'; +import { createMockResponse, MockMembrane } from './helpers/mock-membrane.js'; + +const DROP_BLOCK = 'true asks the provider to drop a signed thinking block whose earlier conversation changed rather than refuse the request, ' + + 'and false or leaving it out sends nothing, so the account\'s default applies'; +const BINDING = { prefixMismatchBehavior: 'drop_block' }; + +const agentWith = (settings: Record) => ({ + name: 'agent', + model: 'claude-opus-5-5', + systemPrompt: 'test', + ...settings, +}); + +/** An Agent built directly. Building an activation request needs only + * compile(), so membrane is touched only when a test sends the request. */ +const oneMessage = { + compile: async () => ({ messages: [{ participant: 'user', content: [{ type: 'text', text: 'Hello.' }] }], systemInjections: [] }), +} as unknown as ContextManager; +const newAgent = (settings: Record) => + new Agent(agentWith(settings) as never, oneMessage, {} as MembraneType); +const sentThinking = async (settings: Record) => + (await newAgent(settings).buildActivationRequest([])).config.thinking; + +/** A stand-in for the Messages API that records each request's body and + * headers and answers with one short text reply. */ +async function messagesApi(): Promise<{ baseURL: string; seen: Array<{ body: Record; headers: IncomingHttpHeaders }>; server: Server }> { + const seen: Array<{ body: Record; headers: IncomingHttpHeaders }> = []; + const server = createServer((req, res) => { + let raw = ''; + req.on('data', (chunk) => { raw += chunk; }); + req.on('end', () => { + const body = JSON.parse(raw) as Record; + seen.push({ body, headers: req.headers }); + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ + id: 'msg_test', type: 'message', role: 'assistant', model: body.model, + content: [{ type: 'text', text: 'ok' }], stop_reason: 'end_turn', stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 1 }, + })); + }); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + return { baseURL: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, seen, server }; +} + +describe('thinkingBinding.dropBlock', () => { + it('adds the binding control to the thinking config the agent sends, and to no request without the setting', async () => { + // Each agent is given a config of its own, and the expectations are + // literals, so a control written into the agent's config would show. + const thinking = () => ({ enabled: true, budgetTokens: 2048 }); + const given = thinking(); + assert.deepEqual(await sentThinking({ thinking: given, thinkingBinding: { dropBlock: true } }), + { enabled: true, budgetTokens: 2048, blockBinding: BINDING }); + assert.deepEqual(given, { enabled: true, budgetTokens: 2048 }, 'the agent\'s own config is left as it was'); + for (const none of [undefined, null, false]) { + assert.deepEqual(await sentThinking({ thinking: none, thinkingBinding: { dropBlock: true } }), { enabled: false, blockBinding: BINDING }, + `an agent whose thinking is ${String(none)} still carries the control, for a model whose thinking is always on`); + } + assert.deepEqual(await sentThinking({ thinking: thinking() }), { enabled: true, budgetTokens: 2048 }); + assert.deepEqual(await sentThinking({ thinking: thinking(), thinkingBinding: { dropBlock: false } }), { enabled: true, budgetTokens: 2048 }); + assert.equal(await sentThinking({ thinkingBinding: { dropBlock: false } }), undefined); + }); + + it('reaches the wire through membrane\'s Anthropic adapter as block_binding with its beta, on the models that take it', async () => { + const api = await messagesApi(); + const membrane = new Membrane(new AnthropicAdapter({ apiKey: 'test-key', baseURL: api.baseURL, cacheKeepalive: { enabled: false } })); + const send = async (settings: Record) => { + const request = await newAgent(settings).buildActivationRequest([]); + await membrane.complete(request); + const { body, headers } = api.seen[api.seen.length - 1]!; + return { thinking: body.thinking, beta: String(headers['anthropic-beta'] ?? '').split(',').filter(Boolean) }; + }; + const binding = { prefix_mismatch_behavior: 'drop_block' }; + const BETA = 'thinking-binding-controls-2026-08-01'; + try { + const opus = await send({ thinking: { enabled: true, budgetTokens: 2048 }, thinkingBinding: { dropBlock: true } }); + assert.deepEqual(opus.thinking, { type: 'enabled', budget_tokens: 2048, block_binding: binding }); + assert.ok(opus.beta.includes(BETA), `the beta goes with it: ${opus.beta.join(',')}`); + + const fable = await send({ model: 'claude-fable-5-1', thinkingBinding: { dropBlock: true } }); + assert.deepEqual(fable.thinking, { type: 'adaptive', block_binding: binding }, + 'a model whose thinking is always on carries it in an adaptive config, with no thinking config of the agent\'s own'); + assert.ok(fable.beta.includes(BETA)); + + // On such a model the agent's `display` goes out in that config, and so only with the setting. + const shown = { enabled: false, display: 'summarized' }; + const fableShown = await send({ model: 'claude-fable-5-1', thinking: shown, thinkingBinding: { dropBlock: true } }); + assert.deepEqual(fableShown.thinking, { type: 'adaptive', display: 'summarized', block_binding: binding }, 'display beside the control'); + const fableUnset = await send({ model: 'claude-fable-5-1', thinking: shown }); + assert.equal(fableUnset.thinking, undefined, 'without the setting, such a model is sent no thinking config, display included'); + assert.equal(fableUnset.beta.includes(BETA), false); + + const off = await send({ thinkingBinding: { dropBlock: true } }); + assert.equal(off.thinking, undefined, 'another model with thinking off sends no thinking config, so no control'); + assert.equal(off.beta.includes(BETA), false); + + const unset = await send({ thinking: { enabled: true, budgetTokens: 2048 } }); + assert.deepEqual(unset.thinking, { type: 'enabled', budget_tokens: 2048 }, 'without the setting nothing changes'); + assert.equal(unset.beta.includes(BETA), false); + } finally { + await new Promise((resolve) => api.server.close(() => resolve())); + } + }); + + it('puts a call\'s report in its inference:usage trace, and gives no calibration sample from a call that dropped blocks or any later call of its stream', async () => { + const dir = mkdtempSync(join(tmpdir(), 'af-thinking-binding-')); + const membrane = new MockMembrane(); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store.chronicle'), membrane: membrane.asMembrane(), agents: [], + modules: [new EchoTools()], syncIntervalMs: 0, + }); + const traces: TraceEvent[] = []; + framework.onTrace((event) => traces.push(event)); + framework.start(); + try { + const dropped = { reason: 'prefix_binding_mismatch', path: 'messages.1.content.0', message: 1 }; + const shown = { reason: 'prefix_binding_mismatch', path: 'messages.3.content.0', message: 3 }; + // Four calls of one stream: no report, a block shown though it failed + // the check, a block dropped, and a report that names nothing. + membrane.pushResponse(withRound(createMockResponse([{ type: 'tool_use', id: 'c1', name: 'test--echo', input: {} }], 'tool_use'), + { inputTokens: 100, outputTokens: 10, cacheCreationTokens: 0, cacheReadTokens: 400 })); + membrane.pushResponse(withRound(createMockResponse([{ type: 'tool_use', id: 'c2', name: 'test--echo', input: {} }], 'tool_use'), + { inputTokens: 130, outputTokens: 15, cacheCreationTokens: 0, cacheReadTokens: 900 }, { dropped: [], mismatchAllowed: [shown] })); + membrane.pushResponse(withRound(createMockResponse([{ type: 'tool_use', id: 'c3', name: 'test--echo', input: {} }], 'tool_use'), + { inputTokens: 150, outputTokens: 20, cacheCreationTokens: 60, cacheReadTokens: 900 }, { dropped: [dropped], mismatchAllowed: [] })); + membrane.pushResponse(withRound(createMockResponse([{ type: 'text', text: 'Done.' }]), + { inputTokens: 170, outputTokens: 25, cacheCreationTokens: 60, cacheReadTokens: 1400 }, { dropped: [], mismatchAllowed: [] })); + + const { agent, contextManager } = await framework.createEphemeralAgent({ + name: 'worker', model: 'claude-opus-5-5', systemPrompt: 'Do the task.', allowedTools: 'all', + proseRouting: 'disabled', thinkingBinding: { dropBlock: true }, + }); + const calibrated: number[] = []; + const strategy = contextManager.getStrategy() as unknown as { reportRealInputTokens?: (n: number) => void }; + strategy.reportRealInputTokens = (n) => { calibrated.push(n); }; + contextManager.addMessage('user', [{ type: 'text', text: 'Run once.' }]); + await framework.runEphemeralToCompletion(agent, contextManager); + + const usage = traces.filter((event): event is Extract => event.type === 'inference:usage'); + assert.equal(usage.length, 4); + assert.equal('thinking' in usage[0]!, false, 'a call with no report carries no thinking field'); + assert.deepEqual(usage[1]!.thinking, { dropped: [], mismatchAllowed: [shown] }, + 'a block shown to the model though it failed the check is reported too'); + assert.deepEqual(usage[2]!.thinking, { dropped: [dropped], mismatchAllowed: [] }); + assert.equal('thinking' in usage[3]!, false, 'a call whose report names nothing carries no thinking field'); + assert.deepEqual(calibrated, [100 + 0 + 400, 30 + 0 + 500], + 'each call before the drop is given, as its own share of the stream\'s usage, a shown block being billed; ' + + 'the call that dropped one and the call after it are not, since the strategy would take the first call given after its compile'); + } finally { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('gives no sample from a stream whose first call dropped blocks, and calibrates the next stream, which dropped none', async () => { + // A stream's skip is the stream's: a stream whose first call drops blocks gives no sample, and the next stream that + // drops none is a sample again. Kept anywhere longer-lived than the stream, such as the agent, the skip would stop + // calibration for good. One agent, two turns, two streams. + const dir = mkdtempSync(join(tmpdir(), 'af-thinking-binding-streams-')); + const membrane = new MockMembrane(); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store.chronicle'), membrane: membrane.asMembrane(), + agents: [{ ...agentWith({ thinkingBinding: { dropBlock: true } }), proseRouting: 'disabled' } as never], + modules: [new InputModule()], syncIntervalMs: 0, maintenanceIntervalMs: 0, + }); + const completed: TraceEvent[] = []; + framework.onTrace((event) => { if (event.type === 'inference:completed') completed.push(event); }); + framework.start(); + try { + const calibrated: number[] = []; + const strategy = framework.getAgent('agent')!.getContextManager().getStrategy() as unknown as { reportRealInputTokens?: (n: number) => void }; + strategy.reportRealInputTokens = (n) => { calibrated.push(n); }; + const dropped = { reason: 'prefix_binding_mismatch', path: 'messages.1.content.0', message: 1 }; + const turn = async (text: string) => { + const before = completed.length; + framework.pushEvent({ type: 'external-message', source: 'test', content: text, metadata: {} } as never); + await framework.runUntilIdle(); + for (const until = Date.now() + 10_000; completed.length === before && Date.now() < until;) await new Promise((r) => setTimeout(r, 10)); + assert.equal(completed.length, before + 1, `the turn for "${text}" completed`); + }; + + membrane.pushResponse(withRound(createMockResponse([{ type: 'text', text: 'First.' }]), + { inputTokens: 100, outputTokens: 10, cacheCreationTokens: 0, cacheReadTokens: 400 }, { dropped: [dropped], mismatchAllowed: [] })); + await turn('Run once.'); + assert.equal(membrane.calls.length, 1, 'the first turn\'s stream ran'); + assert.deepEqual(calibrated, [], 'its first call dropped a block: no sample'); + + membrane.pushResponse(withRound(createMockResponse([{ type: 'text', text: 'Second.' }]), + { inputTokens: 120, outputTokens: 10, cacheCreationTokens: 0, cacheReadTokens: 500 }, { dropped: [], mismatchAllowed: [] })); + await turn('Run again.'); + assert.equal(membrane.calls.length, 2, 'the second turn\'s stream ran'); + assert.deepEqual(calibrated, [120 + 0 + 500], 'the next stream dropped none, and is a sample again'); + } finally { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('refuses a setting that is not true or false, and a thinkingBinding other than an object, null or false', async () => { + const dir = mkdtempSync(join(tmpdir(), 'af-thinking-binding-refused-')); + const storePath = join(dir, 'store.chronicle'); + const createWith = (settings: Record) => AgentFramework.create({ + storePath, membrane: new MockMembrane().asMembrane(), agents: [agentWith(settings)], modules: [], syncIntervalMs: 0, + } as never); + try { + for (const [settings, message] of [ + [{ thinkingBinding: { dropBlock: 'true' } }, `Agent "agent" thinkingBinding.dropBlock must be true or false, unquoted, not a string: ${DROP_BLOCK}`], + [{ thinkingBinding: true }, 'Agent "agent" thinkingBinding must be an object, such as { dropBlock: true }, not a boolean'], + ] as const) { + await assert.rejects(createWith(settings), (error: Error) => { + assert.equal(error.message, message); + return true; + }); + assert.equal(existsSync(storePath), false, 'the refused config opened no store'); + assert.throws(() => newAgent(settings), (error: Error) => { + assert.equal(error.message, message); + return true; + }, 'new Agent() refuses it too'); + } + for (const settings of [{ thinkingBinding: null }, { thinkingBinding: false }, { thinkingBinding: { dropBlock: null } }]) { + assert.equal(await sentThinking(settings), undefined, `${JSON.stringify(settings)} counts as left out`); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + +/** Turns each external message into a user message and a request for inference. */ +class InputModule implements Module { + readonly name = 'input'; + async start() {} + async stop() {} + getTools() { return []; } + async handleToolCall() { return { success: false, isError: true, error: 'none' }; } + async onProcess(event: ProcessEvent): Promise { + if (event.type !== 'external-message') return {}; + return { addMessages: [{ participant: 'User', content: [{ type: 'text', text: String(event.content) }] }], requestInference: true }; + } +} + +class EchoTools implements Module { + readonly name = 'test'; + async start() {} + async stop() {} + async onProcess() { return {}; } + getTools() { + return [{ name: 'echo', description: 'echo', inputSchema: { type: 'object' as const, properties: {} } }]; + } + async handleToolCall(_call: ToolCall) { + return { success: true, data: {} }; + } +} + +type Entry = { reason: string; path: string; message?: number }; + +/** A response whose usage event carries membrane's round report with the + * given thinking entries, or no report when none are given. Usage is + * cumulative across a stream's rounds, as membrane reports it. */ +function withRound( + response: NormalizedResponse, + usage: DetailedUsage, + thinking?: { dropped: Entry[]; mismatchAllowed: Entry[] }, +): NormalizedResponse { + return Object.assign(response, thinking ? { usage, round: { thinking } } : { usage }); +}