From 37ee0354ffa4a5f82cf93cceec3f6027145a7f50 Mon Sep 17 00:00:00 2001 From: Esther-2584 Date: Fri, 23 Oct 2026 05:53:46 -0400 Subject: [PATCH] test(calibration): pin the injected system prompt and the cache-write term in the sample; say the fix waits for context-manager #324 From our internal review of #763. - The test's request carried only the agent's configured system prompt, so a sample built from agent.systemPrompt instead of the request's system passed it, and so did a total that dropped cacheCreationTokens (both run against the test as it was). Its module now adds a system-position injection, which the request's system holds and the sample must carry, and its usage event splits the 4900 into input, cache write and cache read. Each of the two variants now fails it. - The changelog fragment says that context-manager 0.20.0 is among the versions that ignore the second argument, so the multiplier learns as before until the host runs a release with context-manager #324. Co-Authored-By: Claude Opus 5.5 --- changelog.d/calibration-request-overhead.fixed.md | 4 +++- test/calibration-request-overhead.test.ts | 11 ++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/changelog.d/calibration-request-overhead.fixed.md b/changelog.d/calibration-request-overhead.fixed.md index 56ff3d468..80eb97941 100644 --- a/changelog.d/calibration-request-overhead.fixed.md +++ b/changelog.d/calibration-request-overhead.fixed.md @@ -4,4 +4,6 @@ request, not of its compiled window alone (context-manager #323). Before, a tool-heavy agent's calibration multiplier learned its tool schemas as a per-token rate, or sat at 1.00 rejecting every sample as out of band. A - context-manager without the second argument ignores it. + context-manager without the second argument ignores it, 0.20.0 included, + so the multiplier learns as before until the host runs a context-manager + release with context-manager #324. diff --git a/test/calibration-request-overhead.test.ts b/test/calibration-request-overhead.test.ts index 624c8ccfd..398da026d 100644 --- a/test/calibration-request-overhead.test.ts +++ b/test/calibration-request-overhead.test.ts @@ -11,7 +11,7 @@ import { mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { NormalizedRequest, StreamEvent, YieldingStream } from '@animalabs/membrane'; -import { AutobiographicalStrategy } from '@animalabs/context-manager'; +import { AutobiographicalStrategy, type ContextInjection } from '@animalabs/context-manager'; import { AgentFramework } from '../src/index.js'; import type { EventResponse, @@ -47,7 +47,7 @@ class UsageMembrane extends MockMembrane { streamYielding(request: NormalizedRequest): YieldingStream { this.calls.push(request); return new ScriptedStream([ - { type: 'usage', usage: { inputTokens: 900, outputTokens: 5, cacheReadTokens: 4000 } } as StreamEvent, + { type: 'usage', usage: { inputTokens: 900, outputTokens: 5, cacheCreationTokens: 600, cacheReadTokens: 3400 } } as StreamEvent, { type: 'complete', response: createMockResponse([{ type: 'text', text: 'ok' }]) } as StreamEvent, ]) as unknown as YieldingStream; } @@ -68,6 +68,10 @@ class ToolModule implements Module { async handleToolCall(_call: ToolCall): Promise { return { success: true, data: 'found' }; } + /** A system-position injection, which the request's system prompt carries. */ + async gatherContext(_agentName: string): Promise { + return [{ namespace: 'server', position: 'system', content: [{ type: 'text', text: 'The server is up.' }] }]; + } async onProcess(event: ProcessEvent, _state: ProcessState): Promise { if (event.type !== 'external-message') return {}; return { @@ -99,7 +103,8 @@ test('the calibration sample carries the system prompt and tools the request sen const sent = membrane.calls[0]; assert.ok(sent, 'the turn sent a request'); - assert.ok(sent.system && sent.system.length > 0, 'the request had a system prompt'); + assert.ok(sent.system?.includes('You are the resident.'), 'the request had the agent\'s system prompt'); + assert.ok(sent.system?.includes('The server is up.'), 'and the module\'s system-position injection'); assert.ok(sent.tools?.some((tool) => tool.name.endsWith('lookup')), 'the request advertised the tool'); assert.equal(samples.length, 1, 'one call, one sample'); assert.equal(samples[0].total, 4900, 'input + cache write + cache read');