From 86984f6f6bde8e80f4feab24894b9673c7d18234 Mon Sep 17 00:00:00 2001 From: Esther-2584 Date: Fri, 23 Oct 2026 06:28:55 -0400 Subject: [PATCH 1/8] fix(calibration): price message-position injections into the sample; two agreeing declared samples move a store that learned the overhead (#323) From our internal review of #324. - Message-position injections. ContextManager.compile inserts beforeUser and afterUser injections after the select, so the request carries them and the window estimate doesn't. With system and tools declared, an injection large beside its window was learned as a per-token rate, or, as a consistent declared out-of-band run, at the clamp. compile now passes their content to the select as SelectOptions.messageInjections; the adaptive select prices it where it arms the sample, at the multiplier it priced the window at, and the sample adds it to the estimate, declared or not. A dry select puts the count back. system injections stay in the caller's declared system prompt. - A store that learned the overhead as a rate. Declared samples stepped such a multiplier back over about ten applied changes, each a reprice of every message and summary. The record now carries declaredOverhead once declared samples agree with its multiplier. On a record without it, a declared sample inside the band is held in memory, and the next declared sample decides: within OUT_OF_BAND_RUN_AGREEMENT their mean replaces the multiplier, and otherwise the EMA steps; either way the record is marked. Two, not one, since this reaches every store at upgrade and one wild sample shouldn't set a correct store's ruler. A restart or branch switch, a provisional replacement or a run ends a held sample; an undeclared sample doesn't, since an implied multiplier doesn't depend on the multiplier in force. The flag is read on load, kept through a re-stamp, and dropped by an undeclared write that moves the multiplier. A provisional multiplier, or a run, still moves on one sample. - A consistent out-of-band run agrees with its first sample, not the one before, as the changelog says, so a drifting run isn't learned. - Declared means passed, {} included and null not, throughout. - RequestOverhead is exported from the package root, and its doc says passing it claims completeness. Docs, comments and the declared rejection's words follow the code; the changelog keeps the provisional seed for a store whose multiplier puts every compile over budget. - 29 tests in test/calibration-overhead.test.ts; #324's first test sends two samples, since one is held. Co-Authored-By: Claude Opus 5.5 --- .../323-calibration-request-overhead.fixed.md | 40 +- src/context-manager.ts | 8 +- src/index.ts | 1 + src/strategies/autobiographical.ts | 144 ++++-- src/types/strategy.ts | 19 +- test/calibration-overhead.test.ts | 423 +++++++++++++++++- 6 files changed, 591 insertions(+), 44 deletions(-) diff --git a/changelog.d/323-calibration-request-overhead.fixed.md b/changelog.d/323-calibration-request-overhead.fixed.md index 7c06e8fb..3b1a141b 100644 --- a/changelog.d/323-calibration-request-overhead.fixed.md +++ b/changelog.d/323-calibration-request-overhead.fixed.md @@ -5,16 +5,34 @@ estimate doesn't, so a tool-heavy agent's multiplier learned its schemas as a per-token rate and folded history the budget had room for, or sat at 1.00 rejecting every sample as out of band. A caller that passes nothing - gets the old comparison. + gets the old comparison, plus the compile's message-position injections + (below). - With the overhead declared, three out-of-band calibration samples in a row - whose implied multipliers agree within 10% are learned, outright, at the - clamp edge, instead of being rejected for good. At 1.00 the ratio and the - implied multiplier are the same number, so a store whose true ratio was - past 1.8, where every pricing-epoch discard restarts, could never learn. - Undeclared samples are rejected as before, and the rejection line says the - caller may be leaving out the system prompt and tools it sends. + whose implied multipliers agree within 10% of the first are learned, + outright, at the clamp edge, instead of being rejected for good. At 1.00 + the ratio and the implied multiplier are the same number, so a store whose + true ratio was past 1.8, where every pricing-epoch discard restarts, could + never learn. Undeclared samples are rejected as before, and the rejection + line says the caller may be leaving out the system prompt and tools it + sends. +- The sample also counts the compile's message-position injections + (`beforeUser` and `afterUser`), which the request carries and the window + estimate doesn't: `ContextManager.compile` passes their content to the + select as `SelectOptions.messageInjections`, and the select prices it with + the window. Left out, an agent whose injections are large beside its window + learned them as a per-token rate, or, with the overhead declared, at the + clamp. - **Host operators:** a multiplier learned before the caller declared its - overhead holds that overhead, and steps back down over about ten compiles - once samples declare it. To relearn in one sample, seed it as provisional - on the stopped store: `setStateJson('/autobio:calibration', - { multiplier, at, pricing, provisional: true })`. + overhead holds that overhead. Inside the calibration band, the first two + samples that declare it replace the multiplier with their mean when they + agree within 10%, and the EMA steps as before when they don't; either way + the record then carries `declaredOverhead: true`, so later declared + samples step by the EMA. (Out of the band, the three-sample run above + applies.) The first of the two is held in memory, so a restart between + them costs one more sample. An undeclared sample that moves the multiplier + writes the record without the flag. A store whose multiplier puts every + compile over budget gets no sample to move it, as before: to unwedge it, + seed the record as provisional on the stopped store, and the refused + compile drops it to 1: `setStateJson('/autobio:calibration', + { multiplier, at, pricing: , provisional: true })`. + `RequestOverhead` is exported from the package. diff --git a/src/context-manager.ts b/src/context-manager.ts index ea298d8c..04fca26e 100644 --- a/src/context-manager.ts +++ b/src/context-manager.ts @@ -1178,12 +1178,16 @@ export class ContextManager { const branch = branchRefOf(this.store); const view = this.strategyMessageView(); - // Get selected entries from strategy + // Get selected entries from strategy. The injections it will sit among + // go with it, for its calibration's count of the request (SelectOptions). + const messageInjections = (injections ?? []) + .filter((inj) => inj.position === 'beforeUser' || inj.position === 'afterUser') + .map((inj) => inj.content); const entries = this.strategy.select( view, this.contextLog.createView(), effectiveBudget, - opts + messageInjections.length > 0 ? { ...opts, messageInjections } : opts ); const selectionCause = this.strategy.takeSelectionCause?.(); // Read now, before anything else selects: these are this select's own. diff --git a/src/index.ts b/src/index.ts index af88b229..82687b0b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -81,6 +81,7 @@ export type { RenderedLayout, LayoutUnit, // Strategy types + RequestOverhead, MessageStoreView, ContextLogView, StrategyContext, diff --git a/src/strategies/autobiographical.ts b/src/strategies/autobiographical.ts index 615138a7..9472770a 100644 --- a/src/strategies/autobiographical.ts +++ b/src/strategies/autobiographical.ts @@ -1856,8 +1856,10 @@ function imageGroup(block: ContentBlock): unknown { } /** Out-of-band calibration samples in a row it takes to learn from them, and - * how close each one's implied multiplier must be to the one before it to - * continue the run (see reportRealInputTokens). */ + * how close each one's implied multiplier must be to the run's first to + * continue the run (see reportRealInputTokens). The agreement also decides + * whether two declared samples move a store whose multiplier wasn't + * learned from declared samples. */ const OUT_OF_BAND_RUN_LENGTH = 3; const OUT_OF_BAND_RUN_AGREEMENT = 0.1; @@ -2852,6 +2854,7 @@ export class AutobiographicalStrategy implements ResettableStrategy { this._lastRenderStats = null; this._lastSelectStats = null; this._lastCompileEstimate = 0; + this._lastCompileInjected = 0; this._storeView = null; // Prices memoized over the view just cleared: the next one may estimate otherwise. this._answerCostCache.clear(); @@ -2861,7 +2864,9 @@ export class AutobiographicalStrategy implements ResettableStrategy { this._calibrationLoaded = false; this._calibrationReset = null; this._calibrationProvisional = null; + this._calibrationDeclared = false; this._outOfBandRun = null; + this._declaredCandidate = null; this._imageStripReplayMemo = null; this._lastKvStable = null; this._storeOrder = new Map(); @@ -13862,23 +13867,36 @@ export class AutobiographicalStrategy implements ResettableStrategy { this._lastCompileEstimate = rs.head.tokens + rs.tail.tokens + rs.middleRaw.tokens + rs.summaries.l1.tokens + rs.summaries.l2.tokens + rs.summaries.l3.tokens; + // What the compile inserts among these entries after the select (its + // message-position injections) is in the request and not in the + // window: priced here, at the multiplier the window was priced at, so + // the sample counts it as the request does (#323). + let injected = 0; + for (const content of opts?.messageInjections ?? []) { + injected += store.estimateTokens({ content } as unknown as StoredMessage); + } + this._lastCompileInjected = injected; this._calibrationArmed = true; // exactly one sample per compile } return merged; } private _lastCompileEstimate = 0; + /** The armed compile's message-position injections, priced with its window. */ + private _lastCompileInjected = 0; private _storeView: MessageStoreView | null = null; /** * Closed-loop estimator calibration (2026-07-12). Feed the REAL input total * for a request built from the latest compile (fresh + cache_read + * cache_creation, as the provider reports it), and what the request sent - * besides the window: its system prompt and tool definitions (#323). Those - * are priced as the store prices the window and added to the estimate, so - * the sample compares the whole request with the whole request; left out, - * the multiplier learns them as a per-token rate, which no value of it can - * be (see RequestOverhead). Maintains an EMA of real/estimated and applies + * besides the window and the compile's message-position injections: its + * system prompt and tool definitions (#323). The injections are priced at + * the compile (SelectOptions.messageInjections) and the overhead here, as + * the store prices the window, and both are added to the estimate, so the + * sample compares the whole request with the whole request; left out, the + * multiplier learns them as a per-token rate, which no value of it can be + * (see RequestOverhead). Maintains an EMA of real/estimated and applies * it as the store's global multiplier, persisted across restarts. The * per-class rates carry the shape; this carries the residual level. */ @@ -13898,21 +13916,30 @@ export class AutobiographicalStrategy implements ResettableStrategy { // exhausting the picker on every wake. One sample per compile, always. if (!this._calibrationArmed) return; this._calibrationArmed = false; + // Declared: the caller passed what the request sent besides the window + // and the compile's injections ({} included). A JS caller's null isn't. + const declared = overhead != null; // The overhead in the window's units (current calibrated), so the implied // multiplier below is real / raw(window + overhead) and doesn't depend on // the multiplier it's measured at: one sample states it. const overheadTokens = overhead ? this.priceRequestOverhead(overhead) : 0; - const est = window + overheadTokens; + const injected = this._lastCompileInjected; + const est = window + injected + overheadTokens; + const parts = [ + ...(overheadTokens > 0 ? [`${Math.round(overheadTokens / 1000)}k system+tools`] : []), + ...(injected > 0 ? [`${Math.round(injected / 1000)}k injected`] : []), + ]; const counts = `(est=${Math.round(est / 1000)}k` + - `${overheadTokens > 0 ? ` incl. ${Math.round(overheadTokens / 1000)}k system+tools` : ''}` + + `${parts.length > 0 ? ` incl. ${parts.join(', ')}` : ''}` + ` real=${Math.round(realTotal / 1000)}k)`; const ratio = realTotal / est; const current = this._calibration; const observed = ratio * current; // back out the multiplier already applied // SANITY BAND: a representative sample sits near 1. Anything wilder is a // structural mismatch (a request we didn't compile, a partial compile, a - // provider quirk) — never evidence about chars-per-token. Log, don't learn. + // provider quirk) — never evidence about chars-per-token. Log, don't learn, + // except a declared consistent run (below). // The band is also checked on the IMPLIED raw multiplier: when the // multiplier sits at a clamp edge because the per-class rates were wrong // (signed thinking at a flat 600) and the rates are then fixed, every @@ -13926,8 +13953,11 @@ export class AutobiographicalStrategy implements ResettableStrategy { // the same number, so a store whose true ratio is past the band (every // epoch discard restarts there) rejects every sample and never learns. // A single wild sample is still noise, but a run of them that agree with - // each other is a ruler that's off, not a request we didn't compile: the - // third in a row is learned, outright, which lands it on the clamp edge. + // the run's first, within OUT_OF_BAND_RUN_AGREEMENT, is a ruler that's + // off, not a request we didn't compile: the third in a row is learned, + // outright, which lands it on the clamp edge. Agreement is with the first, + // not the one before, so a drifting run (a growing unpriced part) isn't + // learned as a constant. // Only for samples whose caller declared what it sent besides the window: // undeclared, a consistent ratio past the band is what tool schemas and a // system prompt look like, and learning it would fold the window to the @@ -13936,18 +13966,19 @@ export class AutobiographicalStrategy implements ResettableStrategy { if (!ratioInBand && !observedInBand) { const run = this._outOfBandRun; this._outOfBandRun = !overhead ? null - : run && Math.abs(observed / run.observed - 1) <= OUT_OF_BAND_RUN_AGREEMENT - ? { observed, count: run.count + 1 } - : { observed, count: 1 }; + : run && Math.abs(observed / run.first - 1) <= OUT_OF_BAND_RUN_AGREEMENT + ? { first: run.first, count: run.count + 1 } + : { first: observed, count: 1 }; const count = this._outOfBandRun?.count ?? 0; if (count < OUT_OF_BAND_RUN_LENGTH) { this.diag.error( `[estimator-calibration] REJECTED out-of-band sample real/est=${ratio.toFixed(2)} ${counts} — ` + - `not a window-shaped request; multiplier stays ${this._calibration.toFixed(2)}` + (overhead - ? ` (${count} of ${OUT_OF_BAND_RUN_LENGTH} consistent samples to learn from)` - : `; if every sample reads about the same, the caller may be leaving out the system prompt ` + - `and tools it sends (reportRealInputTokens' second argument)`), + ? `multiplier stays ${this._calibration.toFixed(2)} until ${OUT_OF_BAND_RUN_LENGTH} samples in a row ` + + `agree within ${OUT_OF_BAND_RUN_AGREEMENT * 100}% (${count} of ${OUT_OF_BAND_RUN_LENGTH})` + : `not a window-shaped request; multiplier stays ${this._calibration.toFixed(2)}; if every sample ` + + `reads about the same, the caller may be leaving out the system prompt and tools it sends ` + + `(reportRealInputTokens' second argument)`), ); return; } @@ -13957,6 +13988,44 @@ export class AutobiographicalStrategy implements ResettableStrategy { this._outOfBandRun = null; } + // A multiplier learned before samples declared their overhead learned it + // as a rate (#323). Inside the band, the first declared sample on such a + // record is held, and the next declared sample decides: if the two agree + // within OUT_OF_BAND_RUN_AGREEMENT their mean replaces the multiplier, + // instead of about ten EMA steps, each a reprice of every message and + // summary; if not, the EMA steps as for any sample. Two, not one: this + // reaches every store that samples after the upgrade, those whose + // multiplier is already right included, and one wild sample shouldn't + // set their ruler alone. The held sample is in memory only, so a restart + // between them costs one more sample; an undeclared sample between them + // leaves it, since an implied multiplier doesn't depend on the multiplier + // in force. Either way the record then says declared samples agree with + // its multiplier (declaredOverhead), and an undeclared write takes that + // back. A provisional multiplier, or a consistent run (out of the band, + // three samples), is replaced by one sample as before. + const markDeclared = declared && !this._calibrationDeclared; + let replacement: number | null = null; + if (markDeclared && !this._calibrationProvisional && !fromRun) { + const candidate = this._declaredCandidate; + if (candidate === null) { + this._declaredCandidate = observed; + this.diag.error( + `[estimator-calibration] real/est=${ratio.toFixed(2)} implies ${observed.toFixed(2)}: the first sample ` + + `with its overhead declared, held for the next; if they agree within ` + + `${OUT_OF_BAND_RUN_AGREEMENT * 100}% their mean replaces the multiplier, and if not it steps; ` + + `multiplier stays ${current.toFixed(2)} ${counts}`, + ); + return; + } + this._declaredCandidate = null; + if (Math.abs(observed / candidate - 1) <= OUT_OF_BAND_RUN_AGREEMENT) replacement = (candidate + observed) / 2; + } else if (declared) { + // A declared sample that moves the ruler another way (a provisional + // replacement, a run) or meets a marked record ends any held sample: + // it was measured against a ruler this one replaces. + this._declaredCandidate = null; + } + const alpha = 0.2; // slow EMA: one wild request shouldn't yank the ruler const ema = this._calibrationEma ?? current; // A provisional multiplier is a guess (see CALIBRATION_PRICING_EPOCH). This @@ -13966,7 +14035,7 @@ export class AutobiographicalStrategy implements ResettableStrategy { // would leave the next sample just as far out of band. const provisional = this._calibrationProvisional; this._calibrationProvisional = null; - const next = provisional || fromRun ? observed : ema + alpha * (observed - ema); + const next = replacement ?? (provisional || fromRun ? observed : ema + alpha * (observed - ema)); const clamped = Math.min(1.8, Math.max(0.6, next)); this._calibrationEma = clamped; // DEAD BAND. The multiplier is applied only once the EMA has moved more @@ -13985,6 +14054,10 @@ export class AutobiographicalStrategy implements ResettableStrategy { `[estimator-calibration] real/est=${ratio.toFixed(2)} confirms the provisional multiplier ` + `${current.toFixed(2)} ${counts}; stamped`, ); + if (declared) this._calibrationDeclared = true; + this.persistCalibration(); + } else if (markDeclared) { + this._calibrationDeclared = true; this.persistCalibration(); } return; @@ -13992,10 +14065,11 @@ export class AutobiographicalStrategy implements ResettableStrategy { this.diag.error( `[estimator-calibration] real/est=${ratio.toFixed(2)} ` + `multiplier ${current.toFixed(2)} -> ${clamped.toFixed(2)}` + - `${fromRun ? ` (${OUT_OF_BAND_RUN_LENGTH} consistent out-of-band samples: learned at the clamp)` : provisional ? ' (provisional: replaced, not stepped)' : ''} ` + + `${fromRun ? ` (${OUT_OF_BAND_RUN_LENGTH} consistent out-of-band samples: learned at the clamp)` : provisional ? ' (provisional: replaced, not stepped)' : replacement !== null ? ' (two samples with their overhead declared agree: replaced by their mean, not stepped)' : ''} ` + counts, ); this._calibration = clamped; + this._calibrationDeclared = declared; this.applyCalibration(); this.persistCalibration(); } @@ -14024,6 +14098,7 @@ export class AutobiographicalStrategy implements ResettableStrategy { at: Date.now(), pricing: AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH, ...(this._calibrationReset ? { reset: this._calibrationReset } : {}), + ...(this._calibrationDeclared ? { declaredOverhead: true } : {}), }); } catch { /* persistence is best-effort */ } } @@ -14167,10 +14242,19 @@ export class AutobiographicalStrategy implements ResettableStrategy { private _calibration = 1; /** The EMA itself; `_calibration` is its last applied value (see the dead band). */ private _calibrationEma: number | null = null; - /** The out-of-band samples rejected in a row that agree with each other, - * and the last one's implied multiplier (see reportRealInputTokens). In - * memory only: a restart counts afresh. */ - private _outOfBandRun: { observed: number; count: number } | null = null; + /** The out-of-band samples rejected in a row that agree with the first, + * and the first one's implied multiplier (see reportRealInputTokens). In + * memory only: a restart, or a branch's re-initialisation + * (clearBranchMirrors), counts afresh. */ + private _outOfBandRun: { first: number; count: number } | null = null; + /** The implied multiplier of the first declared sample on a record whose + * multiplier wasn't learned from declared samples, held for the next + * declared sample (see reportRealInputTokens). In memory only. */ + private _declaredCandidate: number | null = null; + /** Whether declared samples agree with the multiplier in force: set by + * one, or confirmed by two inside the dead band. Persisted as the record's + * declaredOverhead (see reportRealInputTokens). */ + private _calibrationDeclared = false; private _calibrationLoaded = false; /** Set when an older-epoch multiplier was discarded; persisted with the record. */ private _calibrationReset: CalibrationReset | null = null; @@ -14372,9 +14456,10 @@ export class AutobiographicalStrategy implements ResettableStrategy { this._calibration = 1; this._calibrationReset = null; this._calibrationProvisional = null; + this._calibrationDeclared = false; try { const saved = this.store?.getStateJson(this.calibrationStateId) as - { multiplier?: number; pricing?: number; reset?: CalibrationReset; provisional?: boolean } | null; + { multiplier?: number; pricing?: number; reset?: CalibrationReset; provisional?: boolean; declaredOverhead?: boolean } | null; if (saved && Number.isFinite(saved.multiplier)) { const current = AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH; const stamped = typeof saved.pricing === 'number' && Number.isFinite(saved.pricing); @@ -14389,6 +14474,7 @@ export class AutobiographicalStrategy implements ResettableStrategy { if (epoch === current) { this._calibration = Math.min(1.8, Math.max(0.6, saved.multiplier!)); this._calibrationReset = isCalibrationReset(saved.reset) ? saved.reset : null; + this._calibrationDeclared = saved.declaredOverhead === true; // A discard's 1 that no sample has confirmed yet was persisted as // provisional, and stays a guess across the restart: the first // sample still sets it, instead of stepping toward it. @@ -14409,6 +14495,7 @@ export class AutobiographicalStrategy implements ResettableStrategy { // still a guess. this._calibration = Math.min(1.8, Math.max(0.6, saved.multiplier!)); this._calibrationReset = isCalibrationReset(saved.reset) ? saved.reset : null; + this._calibrationDeclared = saved.declaredOverhead === true; if (saved.provisional === true) { this._calibrationProvisional = { fromEpoch: this._calibrationReset?.fromEpoch ?? null }; } @@ -14429,6 +14516,7 @@ export class AutobiographicalStrategy implements ResettableStrategy { pricing: current, ...(this._calibrationReset ? { reset: this._calibrationReset } : {}), ...(this._calibrationProvisional ? { provisional: true } : {}), + ...(this._calibrationDeclared ? { declaredOverhead: true } : {}), }); } catch { /* persistence is best-effort */ } } @@ -19286,11 +19374,13 @@ export class AutobiographicalStrategy implements ResettableStrategy { transitionBlocked: this.transitionBlocked, lastRenderStats: this._lastRenderStats, lastCompileEstimate: this._lastCompileEstimate, + lastCompileInjected: this._lastCompileInjected, calibrationArmed: this._calibrationArmed, calibrationLoaded: this._calibrationLoaded, calibration: this._calibration, calibrationReset: this._calibrationReset, calibrationProvisional: this._calibrationProvisional, + calibrationDeclared: this._calibrationDeclared, storeCalibration: store.getTokenCalibration?.(), storeView: this._storeView, storeOrder: this._storeOrder, @@ -19313,11 +19403,13 @@ export class AutobiographicalStrategy implements ResettableStrategy { this.transitionBlocked = saved.transitionBlocked; this._lastRenderStats = saved.lastRenderStats; this._lastCompileEstimate = saved.lastCompileEstimate; + this._lastCompileInjected = saved.lastCompileInjected; this._calibrationArmed = saved.calibrationArmed; this._calibrationLoaded = saved.calibrationLoaded; this._calibration = saved.calibration; this._calibrationReset = saved.calibrationReset; this._calibrationProvisional = saved.calibrationProvisional; + this._calibrationDeclared = saved.calibrationDeclared; if (saved.storeCalibration !== undefined) store.setTokenCalibration?.(saved.storeCalibration); this._storeView = saved.storeView; this._storeOrder = saved.storeOrder; diff --git a/src/types/strategy.ts b/src/types/strategy.ts index 45b643a8..813f42c3 100644 --- a/src/types/strategy.ts +++ b/src/types/strategy.ts @@ -370,6 +370,14 @@ export interface SelectOptions { multiplier: number; expiresAt: number; }; + /** The content of each injection the compile inserts among the selected + * entries after the select (`beforeUser` and `afterUser`), which the + * request carries and the select's window doesn't. A strategy whose + * calibration compares the provider's input total with the compile prices + * it there (AutobiographicalStrategy.reportRealInputTokens). + * ContextManager.compile fills it from its injections; `system` ones go + * into the caller's system prompt instead. */ + messageInjections?: ReadonlyArray; } /** @@ -667,10 +675,13 @@ export function isSummaryOverviewStrategy(s: ContextStrategy): s is SummaryOverv } /** - * What a request carried besides the compiled window, as the caller sent it - * (see `AutobiographicalStrategy.reportRealInputTokens`). The provider counts - * it in the input total and the compile estimate doesn't, so a calibration - * sample that leaves it out learns it as a per-token rate. + * What a request carried besides the compiled window and the compile's + * message-position injections (which the compile prices itself), as the + * caller sent it (see `AutobiographicalStrategy.reportRealInputTokens`). The + * provider counts it in the input total and the compile estimate doesn't, so + * a calibration sample that leaves it out learns it as a per-token rate. + * Passing it says this is everything the request sent besides those; a + * caller that can't see the request's head passes nothing. */ export interface RequestOverhead { /** The system prompt the request sent. */ diff --git a/test/calibration-overhead.test.ts b/test/calibration-overhead.test.ts index 9bd493ca..8f1d1bfd 100644 --- a/test/calibration-overhead.test.ts +++ b/test/calibration-overhead.test.ts @@ -16,6 +16,8 @@ import { mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { Membrane, MockAdapter, NativeFormatter, type ContentBlock, type ToolDefinition } from '@animalabs/membrane'; +import type { ContextInjection } from '../src/types/index.js'; +import type { RequestOverhead } from '../src/index.js'; import { ContextManager, AutobiographicalStrategy } from '../src/index.js'; import type { StoredMessage } from '../src/types/index.js'; @@ -82,8 +84,11 @@ describe('estimator calibration and request overhead (#323)', () => { const overhead = overheadPrice(cm, SYSTEM, tools); assert.ok(overhead > window * 0.4 && overhead < window * 0.7, `overhead ${overhead} vs window ${window}`); // Undeclared, this sample reads real/est ≈ 1.5: in band, so it would be - // learned as a per-token rate 50% above the truth. + // learned as a per-token rate 50% above the truth. Two samples, since a + // store's first declared sample is held until a second agrees. strategy.reportRealInputTokens(window + overhead, { system: SYSTEM, tools }); + await cm.compile(BUDGET); + strategy.reportRealInputTokens(internals._lastCompileEstimate + overhead, { system: SYSTEM, tools }); assert.equal(internals._calibration, 1); cm.close(); }); @@ -194,3 +199,419 @@ describe('estimator calibration and request overhead (#323)', () => { cm.close(); }); }); + +/** The store's price of some content, at its current multiplier. */ +function priceOf(cm: ContextManager, content: ContentBlock[]): number { + return (cm as unknown as { messageStore: { estimateTokens(m: StoredMessage): number } }).messageStore.estimateTokens({ content } as unknown as StoredMessage); +} + +/** An injection at a position, of about `tokens` estimated tokens. */ +function injectionOf(position: 'beforeUser' | 'afterUser' | 'system', tokens: number): ContextInjection { + return { namespace: 'probe', position, content: [{ type: 'text', text: 'Retrieved knowledge. '.repeat(Math.ceil(tokens / 4)) }] }; +} + +type DeclaredInternals = Internals & { _calibrationDeclared: boolean; _calibrationArmed: boolean }; +const epoch = AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH; + +async function seeded(name: string, record: Record) { + const first = await openManager(name); + await first.cm.compile(BUDGET); + first.internals.store!.setStateJson(first.internals.calibrationStateId, record); + first.cm.close(); + const opened = await openManager(name); + await opened.cm.compile(BUDGET); + return { ...opened, internals: opened.internals as DeclaredInternals }; +} + +/** Compile, then one sample at implied multiplier k, with system and tools declared or not. */ +async function sampleAt(cm: ContextManager, strategy: AutobiographicalStrategy, internals: Internals, k: number, declared: boolean): Promise { + await cm.compile(BUDGET); + const m = internals._calibration; + const tools = toolsOf(internals._lastCompileEstimate / m / 2); + const rawWindow = internals._lastCompileEstimate / m; + const rawOverhead = overheadPrice(cm, SYSTEM, tools) / m; + strategy.reportRealInputTokens(k * (rawWindow + (declared ? rawOverhead : 0)), declared ? { system: SYSTEM, tools } : undefined); +} + +const declaredOf = (internals: Internals) => + (internals.store!.getStateJson(internals.calibrationStateId) as { declaredOverhead?: boolean } | null)?.declaredOverhead; + +describe('estimator calibration: what the compile adds among the messages', () => { + /** Two compiles with the injection, each sampled with system, tools and the injection in the provider's count. */ + async function twoSamples(name: string, injection: (window: number) => ContextInjection) { + const { cm, strategy, internals } = await openManager(name); + await cm.compile(BUDGET); + const window = internals._lastCompileEstimate; + const inj = injection(window); + const tools = toolsOf(window / 2); + for (let i = 0; i < 2; i++) { + await cm.compile(BUDGET, [inj]); + assert.equal(internals._lastCompileEstimate, window, 'the window estimate leaves the injection out'); + strategy.reportRealInputTokens(window + overheadPrice(cm, SYSTEM, tools) + priceOf(cm, inj.content), { system: SYSTEM, tools }); + } + return { cm, internals, window, inj }; + } + + it('a message-position injection is priced into the sample: in band, it is not learned as a per-token rate', async () => { + const { cm, internals, window, inj } = await twoSamples('injected-in-band', (w) => injectionOf('afterUser', w / 2)); + // Premise: priced as the window and the declared head alone, these samples read about 1.33. + assert.ok(priceOf(cm, inj.content) > window * 0.4, `injection ${priceOf(cm, inj.content)} vs window ${window}`); + assert.equal(internals._calibration, 1); + assert.equal(declaredOf(internals), true, 'both samples were taken, and agreed'); + cm.close(); + }); + + it('a large injection on a small window is not learned at the clamp', async () => { + const { cm, strategy, internals } = await openManager('injected-out-of-band'); + await cm.compile(BUDGET); + const window = internals._lastCompileEstimate; + const injection = injectionOf('beforeUser', window * 3); + const tools = toolsOf(window / 2); + for (let i = 0; i < 4; i++) { + await cm.compile(BUDGET, [injection]); + strategy.reportRealInputTokens(internals._lastCompileEstimate + overheadPrice(cm, SYSTEM, tools) + priceOf(cm, injection.content), { system: SYSTEM, tools }); + } + assert.equal(internals._calibration, 1); + assert.equal(declaredOf(internals), true, 'the samples were taken in band'); + cm.close(); + }); + + it('a dry compile between the compile and its sample leaves the compile\'s injection priced', async () => { + const { cm, strategy, internals } = await openManager('injected-dry'); + await cm.compile(BUDGET); + const window = internals._lastCompileEstimate; + const injection = injectionOf('afterUser', window / 2); + const tools = toolsOf(window / 2); + for (let i = 0; i < 2; i++) { + await cm.compile(BUDGET, [injection]); + await cm.compile(BUDGET, undefined, { dryRun: true }); + strategy.reportRealInputTokens(window + overheadPrice(cm, SYSTEM, tools) + priceOf(cm, injection.content), { system: SYSTEM, tools }); + } + assert.equal(internals._calibration, 1); + assert.equal(declaredOf(internals), true, 'both samples were taken, and agreed'); + cm.close(); + }); + + it('a compile without injections prices none, after one with', async () => { + const { cm, strategy, internals } = await openManager('injected-then-none'); + await cm.compile(BUDGET); + const window = internals._lastCompileEstimate; + const injection = injectionOf('afterUser', window / 2); + const tools = toolsOf(window / 2); + await cm.compile(BUDGET, [injection]); + strategy.reportRealInputTokens(window + overheadPrice(cm, SYSTEM, tools) + priceOf(cm, injection.content), { system: SYSTEM, tools }); + await cm.compile(BUDGET); + strategy.reportRealInputTokens(internals._lastCompileEstimate + overheadPrice(cm, SYSTEM, tools), { system: SYSTEM, tools }); + assert.equal(internals._calibration, 1); + assert.equal(declaredOf(internals), true, 'both samples were taken, and agreed'); + cm.close(); + }); + + it('a system-position injection is priced once, as part of the declared system prompt', async () => { + const { cm, strategy, internals } = await openManager('injected-system'); + await cm.compile(BUDGET); + const window = internals._lastCompileEstimate; + const injection = injectionOf('system', window / 2); + const tools = toolsOf(window / 2); + for (let i = 0; i < 2; i++) { + const result = await cm.compile(BUDGET, [injection]); + // The caller appends systemInjections to its system prompt, and declares that. + const system = `${SYSTEM}\n${result.systemInjections.map((b) => (b as { text: string }).text).join('\n')}`; + strategy.reportRealInputTokens(internals._lastCompileEstimate + overheadPrice(cm, system, tools), { system, tools }); + } + assert.equal(internals._calibration, 1); + assert.equal(declaredOf(internals), true, 'both samples were taken, and agreed'); + cm.close(); + }); + + it('a dry compile with injections leaves the committing compile\'s count', async () => { + const { cm, strategy, internals } = await openManager('dry-with-injections'); + await cm.compile(BUDGET); + const window = internals._lastCompileEstimate; + const tools = toolsOf(window / 2); + for (let i = 0; i < 2; i++) { + await cm.compile(BUDGET); + await cm.compile(BUDGET, [injectionOf('afterUser', window / 2)], { dryRun: true }); + strategy.reportRealInputTokens(internals._lastCompileEstimate + overheadPrice(cm, SYSTEM, tools), { system: SYSTEM, tools }); + } + assert.equal(internals._calibration, 1); + assert.equal(declaredOf(internals), true, 'both samples were taken, and agreed'); + cm.close(); + }); + + it('several message-position injections are each priced', async () => { + const { cm, strategy, internals } = await openManager('two-injections'); + await cm.compile(BUDGET); + const window = internals._lastCompileEstimate; + const injections = [injectionOf('beforeUser', window / 4), injectionOf('afterUser', window / 4)]; + const tools = toolsOf(window / 2); + for (let i = 0; i < 2; i++) { + await cm.compile(BUDGET, injections); + const injected = injections.reduce((sum, inj) => sum + priceOf(cm, inj.content), 0); + strategy.reportRealInputTokens(internals._lastCompileEstimate + overheadPrice(cm, SYSTEM, tools) + injected, { system: SYSTEM, tools }); + } + assert.equal(internals._calibration, 1); + assert.equal(declaredOf(internals), true, 'both samples were taken, and agreed'); + cm.close(); + }); + + it('an injection is priced at the multiplier in force, as the window is', async () => { + const { cm, strategy, internals } = await seeded('injected-calibrated', { multiplier: 1.5, at: 1, pricing: epoch, declaredOverhead: true }); + assert.equal(internals._calibration, 1.5); + const tools = toolsOf(internals._lastCompileEstimate / 2); + const injection = injectionOf('afterUser', internals._lastCompileEstimate / 2); + await cm.compile(BUDGET, [injection]); + // A provider at k = 1.5: the window, the head and the injection, each at the store's price. + strategy.reportRealInputTokens(internals._lastCompileEstimate + overheadPrice(cm, SYSTEM, tools) + priceOf(cm, injection.content), { system: SYSTEM, tools }); + assert.equal(internals._calibration, 1.5); + assert.equal((internals as unknown as { _calibrationEma: number | null })._calibrationEma, 1.5, 'the sample was taken'); + cm.close(); + }); + + it('an undeclared sample counts the injections too', async () => { + const { cm, strategy, internals } = await openManager('injected-undeclared'); + await cm.compile(BUDGET); + const window = internals._lastCompileEstimate; + const injection = injectionOf('afterUser', window / 2); + await cm.compile(BUDGET, [injection]); + strategy.reportRealInputTokens(window + priceOf(cm, injection.content)); + assert.equal(internals._calibration, 1); + assert.equal((internals as unknown as { _calibrationEma: number | null })._calibrationEma, 1, 'the sample was taken'); + cm.close(); + }); + + it('a system prompt alone is priced into the sample', async () => { + const { cm, strategy, internals } = await openManager('system-only'); + await cm.compile(BUDGET); + const window = internals._lastCompileEstimate; + const system = `You are a resident. ${'Keep to your purpose. '.repeat(Math.ceil(window / 14))}`; + // Premise: left out, the samples read 1.25 to 1.7, inside the band, where they'd be learned. + const share = overheadPrice(cm, system, []) / window; + assert.ok(share > 0.25 && share < 0.7, `the system prompt is ${share} of the window`); + for (let i = 0; i < 2; i++) { + await cm.compile(BUDGET); + strategy.reportRealInputTokens(internals._lastCompileEstimate + overheadPrice(cm, system, []), { system }); + } + assert.equal(internals._calibration, 1); + assert.equal(declaredOf(internals), true, 'both samples were taken, and agreed'); + cm.close(); + }); +}); + +describe('estimator calibration: consistent out-of-band runs', () => { + /** Compile, then one declared sample at `ratio` of the compile's estimate. */ + async function oob(cm: ContextManager, strategy: AutobiographicalStrategy, internals: Internals, ratio: number, declared = true) { + await cm.compile(BUDGET); + const nothingBesides: RequestOverhead = {}; // the package's type, from its root + strategy.reportRealInputTokens(internals._lastCompileEstimate * ratio, declared ? nothingBesides : undefined); + } + + it('an undeclared out-of-band sample ends a declared run', async () => { + const { cm, strategy, internals } = await openManager('undeclared-ends-run'); + await oob(cm, strategy, internals, 2.1); + await oob(cm, strategy, internals, 2.1); + await oob(cm, strategy, internals, 2.1, false); + await oob(cm, strategy, internals, 2.1); + assert.equal(internals._calibration, 1); + await oob(cm, strategy, internals, 2.1); + await oob(cm, strategy, internals, 2.1); + assert.equal(internals._calibration, 1.8, 'a fresh run of three is learned'); + cm.close(); + }); + + it('a run agrees with its first sample: within 10% it is learned, and a drift past 10% of the first is not', async () => { + const drift = await openManager('run-drift'); + for (const ratio of [2.0, 2.18, 2.38]) await oob(drift.cm, drift.strategy, drift.internals, ratio); + assert.equal(drift.internals._calibration, 1, 'each within 10% of the one before, the third 19% past the first'); + drift.cm.close(); + const near = await openManager('run-near'); + for (const ratio of [2.0, 2.18, 2.08]) await oob(near.cm, near.strategy, near.internals, ratio); + assert.equal(near.internals._calibration, 1.8, 'each within 10% of the first'); + near.cm.close(); + }); + + it('a consistent run below the band is learned at the floor', async () => { + const { cm, strategy, internals } = await openManager('run-low'); + for (let i = 0; i < 3; i++) await oob(cm, strategy, internals, 0.3); + assert.equal(internals._calibration, 0.6); + cm.close(); + }); + + it('one compile counts once toward a run, however many reports follow it', async () => { + const { cm, strategy, internals } = await openManager('run-one-arm'); + await cm.compile(BUDGET); + for (let i = 0; i < 3; i++) strategy.reportRealInputTokens(internals._lastCompileEstimate * 2.1, {}); + assert.equal(internals._calibration, 1, 'three reports of one compile'); + await oob(cm, strategy, internals, 2.1); + assert.equal(internals._calibration, 1, 'the second compile is the run\'s second sample'); + await oob(cm, strategy, internals, 2.1); + assert.equal(internals._calibration, 1.8, 'the third compile is its third'); + cm.close(); + }); + + it('a branch switch starts a run afresh', async () => { + const { cm, strategy, internals } = await openManager('run-branch'); + await oob(cm, strategy, internals, 2.1); + await oob(cm, strategy, internals, 2.1); + await cm.fork('other'); + await oob(cm, strategy, internals, 2.1); + assert.equal(internals._calibration, 1); + assert.equal((internals as unknown as { _outOfBandRun: { count: number } | null })._outOfBandRun?.count, 1, 'the sample on the new branch starts a run'); + cm.close(); + }); +}); + +describe('estimator calibration: a store that learned the overhead as a rate', () => { + it('two declared samples that agree replace a multiplier learned without them, by their mean', async () => { + const { cm, strategy, internals } = await seeded('learned-undeclared', { multiplier: 1.8, at: 1, pricing: epoch }); + assert.equal(internals._calibration, 1.8); + await sampleAt(cm, strategy, internals, 1, true); + assert.equal(internals._calibration, 1.8, 'the first is held'); + assert.equal(declaredOf(internals), undefined); + await sampleAt(cm, strategy, internals, 1.04, true); + assert.ok(Math.abs(internals._calibration - 1.02) <= 0.01, `replaced by the mean, got ${internals._calibration}`); + assert.equal(declaredOf(internals), true, 'and the record says its samples declare their overhead'); + cm.close(); + }); + + it('two declared samples that disagree step by the EMA, and mark the record declared', async () => { + const { cm, strategy, internals } = await seeded('learned-disagree', { multiplier: 1.8, at: 1, pricing: epoch }); + await sampleAt(cm, strategy, internals, 1, true); + await sampleAt(cm, strategy, internals, 1.5, true); + assert.ok(Math.abs(internals._calibration - 1.74) <= 0.01, `stepped toward the second, got ${internals._calibration}`); + assert.equal(declaredOf(internals), true); + await sampleAt(cm, strategy, internals, 1, true); + assert.ok(Math.abs(internals._calibration - 1.592) <= 0.01, `then steps by the EMA, got ${internals._calibration}`); + cm.close(); + }); + + it('a branch switch between the two loses the held sample', async () => { + const { cm, strategy, internals } = await seeded('held-branch', { multiplier: 1.8, at: 1, pricing: epoch }); + await sampleAt(cm, strategy, internals, 1, true); + await cm.fork('other'); + await cm.compile(BUDGET); + assert.equal(internals._calibration, 1.8, 'the fork has the record'); + await sampleAt(cm, strategy, internals, 1, true); + assert.equal(internals._calibration, 1.8, 'held, not paired with the other branch\'s sample'); + assert.notEqual((internals as unknown as { _declaredCandidate: number | null })._declaredCandidate, null, 'and held afresh'); + cm.close(); + }); + + it('a consistent run between the two ends the held sample', async () => { + const { cm, strategy, internals } = await seeded('held-run', { multiplier: 1, at: 1, pricing: epoch }); + await sampleAt(cm, strategy, internals, 1, true); + for (let i = 0; i < 3; i++) { + await cm.compile(BUDGET); + strategy.reportRealInputTokens(internals._lastCompileEstimate * 2.1, {}); + } + assert.equal(internals._calibration, 1.8, 'the run is learned'); + await sampleAt(cm, strategy, internals, 1.5, false); + assert.ok(Math.abs(internals._calibration - 1.74) <= 0.01, `an undeclared sample steps and unmarks, got ${internals._calibration}`); + await sampleAt(cm, strategy, internals, 1, true); + assert.ok(Math.abs(internals._calibration - 1.74) <= 0.01, `held, not paired with the sample from before the run, got ${internals._calibration}`); + cm.close(); + }); + + it('a restart between the two loses the held sample: it starts over, and the record is not marked', async () => { + const first = await seeded('held-restart', { multiplier: 1.8, at: 1, pricing: epoch }); + await sampleAt(first.cm, first.strategy, first.internals, 1, true); + first.cm.close(); + const { cm, strategy, internals } = await openManager('held-restart'); + await cm.compile(BUDGET); + const typed = internals as DeclaredInternals; + assert.deepEqual([internals._calibration, typed._calibrationDeclared, declaredOf(internals)], [1.8, false, undefined]); + await sampleAt(cm, strategy, internals, 1, true); + assert.equal(internals._calibration, 1.8, 'held again'); + await sampleAt(cm, strategy, internals, 1, true); + assert.ok(Math.abs(internals._calibration - 1) <= 0.02, `then replaced, got ${internals._calibration}`); + cm.close(); + }); + + it('a multiplier learned from declared samples steps by the EMA', async () => { + const { cm, strategy, internals } = await seeded('learned-declared', { multiplier: 1.8, at: 1, pricing: epoch, declaredOverhead: true }); + await sampleAt(cm, strategy, internals, 1, true); + assert.ok(Math.abs(internals._calibration - 1.64) <= 0.01, `stepped, got ${internals._calibration}`); + cm.close(); + }); + + it('after the replacement, a restart steps by the EMA', async () => { + const first = await seeded('declared-restart', { multiplier: 1.8, at: 1, pricing: epoch }); + await sampleAt(first.cm, first.strategy, first.internals, 1, true); + await sampleAt(first.cm, first.strategy, first.internals, 1, true); + first.cm.close(); + const { cm, strategy, internals } = await openManager('declared-restart'); + await cm.compile(BUDGET); + await sampleAt(cm, strategy, internals, 1.3, true); + assert.ok(Math.abs(internals._calibration - 1.06) <= 0.01, `stepped, got ${internals._calibration}`); + cm.close(); + }); + + it('an undeclared sample that moves the multiplier unmarks the record, and the next two declared replace it', async () => { + const { cm, strategy, internals } = await seeded('undeclared-after-declared', { multiplier: 1, at: 1, pricing: epoch, declaredOverhead: true }); + await sampleAt(cm, strategy, internals, 1.5, false); + assert.ok(internals._calibration > 1.05, `the undeclared sample stepped, got ${internals._calibration}`); + assert.equal(declaredOf(internals), undefined); + await sampleAt(cm, strategy, internals, 1, true); + await sampleAt(cm, strategy, internals, 1, true); + assert.ok(Math.abs(internals._calibration - 1) <= 0.02, `replaced, got ${internals._calibration}`); + cm.close(); + }); + + it('two declared samples that confirm the multiplier mark the record declared', async () => { + const { cm, strategy, internals } = await seeded('confirmed-declared', { multiplier: 1, at: 1, pricing: epoch }); + await sampleAt(cm, strategy, internals, 1, true); + await sampleAt(cm, strategy, internals, 1, true); + assert.equal(internals._calibration, 1); + assert.equal(declaredOf(internals), true); + await sampleAt(cm, strategy, internals, 1.5, true); + assert.ok(Math.abs(internals._calibration - 1.1) <= 0.01, `stepped, got ${internals._calibration}`); + cm.close(); + }); + + it('a provisional multiplier is replaced by one declared sample, and marked declared', async () => { + const { cm, strategy, internals } = await seeded('provisional-declared', { multiplier: 1, at: 1, pricing: epoch, provisional: true }); + await sampleAt(cm, strategy, internals, 1, true); + const saved = internals.store!.getStateJson(internals.calibrationStateId) as { provisional?: boolean; declaredOverhead?: boolean }; + assert.deepEqual([saved.provisional, saved.declaredOverhead], [undefined, true]); + cm.close(); + }); + + it('a dry compile after the load keeps a record\'s declared overhead', async () => { + const { cm, strategy, internals } = await seeded('declared-dry', { multiplier: 1.8, at: 1, pricing: epoch, declaredOverhead: true }); + await cm.compile(BUDGET, undefined, { dryRun: true }); + assert.equal(internals._calibrationDeclared, true); + await sampleAt(cm, strategy, internals, 1, true); + assert.ok(Math.abs(internals._calibration - 1.64) <= 0.01, `stepped, not held, got ${internals._calibration}`); + cm.close(); + }); + + it('an undeclared sample inside the dead band leaves a record marked, and a rejected declared one leaves it unmarked', async () => { + const marked = await seeded('deadband-undeclared', { multiplier: 1, at: 1, pricing: epoch, declaredOverhead: true }); + await sampleAt(marked.cm, marked.strategy, marked.internals, 1.01, false); + assert.deepEqual([marked.internals._calibration, marked.internals._calibrationDeclared, declaredOf(marked.internals)], [1, true, true]); + marked.cm.close(); + const unmarked = await seeded('rejected-declared', { multiplier: 1, at: 1, pricing: epoch }); + await unmarked.cm.compile(BUDGET); + unmarked.strategy.reportRealInputTokens(unmarked.internals._lastCompileEstimate * 2.1, {}); + assert.deepEqual([unmarked.internals._calibration, unmarked.internals._calibrationDeclared, declaredOf(unmarked.internals)], [1, false, undefined]); + unmarked.cm.close(); + }); + + it('a null overhead from a JS caller counts as undeclared', async () => { + const { cm, strategy, internals } = await seeded('null-overhead', { multiplier: 1, at: 1, pricing: epoch }); + await cm.compile(BUDGET); + strategy.reportRealInputTokens(internals._lastCompileEstimate * 1.5, null as unknown as RequestOverhead); + assert.ok(Math.abs(internals._calibration - 1.1) <= 0.01, `stepped as undeclared, not held, got ${internals._calibration}`); + assert.deepEqual([internals._calibrationDeclared, declaredOf(internals)], [false, undefined]); + cm.close(); + }); + + it('a re-stamp keeps a record\'s declared overhead', async () => { + const { cm, internals } = await seeded('restamp-declared', { multiplier: 1.2, at: 1, pricing: epoch - 1, declaredOverhead: true }); + const saved = internals.store!.getStateJson(internals.calibrationStateId) as { pricing?: number; declaredOverhead?: boolean }; + assert.equal(saved.pricing, epoch, 're-stamped'); + assert.equal(saved.declaredOverhead, true); + assert.equal(internals._calibrationDeclared, true); + cm.close(); + }); +}); From 7df5bb6e50af1de63a36bc3e30653e45d59a818c Mon Sep 17 00:00:00 2001 From: Esther-2584 Date: Fri, 23 Oct 2026 09:10:01 -0400 Subject: [PATCH 2/8] fix(calibration): a store with no record starts from a provisional 1 (#328) A store with no calibration record loaded 1 as learned, so the store with the least evidence took the slowest path: nine applied EMA steps to reach a true 1.4, each a reprice of every message and summary. A discard already restarts from a provisional 1, which the first sample the band accepts sets. A store with no record, or a record with no finite multiplier, now loads the same, with fromEpoch null, and writes nothing: with no record the next load decides the same, and the first sample's write gives the stamp. So its first in-band sample sets it, declared or not; a declared one isn't held, as for any provisional multiplier. A record from a newer epoch and a read of the record that throws still load a learned 1: each may hold a multiplier worth keeping, which a provisional 1's first sample would write over even when it only confirms the 1. Render stats report calibration.provisional: true for such a store until its first sample. Tests: a store with no record loads a provisional 1, a rejected sample leaves it, the first in-band sample sets and stamps it, and the EMA steps after; a first sample inside the dead band confirms and stamps it; a restart before any sample leaves it provisional; a record with no usable multiplier loads as none; a read that throws loads a learned 1; a first declared sample sets it with nothing held; a consistent run on a learned record without the flag is learned at the clamp, not held. Two tests that meant a learned multiplier now seed one, and the no-history render-stats test expects provisional. The new assertions that compare with null are strict, since the legacy deepEqual these files use passes an object against null. Co-Authored-By: Claude Opus 5.5 --- ...328-fresh-calibration-provisional.fixed.md | 7 ++ src/strategies/autobiographical.ts | 19 +++- src/types/strategy.ts | 3 +- test/calibration-band.test.ts | 105 +++++++++++++++++- test/calibration-overhead.test.ts | 40 +++++-- 5 files changed, 157 insertions(+), 17 deletions(-) create mode 100644 changelog.d/328-fresh-calibration-provisional.fixed.md diff --git a/changelog.d/328-fresh-calibration-provisional.fixed.md b/changelog.d/328-fresh-calibration-provisional.fixed.md new file mode 100644 index 00000000..a1be6553 --- /dev/null +++ b/changelog.d/328-fresh-calibration-provisional.fixed.md @@ -0,0 +1,7 @@ +- A store with no calibration record starts from a provisional 1, as a + discard's restart does (#328): its first sample inside the calibration + band sets the multiplier, declared or not, instead of stepping a fifth of + the way per sample (nine applied changes, each a reprice of every message + and summary, to reach a true 1.4). Until that sample, render stats report + `calibration.provisional: true`. A record from a newer pricing epoch, or a + read of the record that throws, still loads a learned 1. diff --git a/src/strategies/autobiographical.ts b/src/strategies/autobiographical.ts index 9472770a..b677f878 100644 --- a/src/strategies/autobiographical.ts +++ b/src/strategies/autobiographical.ts @@ -14261,7 +14261,7 @@ export class AutobiographicalStrategy implements ResettableStrategy { /** Set while the multiplier is provisional (see CALIBRATION_PRICING_EPOCH): * the first sample the band accepts replaces it outright, and one above 1 * that puts a compile over budget is dropped to 1. `fromEpoch` is the - * record's it came from, null for one without a stamp. */ + * record's it came from, null for one without a stamp or for no record. */ private _calibrationProvisional: { fromEpoch: number | null } | null = null; protected applyCalibration(): void { @@ -14320,7 +14320,9 @@ export class AutobiographicalStrategy implements ResettableStrategy { * is persisted as such (`provisional: true`), so a restart before that * sample doesn't turn the guess into a learned value, and a later epoch's * load re-stamps it, still provisional, rather than discarding it again - * over the reset it carries. A dry run under a + * over the reset it carries. A store with no record starts from a + * provisional 1 too, and writes nothing until its first sample, since no + * record already loads that way (#328). A dry run under a * provisional multiplier above 1 takes the same fallback at the picker's * refusal, which it otherwise only reports. * @@ -14574,6 +14576,19 @@ export class AutobiographicalStrategy implements ResettableStrategy { // re-stamping it as the discard warning advises would destroy it. // This binary's learner still writes its own record over it on its // first applied sample (see CALIBRATION_PRICING_EPOCH). + } else { + // No record, or none with a multiplier to use: no sample has ever + // set this store's 1, so it's a guess, as a discard's restart is, + // and the first sample the band accepts sets it (#328). Learned, it + // would take the slowest path for the store with the least + // evidence, nine applied EMA steps to reach a true 1.4, each a + // reprice of every message and summary. Nothing is written: with no + // record, the next load decides the same, and the first sample's + // write gives the stamp. A record from a newer epoch (above) and a + // read that throws (below) aren't this case: each may hold a + // multiplier worth keeping, which a provisional 1's first sample + // would write over even when it only confirms the 1. + this._calibrationProvisional = { fromEpoch: null }; } } catch { /* absent slot is fine */ } } diff --git a/src/types/strategy.ts b/src/types/strategy.ts index 813f42c3..fbc2dfd1 100644 --- a/src/types/strategy.ts +++ b/src/types/strategy.ts @@ -746,7 +746,8 @@ export interface RenderStats { * provider sample the calibration's band accepts replaces outright, rather * than stepping toward it (see * `AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH`): one kept from a - * record without a stamp, or the 1 a discard restarts from. That 1 stays + * record without a stamp, the 1 a discard restarts from, or the 1 a store + * with no calibration record starts from. A discard's 1 stays * provisional through a later epoch's load, which re-stamps it whatever * that epoch repriced. * `reset` is present once a multiplier from an older or unknown epoch has diff --git a/test/calibration-band.test.ts b/test/calibration-band.test.ts index 0c108c8c..fcc391e0 100644 --- a/test/calibration-band.test.ts +++ b/test/calibration-band.test.ts @@ -95,11 +95,16 @@ async function openManager(name: string, opts: { signedThinking?: boolean; stamp describe('estimator calibration band', () => { it('a multiplier pinned at the ceiling comes back down once the raw estimate is honest', async () => { + // A learned 1.8: a store with no record would be provisional, and set + // outright by the first sample (#328). + const first = await openManager('pinned'); + await first.cm.compile(BUDGET); + first.internals.store!.setStateJson(first.internals.calibrationStateId, + { multiplier: 1.8, at: 1, pricing: AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH }); + first.cm.close(); const { cm, strategy, internals } = await openManager('pinned'); - await cm.compile(BUDGET); - internals._calibration = 1.8; - internals.applyCalibration(); await cm.compile(BUDGET); // arms one sample; estimate is in calibrated units (raw × 1.8) + assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1.8, null], 'learned'); const est = internals._lastCompileEstimate; assert.ok(est > 0); // The provider bills exactly the raw estimate: real/est = 1/1.8 ≈ 0.56. @@ -120,8 +125,15 @@ describe('estimator calibration band', () => { }); it('an in-band ratio is learned from as before', async () => { + // A learned 1: a store with no record would be set outright (#328). + const first = await openManager('inband'); + await first.cm.compile(BUDGET); + first.internals.store!.setStateJson(first.internals.calibrationStateId, + { multiplier: 1, at: 1, pricing: AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH }); + first.cm.close(); const { cm, strategy, internals } = await openManager('inband'); await cm.compile(BUDGET); + assert.equal(internals._calibrationProvisional, null, 'learned'); const est = internals._lastCompileEstimate; strategy.reportRealInputTokens(est * 1.5); assert.ok(internals._calibration > 1.0 && internals._calibration < 1.5, `got ${internals._calibration}`); @@ -401,10 +413,93 @@ describe('estimator calibration band', () => { reopened.cm.close(); }); - it('a store with no calibration history reports its multiplier and no reset', async () => { + it('a store with no calibration history reports a provisional 1 and no reset', async () => { const { cm } = await openManager('no-history'); await cm.compile(BUDGET); - assert.deepEqual(cm.getRenderStats()!.calibration, { multiplier: 1, pricingEpoch: AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH }); + assert.deepStrictEqual(cm.getRenderStats()!.calibration, + { multiplier: 1, pricingEpoch: AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH, provisional: true }); + cm.close(); + }); + + it('a store with no calibration record loads a provisional 1: the first sample the band accepts sets it, and stamps the record', async () => { + // The store with the least evidence: learned, its 1 would step a fifth of + // the way per sample, nine applied changes to reach a true 1.4 (#328). + const epoch = AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH; + const { cm, strategy, internals } = await openManager('fresh-provisional'); + await cm.compile(BUDGET); + assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1, { fromEpoch: null }]); + assert.equal(internals.store!.getStateJson(internals.calibrationStateId), null, 'the load writes nothing'); + // A sample the band rejects is no evidence: the 1 stays provisional. + strategy.reportRealInputTokens(internals._lastCompileEstimate * 3); + assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1, { fromEpoch: null }]); + await cm.compile(BUDGET); + strategy.reportRealInputTokens(internals._lastCompileEstimate * 1.4); + assert.ok(Math.abs(internals._calibration - 1.4) < 1e-9, `set by the sample, got ${internals._calibration}`); + assert.equal(internals._calibrationProvisional, null); + const saved = internals.store!.getStateJson(internals.calibrationStateId) as + { multiplier?: number; pricing?: number; provisional?: boolean; reset?: unknown }; + assert.deepStrictEqual([saved.multiplier, saved.pricing, saved.provisional, saved.reset], [1.4, epoch, undefined, undefined]); + // From here the EMA steps as it always did. + await cm.compile(BUDGET); + strategy.reportRealInputTokens(internals._lastCompileEstimate * 1.2); // implies 1.68 + assert.ok(internals._calibration > 1.4 && internals._calibration < 1.6, `stepped, not replaced: ${internals._calibration}`); + cm.close(); + }); + + it("a store with no calibration record whose first sample falls inside the dead band confirms its 1, and stamps the record", async () => { + const { cm, strategy, internals } = await openManager('fresh-confirm'); + await cm.compile(BUDGET); + strategy.reportRealInputTokens(internals._lastCompileEstimate * 1.01); + assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1, null]); + const saved = internals.store!.getStateJson(internals.calibrationStateId) as { multiplier?: number; pricing?: number; provisional?: boolean }; + assert.deepStrictEqual([saved.multiplier, saved.pricing, saved.provisional], [1, AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH, undefined]); + cm.close(); + }); + + it('a store with no calibration record is still provisional after a restart before its first sample', async () => { + const first = await openManager('fresh-restart'); + await first.cm.compile(BUDGET); + first.cm.close(); + const { cm, strategy, internals } = await openManager('fresh-restart'); + await cm.compile(BUDGET); + assert.deepStrictEqual(internals._calibrationProvisional, { fromEpoch: null }); + assert.equal(internals.store!.getStateJson(internals.calibrationStateId), null); + strategy.reportRealInputTokens(internals._lastCompileEstimate * 0.8); + assert.ok(Math.abs(internals._calibration - 0.8) < 1e-9, `set by the sample, got ${internals._calibration}`); + cm.close(); + }); + + it('a record with no multiplier to use loads as no record: a provisional 1', async () => { + const epoch = AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH; + for (const [i, record] of ([{ at: 1, pricing: epoch }, { multiplier: '1.4', at: 1, pricing: epoch }, { multiplier: null, at: 1, pricing: epoch }, 7] as unknown[]).entries()) { + const name = `fresh-unusable-${i}`; + const first = await openManager(name); + await first.cm.compile(BUDGET); + first.internals.store!.setStateJson(first.internals.calibrationStateId, record); + first.cm.close(); + const { cm, internals } = await openManager(name); + await cm.compile(BUDGET); + assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1, { fromEpoch: null }], JSON.stringify(record)); + cm.close(); + } + }); + + it('a calibration read that throws loads a learned 1, as before: the record it could not read may hold one worth keeping', async () => { + const { cm, internals } = await openManager('fresh-unreadable'); + const store = internals.store as unknown as { getStateJson(id: string): unknown }; + const read = store.getStateJson; + let refused = 0; + store.getStateJson = (id: string) => { + if (id === internals.calibrationStateId) { refused++; throw new Error('unreadable'); } + return read.call(store, id); + }; + try { + await cm.compile(BUDGET); + } finally { + store.getStateJson = read; + } + assert.ok(refused > 0, 'premise: the load read the record, and the read threw'); + assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1, null]); cm.close(); }); diff --git a/test/calibration-overhead.test.ts b/test/calibration-overhead.test.ts index 8f1d1bfd..c80bdb45 100644 --- a/test/calibration-overhead.test.ts +++ b/test/calibration-overhead.test.ts @@ -84,8 +84,8 @@ describe('estimator calibration and request overhead (#323)', () => { const overhead = overheadPrice(cm, SYSTEM, tools); assert.ok(overhead > window * 0.4 && overhead < window * 0.7, `overhead ${overhead} vs window ${window}`); // Undeclared, this sample reads real/est ≈ 1.5: in band, so it would be - // learned as a per-token rate 50% above the truth. Two samples, since a - // store's first declared sample is held until a second agrees. + // learned as a per-token rate 50% above the truth. The first sample sets + // a store with no record; the second steps it. strategy.reportRealInputTokens(window + overhead, { system: SYSTEM, tools }); await cm.compile(BUDGET); strategy.reportRealInputTokens(internals._lastCompileEstimate + overhead, { system: SYSTEM, tools }); @@ -257,7 +257,7 @@ describe('estimator calibration: what the compile adds among the messages', () = // Premise: priced as the window and the declared head alone, these samples read about 1.33. assert.ok(priceOf(cm, inj.content) > window * 0.4, `injection ${priceOf(cm, inj.content)} vs window ${window}`); assert.equal(internals._calibration, 1); - assert.equal(declaredOf(internals), true, 'both samples were taken, and agreed'); + assert.equal(declaredOf(internals), true, 'the samples were taken'); cm.close(); }); @@ -288,7 +288,7 @@ describe('estimator calibration: what the compile adds among the messages', () = strategy.reportRealInputTokens(window + overheadPrice(cm, SYSTEM, tools) + priceOf(cm, injection.content), { system: SYSTEM, tools }); } assert.equal(internals._calibration, 1); - assert.equal(declaredOf(internals), true, 'both samples were taken, and agreed'); + assert.equal(declaredOf(internals), true, 'the samples were taken'); cm.close(); }); @@ -303,7 +303,7 @@ describe('estimator calibration: what the compile adds among the messages', () = await cm.compile(BUDGET); strategy.reportRealInputTokens(internals._lastCompileEstimate + overheadPrice(cm, SYSTEM, tools), { system: SYSTEM, tools }); assert.equal(internals._calibration, 1); - assert.equal(declaredOf(internals), true, 'both samples were taken, and agreed'); + assert.equal(declaredOf(internals), true, 'the samples were taken'); cm.close(); }); @@ -320,7 +320,7 @@ describe('estimator calibration: what the compile adds among the messages', () = strategy.reportRealInputTokens(internals._lastCompileEstimate + overheadPrice(cm, system, tools), { system, tools }); } assert.equal(internals._calibration, 1); - assert.equal(declaredOf(internals), true, 'both samples were taken, and agreed'); + assert.equal(declaredOf(internals), true, 'the samples were taken'); cm.close(); }); @@ -335,7 +335,7 @@ describe('estimator calibration: what the compile adds among the messages', () = strategy.reportRealInputTokens(internals._lastCompileEstimate + overheadPrice(cm, SYSTEM, tools), { system: SYSTEM, tools }); } assert.equal(internals._calibration, 1); - assert.equal(declaredOf(internals), true, 'both samples were taken, and agreed'); + assert.equal(declaredOf(internals), true, 'the samples were taken'); cm.close(); }); @@ -351,7 +351,7 @@ describe('estimator calibration: what the compile adds among the messages', () = strategy.reportRealInputTokens(internals._lastCompileEstimate + overheadPrice(cm, SYSTEM, tools) + injected, { system: SYSTEM, tools }); } assert.equal(internals._calibration, 1); - assert.equal(declaredOf(internals), true, 'both samples were taken, and agreed'); + assert.equal(declaredOf(internals), true, 'the samples were taken'); cm.close(); }); @@ -393,7 +393,7 @@ describe('estimator calibration: what the compile adds among the messages', () = strategy.reportRealInputTokens(internals._lastCompileEstimate + overheadPrice(cm, system, []), { system }); } assert.equal(internals._calibration, 1); - assert.equal(declaredOf(internals), true, 'both samples were taken, and agreed'); + assert.equal(declaredOf(internals), true, 'the samples were taken'); cm.close(); }); }); @@ -406,6 +406,17 @@ describe('estimator calibration: consistent out-of-band runs', () => { strategy.reportRealInputTokens(internals._lastCompileEstimate * ratio, declared ? nothingBesides : undefined); } + it('a consistent run on a learned record without the flag is learned at the clamp, not held', async () => { + // The upgrade case: a learned multiplier no declared sample has confirmed. + // A store with no record is provisional, which skips the hold anyway. + const { cm, strategy, internals } = await seeded('run-on-learned', { multiplier: 1, at: 1, pricing: epoch }); + assert.equal(internals._calibrationProvisional, null, 'learned'); + for (let i = 0; i < 3; i++) await oob(cm, strategy, internals, 2.1); + assert.equal(internals._calibration, 1.8); + assert.equal(declaredOf(internals), true); + cm.close(); + }); + it('an undeclared out-of-band sample ends a declared run', async () => { const { cm, strategy, internals } = await openManager('undeclared-ends-run'); await oob(cm, strategy, internals, 2.1); @@ -568,6 +579,17 @@ describe('estimator calibration: a store that learned the overhead as a rate', ( cm.close(); }); + it('a store with no record is set by its first declared sample, with nothing held, and marked declared', async () => { + const { cm, strategy, internals } = await openManager('fresh-declared'); + await cm.compile(BUDGET); + assert.deepEqual(internals._calibrationProvisional, { fromEpoch: null }); + await sampleAt(cm, strategy, internals as DeclaredInternals, 1.4, true); + assert.ok(Math.abs(internals._calibration - 1.4) <= 0.01, `set by one sample, got ${internals._calibration}`); + assert.equal((internals as unknown as { _declaredCandidate: number | null })._declaredCandidate, null, 'nothing held'); + assert.equal(declaredOf(internals), true); + cm.close(); + }); + it('a provisional multiplier is replaced by one declared sample, and marked declared', async () => { const { cm, strategy, internals } = await seeded('provisional-declared', { multiplier: 1, at: 1, pricing: epoch, provisional: true }); await sampleAt(cm, strategy, internals, 1, true); From e93803d74b65ac421839b968117e8542180d794d Mon Sep 17 00:00:00 2001 From: Esther-2584 Date: Fri, 23 Oct 2026 09:10:24 -0400 Subject: [PATCH 3/8] docs(calibration): give a consistent run's actual reason for replacing outright The comment said an EMA step from 1.00 toward a ratio past the band would leave the next sample just as far out of band. It wouldn't: with the overhead declared, one step toward an implied 2.1 gives 1.22, and the next sample's ratio, 1.72, is inside the band (#324's review, R1-5). The run replaces outright because its samples already agree on the multiplier, and stepping toward it would take several more applied changes, each a reprice of every message and summary. Co-Authored-By: Claude Opus 5.5 --- src/strategies/autobiographical.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/strategies/autobiographical.ts b/src/strategies/autobiographical.ts index b677f878..f5947840 100644 --- a/src/strategies/autobiographical.ts +++ b/src/strategies/autobiographical.ts @@ -14031,8 +14031,9 @@ export class AutobiographicalStrategy implements ResettableStrategy { // A provisional multiplier is a guess (see CALIBRATION_PRICING_EPOCH). This // sample is the first evidence under this release's pricing, so it replaces // the guess outright; the EMA steps from here on. A consistent run replaces - // it outright too: an EMA step from 1.00 toward a ratio past the band - // would leave the next sample just as far out of band. + // it outright too: its samples agree on the multiplier, and stepping toward + // it would take several more applied changes, each a reprice of every + // message and summary. const provisional = this._calibrationProvisional; this._calibrationProvisional = null; const next = replacement ?? (provisional || fromRun ? observed : ema + alpha * (observed - ema)); From 0215b1bec118ee4e054c3a66bbbdca95caa3513b Mon Sep 17 00:00:00 2001 From: Esther-2584 Date: Fri, 23 Oct 2026 09:33:47 -0400 Subject: [PATCH 4/8] fix(calibration): answer our internal review of the round - A record whose pricing stamp is newer than this binary's is left alone whatever its multiplier field holds, so a later binary's record in a shape this one can't read isn't treated as no record and written over by a provisional 1's first sample. - Dropping a provisional multiplier to 1 clears the declared mark in memory as the record it writes does, so an operator's seed over a marked record (the unwedge advice) can't bring the mark back on the 1. - Declared-ness is one test, overhead != null, at the run gate, the rejection line and the pricing too, as at the mark. - Text: the declared mark means the two-sample move has run for the multiplier (a disagreeing pair steps and marks too); a held sample ends when what it was held for is settled another way; a store with no record writes nothing because each load decides the same; the unwedge advice is for a multiplier above 1; the #328 fragment states the trade, one sample without the two-sample hold. - Tests: an undeclared sample between the two leaves the held sample; a falsy non-null overhead is declared in the run; a newer-epoch record with no readable multiplier is left alone; the drop clears the mark; the dead-band and rejected-declared tests show their samples were taken; the provisional-replacement test samples at 1.4; the dead-band drift test seeds a learned 1. Co-Authored-By: Claude Opus 5.5 --- .../323-calibration-request-overhead.fixed.md | 8 +-- ...328-fresh-calibration-provisional.fixed.md | 9 +-- src/strategies/autobiographical.ts | 63 +++++++++++-------- test/calibration-band.test.ts | 45 ++++++++++++- test/calibration-overhead.test.ts | 37 +++++++++-- 5 files changed, 121 insertions(+), 41 deletions(-) diff --git a/changelog.d/323-calibration-request-overhead.fixed.md b/changelog.d/323-calibration-request-overhead.fixed.md index 3b1a141b..68050e01 100644 --- a/changelog.d/323-calibration-request-overhead.fixed.md +++ b/changelog.d/323-calibration-request-overhead.fixed.md @@ -30,9 +30,9 @@ samples step by the EMA. (Out of the band, the three-sample run above applies.) The first of the two is held in memory, so a restart between them costs one more sample. An undeclared sample that moves the multiplier - writes the record without the flag. A store whose multiplier puts every - compile over budget gets no sample to move it, as before: to unwedge it, - seed the record as provisional on the stopped store, and the refused - compile drops it to 1: `setStateJson('/autobio:calibration', + writes the record without the flag. A store whose multiplier, above 1, + puts every compile over budget gets no sample to move it, as before: to + unwedge it, seed the record as provisional on the stopped store, and the + refused compile drops it to 1: `setStateJson('/autobio:calibration', { multiplier, at, pricing: , provisional: true })`. `RequestOverhead` is exported from the package. diff --git a/changelog.d/328-fresh-calibration-provisional.fixed.md b/changelog.d/328-fresh-calibration-provisional.fixed.md index a1be6553..354929cf 100644 --- a/changelog.d/328-fresh-calibration-provisional.fixed.md +++ b/changelog.d/328-fresh-calibration-provisional.fixed.md @@ -1,7 +1,8 @@ - A store with no calibration record starts from a provisional 1, as a - discard's restart does (#328): its first sample inside the calibration - band sets the multiplier, declared or not, instead of stepping a fifth of - the way per sample (nine applied changes, each a reprice of every message - and summary, to reach a true 1.4). Until that sample, render stats report + discard's restart does (#328). Its first sample inside the calibration + band sets the multiplier, declared or not: one sample, without the + two-sample hold, where it stepped a fifth of the way per sample (nine + applied changes, each a reprice of every message and summary, to reach a + true 1.4). Until that sample, render stats report `calibration.provisional: true`. A record from a newer pricing epoch, or a read of the record that throws, still loads a learned 1. diff --git a/src/strategies/autobiographical.ts b/src/strategies/autobiographical.ts index f5947840..9f4d86a3 100644 --- a/src/strategies/autobiographical.ts +++ b/src/strategies/autobiographical.ts @@ -13923,7 +13923,7 @@ export class AutobiographicalStrategy implements ResettableStrategy { // The overhead in the window's units (current calibrated), so the implied // multiplier below is real / raw(window + overhead) and doesn't depend on // the multiplier it's measured at: one sample states it. - const overheadTokens = overhead ? this.priceRequestOverhead(overhead) : 0; + const overheadTokens = declared ? this.priceRequestOverhead(overhead!) : 0; const injected = this._lastCompileInjected; const est = window + injected + overheadTokens; const parts = [ @@ -13965,7 +13965,7 @@ export class AutobiographicalStrategy implements ResettableStrategy { let fromRun = false; if (!ratioInBand && !observedInBand) { const run = this._outOfBandRun; - this._outOfBandRun = !overhead ? null + this._outOfBandRun = !declared ? null : run && Math.abs(observed / run.first - 1) <= OUT_OF_BAND_RUN_AGREEMENT ? { first: run.first, count: run.count + 1 } : { first: observed, count: 1 }; @@ -13973,7 +13973,7 @@ export class AutobiographicalStrategy implements ResettableStrategy { if (count < OUT_OF_BAND_RUN_LENGTH) { this.diag.error( `[estimator-calibration] REJECTED out-of-band sample real/est=${ratio.toFixed(2)} ${counts} — ` + - (overhead + (declared ? `multiplier stays ${this._calibration.toFixed(2)} until ${OUT_OF_BAND_RUN_LENGTH} samples in a row ` + `agree within ${OUT_OF_BAND_RUN_AGREEMENT * 100}% (${count} of ${OUT_OF_BAND_RUN_LENGTH})` : `not a window-shaped request; multiplier stays ${this._calibration.toFixed(2)}; if every sample ` + @@ -13999,10 +13999,11 @@ export class AutobiographicalStrategy implements ResettableStrategy { // set their ruler alone. The held sample is in memory only, so a restart // between them costs one more sample; an undeclared sample between them // leaves it, since an implied multiplier doesn't depend on the multiplier - // in force. Either way the record then says declared samples agree with - // its multiplier (declaredOverhead), and an undeclared write takes that - // back. A provisional multiplier, or a consistent run (out of the band, - // three samples), is replaced by one sample as before. + // in force. Either way the record is then marked (declaredOverhead): the + // move has run for it, and later declared samples step by the EMA; an + // undeclared sample that moves the multiplier takes the mark back. A + // provisional multiplier, or a consistent run (out of the band, three + // samples), is replaced by one sample as before. const markDeclared = declared && !this._calibrationDeclared; let replacement: number | null = null; if (markDeclared && !this._calibrationProvisional && !fromRun) { @@ -14020,9 +14021,10 @@ export class AutobiographicalStrategy implements ResettableStrategy { this._declaredCandidate = null; if (Math.abs(observed / candidate - 1) <= OUT_OF_BAND_RUN_AGREEMENT) replacement = (candidate + observed) / 2; } else if (declared) { - // A declared sample that moves the ruler another way (a provisional - // replacement, a run) or meets a marked record ends any held sample: - // it was measured against a ruler this one replaces. + // A declared sample that decides the multiplier another way (a + // provisional replacement, a run) or meets a marked record ends any + // held sample: what it was held for, whether two declared samples + // should replace an unmarked multiplier, has been settled. this._declaredCandidate = null; } @@ -14252,9 +14254,12 @@ export class AutobiographicalStrategy implements ResettableStrategy { * multiplier wasn't learned from declared samples, held for the next * declared sample (see reportRealInputTokens). In memory only. */ private _declaredCandidate: number | null = null; - /** Whether declared samples agree with the multiplier in force: set by - * one, or confirmed by two inside the dead band. Persisted as the record's - * declaredOverhead (see reportRealInputTokens). */ + /** Whether the two-sample move has run for the multiplier in force: set + * once declared samples have set, stepped or confirmed it (one sample for a + * provisional multiplier or a run, two for a learned one). Persisted as the + * record's declaredOverhead; an undeclared sample that moves the multiplier + * clears it, and so does dropping a provisional multiplier to 1 (see + * reportRealInputTokens). */ private _calibrationDeclared = false; private _calibrationLoaded = false; /** Set when an older-epoch multiplier was discarded; persisted with the record. */ @@ -14322,8 +14327,8 @@ export class AutobiographicalStrategy implements ResettableStrategy { * sample doesn't turn the guess into a learned value, and a later epoch's * load re-stamps it, still provisional, rather than discarding it again * over the reset it carries. A store with no record starts from a - * provisional 1 too, and writes nothing until its first sample, since no - * record already loads that way (#328). A dry run under a + * provisional 1 too (#328). It writes nothing until its first sample: with + * no record, each load decides the same. A dry run under a * provisional multiplier above 1 takes the same fallback at the picker's * refusal, which it otherwise only reports. * @@ -14577,18 +14582,19 @@ export class AutobiographicalStrategy implements ResettableStrategy { // re-stamping it as the discard warning advises would destroy it. // This binary's learner still writes its own record over it on its // first applied sample (see CALIBRATION_PRICING_EPOCH). - } else { - // No record, or none with a multiplier to use: no sample has ever - // set this store's 1, so it's a guess, as a discard's restart is, - // and the first sample the band accepts sets it (#328). Learned, it - // would take the slowest path for the store with the least - // evidence, nine applied EMA steps to reach a true 1.4, each a - // reprice of every message and summary. Nothing is written: with no - // record, the next load decides the same, and the first sample's - // write gives the stamp. A record from a newer epoch (above) and a - // read that throws (below) aren't this case: each may hold a - // multiplier worth keeping, which a provisional 1's first sample - // would write over even when it only confirms the 1. + } else if (!(typeof saved?.pricing === 'number' && saved.pricing > AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH)) { + // No record, or none with a multiplier to use: nothing this store + // kept says what its multiplier is, so its 1 is a guess, as a + // discard's restart is, and the first sample the band accepts sets + // it (#328). Learned, it would take the slowest path for the store + // with the least evidence, nine applied EMA steps to reach a true + // 1.4, each a reprice of every message and summary. Nothing is + // written: with no record, the next load decides the same, and the + // first sample's write gives the stamp. A record from a newer epoch, + // with a multiplier this binary can read or not, and a read that + // throws (below) aren't this case: each may hold what a later binary + // or the store keeps, which a provisional 1's first sample would + // write over even when it only confirms the 1. this._calibrationProvisional = { fromEpoch: null }; } } catch { /* absent slot is fine */ } @@ -14619,6 +14625,9 @@ export class AutobiographicalStrategy implements ResettableStrategy { const reset: CalibrationReset = { discardedMultiplier: discarded, fromEpoch: provisional.fromEpoch, at: Date.now() }; this._calibration = 1; this._calibrationReset = reset; + // The mark was about the multiplier dropped; the record below is written + // without it, and memory agrees. + this._calibrationDeclared = false; this.applyCalibration(); if (dryRun) return true; this.diag.warn( diff --git a/test/calibration-band.test.ts b/test/calibration-band.test.ts index fcc391e0..f40bc215 100644 --- a/test/calibration-band.test.ts +++ b/test/calibration-band.test.ts @@ -216,6 +216,21 @@ describe('estimator calibration band', () => { assert.deepEqual(warned.filter((l) => l.includes('[estimator-calibration]')), []); }); + it('a record from a newer pricing epoch with no multiplier this binary can read is left alone too', async () => { + const first = await openManager('newer-epoch-unreadable'); + await first.cm.compile(BUDGET); + const future = { multipliers: { default: 1.4 }, at: 123, pricing: AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH + 1 }; + first.internals.store!.setStateJson(first.internals.calibrationStateId, future); + first.cm.close(); + const { cm, strategy, internals } = await openManager('newer-epoch-unreadable'); + await cm.compile(BUDGET); + assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1, null]); + // A first sample that only confirms the 1 writes nothing over it. + strategy.reportRealInputTokens(internals._lastCompileEstimate * 1.01); + assert.deepStrictEqual(internals.store!.getStateJson(internals.calibrationStateId), future); + cm.close(); + }); + it('a multiplier learned under the current pricing epoch survives reopen', async () => { const first = await openManager('current-epoch'); await first.cm.compile(BUDGET); @@ -731,6 +746,28 @@ describe('estimator calibration band', () => { belowReopened.cm.close(); }); + it('dropping a provisional multiplier to 1 drops its declared mark in memory as on disk', async () => { + // An operator following the unwedge advice may seed provisional over a + // record a declared caller had marked. + const epoch = AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH; + const tight = await tightBudget('drop-declared-probe', 1.2); // fits at 1.0, not at 1.45 + const first = await openManager('drop-declared'); + await first.cm.compile(BUDGET); + first.internals.store!.setStateJson(first.internals.calibrationStateId, + { multiplier: 1.45, at: 1, pricing: epoch, provisional: true, declaredOverhead: true }); + first.cm.close(); + const { cm, strategy, internals } = await openManager('drop-declared'); + const declared = () => (internals as unknown as { _calibrationDeclared: boolean })._calibrationDeclared; + await calibrationWarnings(() => cm.compile(tight)); + assert.equal(internals._calibration, 1, 'premise: the refused compile dropped it'); + assert.equal(declared(), false); + // An undeclared sample confirming the 1 writes no mark either. + strategy.reportRealInputTokens(internals._lastCompileEstimate * 1.01); + const saved = internals.store!.getStateJson(internals.calibrationStateId) as { multiplier?: number; declaredOverhead?: boolean }; + assert.deepStrictEqual([saved.multiplier, saved.declaredOverhead], [1, undefined]); + cm.close(); + }); + it('a discard\'s provisional 1 survives a restart until a sample confirms it', async () => { // Both discard writers (the load's and the drop's) persist the 1 as // provisional; a reopen before the first sample must not read it as @@ -1260,9 +1297,15 @@ describe('estimator calibration band', () => { }); it('a drift inside 2% is not applied; once the EMA passes 2% it is', async () => { + // A learned 1: a store with no record would be set by its first sample (#328). + const first = await openManager('deadband'); + await first.cm.compile(BUDGET); + first.internals.store!.setStateJson(first.internals.calibrationStateId, + { multiplier: 1, at: 1, pricing: AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH }); + first.cm.close(); const { cm, strategy, internals } = await openManager('deadband'); await cm.compile(BUDGET); - assert.equal(internals._calibration, 1); + assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1, null], 'learned'); // real/est = 1.01 every compile: the EMA creeps toward 1.01 and never // leaves the band, so nothing is repriced between compiles. for (let i = 0; i < 12; i++) { diff --git a/test/calibration-overhead.test.ts b/test/calibration-overhead.test.ts index c80bdb45..9841942f 100644 --- a/test/calibration-overhead.test.ts +++ b/test/calibration-overhead.test.ts @@ -508,6 +508,18 @@ describe('estimator calibration: a store that learned the overhead as a rate', ( cm.close(); }); + it('an undeclared sample between the two leaves the held sample, and the pair still replaces', async () => { + const { cm, strategy, internals } = await seeded('held-undeclared', { multiplier: 1.8, at: 1, pricing: epoch }); + await sampleAt(cm, strategy, internals, 1, true); + assert.equal(internals._calibration, 1.8, 'the first is held'); + await sampleAt(cm, strategy, internals, 1, false); + assert.ok(Math.abs(internals._calibration - 1.64) <= 0.01, `the undeclared sample steps, got ${internals._calibration}`); + await sampleAt(cm, strategy, internals, 1, true); + assert.ok(Math.abs(internals._calibration - 1) <= 0.01, `paired with the held sample and replaced, got ${internals._calibration}`); + assert.equal(declaredOf(internals), true); + cm.close(); + }); + it('a consistent run between the two ends the held sample', async () => { const { cm, strategy, internals } = await seeded('held-run', { multiplier: 1, at: 1, pricing: epoch }); await sampleAt(cm, strategy, internals, 1, true); @@ -592,9 +604,10 @@ describe('estimator calibration: a store that learned the overhead as a rate', ( it('a provisional multiplier is replaced by one declared sample, and marked declared', async () => { const { cm, strategy, internals } = await seeded('provisional-declared', { multiplier: 1, at: 1, pricing: epoch, provisional: true }); - await sampleAt(cm, strategy, internals, 1, true); - const saved = internals.store!.getStateJson(internals.calibrationStateId) as { provisional?: boolean; declaredOverhead?: boolean }; - assert.deepEqual([saved.provisional, saved.declaredOverhead], [undefined, true]); + await sampleAt(cm, strategy, internals, 1.4, true); + assert.ok(Math.abs(internals._calibration - 1.4) <= 0.01, `replaced, not stepped, got ${internals._calibration}`); + assert.equal(internals._calibrationProvisional, null); + assert.equal(declaredOf(internals), true); cm.close(); }); @@ -610,15 +623,29 @@ describe('estimator calibration: a store that learned the overhead as a rate', ( it('an undeclared sample inside the dead band leaves a record marked, and a rejected declared one leaves it unmarked', async () => { const marked = await seeded('deadband-undeclared', { multiplier: 1, at: 1, pricing: epoch, declaredOverhead: true }); await sampleAt(marked.cm, marked.strategy, marked.internals, 1.01, false); - assert.deepEqual([marked.internals._calibration, marked.internals._calibrationDeclared, declaredOf(marked.internals)], [1, true, true]); + const ema = (marked.internals as unknown as { _calibrationEma: number | null })._calibrationEma; + assert.ok(ema !== null && Math.abs(ema - 1.002) < 0.001, `premise: the sample reached the EMA, got ${ema}`); + assert.deepStrictEqual([marked.internals._calibration, marked.internals._calibrationDeclared, declaredOf(marked.internals)], [1, true, true]); marked.cm.close(); const unmarked = await seeded('rejected-declared', { multiplier: 1, at: 1, pricing: epoch }); await unmarked.cm.compile(BUDGET); unmarked.strategy.reportRealInputTokens(unmarked.internals._lastCompileEstimate * 2.1, {}); - assert.deepEqual([unmarked.internals._calibration, unmarked.internals._calibrationDeclared, declaredOf(unmarked.internals)], [1, false, undefined]); + assert.equal((unmarked.internals as unknown as { _outOfBandRun: { count: number } | null })._outOfBandRun?.count, 1, 'premise: the sample was rejected as the start of a run'); + assert.deepStrictEqual([unmarked.internals._calibration, unmarked.internals._calibrationDeclared, declaredOf(unmarked.internals)], [1, false, undefined]); unmarked.cm.close(); }); + it('any overhead a JS caller passes but null or undefined counts as declared, in the run as in the mark', async () => { + const { cm, strategy, internals } = await seeded('falsy-overhead', { multiplier: 1, at: 1, pricing: epoch }); + for (let i = 0; i < 3; i++) { + await cm.compile(BUDGET); + strategy.reportRealInputTokens(internals._lastCompileEstimate * 2.1, 0 as unknown as RequestOverhead); + } + assert.equal(internals._calibration, 1.8, 'learned as a declared run'); + assert.equal(declaredOf(internals), true); + cm.close(); + }); + it('a null overhead from a JS caller counts as undeclared', async () => { const { cm, strategy, internals } = await seeded('null-overhead', { multiplier: 1, at: 1, pricing: epoch }); await cm.compile(BUDGET); From 7099a7b186c7621a2465e394dbf57e9c7d6967b4 Mon Sep 17 00:00:00 2001 From: Esther-2584 Date: Fri, 23 Oct 2026 09:51:00 -0400 Subject: [PATCH 5/8] docs(calibration): say what the no-record, newer-epoch and drop paths do; pin the drop's dry restore From our internal review of the round. No behaviour changes. - The load comment separates the callers: an undeclared one took nine applied EMA steps to reach a true 1.4, a declared one two samples with the first held. - The newer-epoch comment and the epoch doc say this binary also writes its own record over a newer one when two declared samples confirm and mark the 1, beside its first applied sample. - The catch says a record the store can't read loads 1, not provisional. - The drop's comment says the mark goes with the multiplier dropped. - The #328 fragment and RenderStats.calibration's doc include a record with no multiplier to use, and say a newer-epoch or unreadable record loads 1 as before, not provisional. - Tests: the drop test loads the marked seed with a committing compile, shows a dry compile's drop is put back, then reads the record the live drop writes, with the seeded 1.45 in the reset; the newer-epoch test shows its confirming sample reached the EMA. Co-Authored-By: Claude Opus 5.5 --- .../328-fresh-calibration-provisional.fixed.md | 17 +++++++++-------- src/strategies/autobiographical.ts | 18 +++++++++++------- src/types/strategy.ts | 3 ++- test/calibration-band.test.ts | 13 ++++++++++++- 4 files changed, 34 insertions(+), 17 deletions(-) diff --git a/changelog.d/328-fresh-calibration-provisional.fixed.md b/changelog.d/328-fresh-calibration-provisional.fixed.md index 354929cf..1c7c4836 100644 --- a/changelog.d/328-fresh-calibration-provisional.fixed.md +++ b/changelog.d/328-fresh-calibration-provisional.fixed.md @@ -1,8 +1,9 @@ -- A store with no calibration record starts from a provisional 1, as a - discard's restart does (#328). Its first sample inside the calibration - band sets the multiplier, declared or not: one sample, without the - two-sample hold, where it stepped a fifth of the way per sample (nine - applied changes, each a reprice of every message and summary, to reach a - true 1.4). Until that sample, render stats report - `calibration.provisional: true`. A record from a newer pricing epoch, or a - read of the record that throws, still loads a learned 1. +- A store with no calibration record, or a record with no finite + multiplier, starts from a provisional 1, as a discard's restart does + (#328). Its first sample inside the calibration band sets the multiplier, + declared or not: one sample, without the two-sample hold, where it + stepped a fifth of the way per sample (nine applied changes, each a + reprice of every message and summary, to reach a true 1.4). Until that + sample, render stats report `calibration.provisional: true`. A record + from a newer pricing epoch, or a read of the record that throws, still + loads 1 as before, not provisional. diff --git a/src/strategies/autobiographical.ts b/src/strategies/autobiographical.ts index 9f4d86a3..a11798ca 100644 --- a/src/strategies/autobiographical.ts +++ b/src/strategies/autobiographical.ts @@ -14333,7 +14333,8 @@ export class AutobiographicalStrategy implements ResettableStrategy { * refusal, which it otherwise only reports. * * A binary from before a bump still learns after a rollback, and its first - * applied sample overwrites the newer epoch's record with its own stamp + * applied sample, or two declared samples that confirm and mark its 1, + * overwrite the newer epoch's record with its own stamp * (reportRealInputTokens); rolling forward again treats that as an older * record. */ @@ -14581,14 +14582,17 @@ export class AutobiographicalStrategy implements ResettableStrategy { // leaves it on disk and doesn't use it, and logs nothing, since // re-stamping it as the discard warning advises would destroy it. // This binary's learner still writes its own record over it on its - // first applied sample (see CALIBRATION_PRICING_EPOCH). + // first applied sample, or when two declared samples confirm and + // mark the 1 (see CALIBRATION_PRICING_EPOCH). } else if (!(typeof saved?.pricing === 'number' && saved.pricing > AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH)) { // No record, or none with a multiplier to use: nothing this store // kept says what its multiplier is, so its 1 is a guess, as a // discard's restart is, and the first sample the band accepts sets // it (#328). Learned, it would take the slowest path for the store - // with the least evidence, nine applied EMA steps to reach a true - // 1.4, each a reprice of every message and summary. Nothing is + // with the least evidence: for an undeclared caller nine applied + // EMA steps to reach a true 1.4, each a reprice of every message and + // summary, and for a declared one two samples, the first held. + // Nothing is // written: with no record, the next load decides the same, and the // first sample's write gives the stamp. A record from a newer epoch, // with a multiplier this binary can read or not, and a read that @@ -14597,7 +14601,7 @@ export class AutobiographicalStrategy implements ResettableStrategy { // write over even when it only confirms the 1. this._calibrationProvisional = { fromEpoch: null }; } - } catch { /* absent slot is fine */ } + } catch { /* a record the store can't read loads 1, not provisional: see the no-record branch above */ } } // A dry select changes the store's multiplier only where select() read it // first and can put it back (see MessageStoreView). @@ -14625,8 +14629,8 @@ export class AutobiographicalStrategy implements ResettableStrategy { const reset: CalibrationReset = { discardedMultiplier: discarded, fromEpoch: provisional.fromEpoch, at: Date.now() }; this._calibration = 1; this._calibrationReset = reset; - // The mark was about the multiplier dropped; the record below is written - // without it, and memory agrees. + // The mark was about the multiplier dropped, and goes with it, as the + // record below is written without it. this._calibrationDeclared = false; this.applyCalibration(); if (dryRun) return true; diff --git a/src/types/strategy.ts b/src/types/strategy.ts index fbc2dfd1..096b557d 100644 --- a/src/types/strategy.ts +++ b/src/types/strategy.ts @@ -747,7 +747,8 @@ export interface RenderStats { * than stepping toward it (see * `AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH`): one kept from a * record without a stamp, the 1 a discard restarts from, or the 1 a store - * with no calibration record starts from. A discard's 1 stays + * with no calibration record, or none with a multiplier to use, starts + * from. A discard's 1 stays * provisional through a later epoch's load, which re-stamps it whatever * that epoch repriced. * `reset` is present once a multiplier from an older or unknown epoch has diff --git a/test/calibration-band.test.ts b/test/calibration-band.test.ts index f40bc215..0325ef91 100644 --- a/test/calibration-band.test.ts +++ b/test/calibration-band.test.ts @@ -227,6 +227,8 @@ describe('estimator calibration band', () => { assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1, null]); // A first sample that only confirms the 1 writes nothing over it. strategy.reportRealInputTokens(internals._lastCompileEstimate * 1.01); + const ema = (internals as unknown as { _calibrationEma: number | null })._calibrationEma; + assert.ok(ema !== null && Math.abs(ema - 1.002) < 0.001, `premise: the sample reached the EMA, got ${ema}`); assert.deepStrictEqual(internals.store!.getStateJson(internals.calibrationStateId), future); cm.close(); }); @@ -758,9 +760,18 @@ describe('estimator calibration band', () => { first.cm.close(); const { cm, strategy, internals } = await openManager('drop-declared'); const declared = () => (internals as unknown as { _calibrationDeclared: boolean })._calibrationDeclared; + await cm.compile(BUDGET); // loads the seed, which fits this budget + assert.deepStrictEqual([internals._calibration, declared()], [1.45, true], 'premise: loaded, marked'); + // A dry compile drops it too, in the dry compile only: select() puts the mark back. + const dry = await calibrationWarnings(() => cm.compile(tight, undefined, { dryRun: true })); + assert.equal(dry.result.renderStats!.calibration!.multiplier, 1, 'premise: the dry compile dropped it'); + assert.deepStrictEqual([internals._calibration, declared()], [1.45, true], 'put back after the dry compile that dropped it'); await calibrationWarnings(() => cm.compile(tight)); - assert.equal(internals._calibration, 1, 'premise: the refused compile dropped it'); + assert.equal(internals._calibration, 1); + assert.equal(cm.getRenderStats()!.calibration!.reset?.discardedMultiplier, 1.45, 'premise: the seeded 1.45 was loaded and dropped'); assert.equal(declared(), false); + const dropped = internals.store!.getStateJson(internals.calibrationStateId) as { multiplier?: number; provisional?: boolean; declaredOverhead?: boolean }; + assert.deepStrictEqual([dropped.multiplier, dropped.provisional, dropped.declaredOverhead], [1, true, undefined], 'the record the drop writes'); // An undeclared sample confirming the 1 writes no mark either. strategy.reportRealInputTokens(internals._lastCompileEstimate * 1.01); const saved = internals.store!.getStateJson(internals.calibrationStateId) as { multiplier?: number; declaredOverhead?: boolean }; From e0dea9ac16029ca491faea0698db55dc0e7e0630 Mon Sep 17 00:00:00 2001 From: Esther-2584 Date: Sat, 24 Oct 2026 21:34:27 -0400 Subject: [PATCH 6/8] docs(calibration): the exceptions to learned and provisional, and what a declared caller paid - RenderStats.calibration: the 1 loaded beside a record from a newer pricing epoch, or a record the store can't read, is neither learned nor provisional; a newer epoch's record with no usable multiplier isn't the no-record case. - The fragment for anima-research/context-manager#328: a learned 1 took nine applied changes per undeclared caller, and two samples, the first held, for a declared one. - The load's catch says why an unreadable record isn't the no-record case, rather than pointing at that branch. Comments, docs and the changelog only. Co-Authored-By: Claude Opus 5.5 --- changelog.d/328-fresh-calibration-provisional.fixed.md | 7 ++++--- src/strategies/autobiographical.ts | 2 +- src/types/strategy.ts | 7 +++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/changelog.d/328-fresh-calibration-provisional.fixed.md b/changelog.d/328-fresh-calibration-provisional.fixed.md index 1c7c4836..55485a4b 100644 --- a/changelog.d/328-fresh-calibration-provisional.fixed.md +++ b/changelog.d/328-fresh-calibration-provisional.fixed.md @@ -1,9 +1,10 @@ - A store with no calibration record, or a record with no finite multiplier, starts from a provisional 1, as a discard's restart does (#328). Its first sample inside the calibration band sets the multiplier, - declared or not: one sample, without the two-sample hold, where it - stepped a fifth of the way per sample (nine applied changes, each a - reprice of every message and summary, to reach a true 1.4). Until that + declared or not: one sample, without the two-sample hold. Learned, a 1 + stepped a fifth of the way per undeclared sample (nine applied changes, + each a reprice of every message and summary, to reach a true 1.4), and + took two declared samples, the first held. Until that sample, render stats report `calibration.provisional: true`. A record from a newer pricing epoch, or a read of the record that throws, still loads 1 as before, not provisional. diff --git a/src/strategies/autobiographical.ts b/src/strategies/autobiographical.ts index a11798ca..bce8c041 100644 --- a/src/strategies/autobiographical.ts +++ b/src/strategies/autobiographical.ts @@ -14601,7 +14601,7 @@ export class AutobiographicalStrategy implements ResettableStrategy { // write over even when it only confirms the 1. this._calibrationProvisional = { fromEpoch: null }; } - } catch { /* a record the store can't read loads 1, not provisional: see the no-record branch above */ } + } catch { /* a record the store can't read loads 1, not provisional: unlike no record, it may hold what the store keeps (see the no-record branch above) */ } } // A dry select changes the store's multiplier only where select() read it // first and can put it back (see MessageStoreView). diff --git a/src/types/strategy.ts b/src/types/strategy.ts index 096b557d..0ac87ed1 100644 --- a/src/types/strategy.ts +++ b/src/types/strategy.ts @@ -742,13 +742,16 @@ export interface RenderStats { * that isn't provisional was learned under that epoch's pricing, or under * an older epoch's and kept, re-stamped, because nothing this store holds * was repriced since; either way it's reported under the epoch in force. + * The exception is the 1 loaded beside a record from a newer pricing + * epoch, or a record the store can't read: neither is used, and that 1 + * is neither learned nor provisional. * `provisional` is true while the multiplier is a guess, which the first * provider sample the calibration's band accepts replaces outright, rather * than stepping toward it (see * `AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH`): one kept from a * record without a stamp, the 1 a discard restarts from, or the 1 a store - * with no calibration record, or none with a multiplier to use, starts - * from. A discard's 1 stays + * with no calibration record, or none with a multiplier to use other than + * a newer epoch's, starts from. A discard's 1 stays * provisional through a later epoch's load, which re-stamps it whatever * that epoch repriced. * `reset` is present once a multiplier from an older or unknown epoch has From 9cbe7d7becadfc9d43a622bcd37403ba536f23f8 Mon Sep 17 00:00:00 2001 From: Esther-2584 Date: Sun, 25 Oct 2026 19:51:44 -0400 Subject: [PATCH 7/8] docs(calibration): keep only the comments the code can't say The comments this branch adds now keep only public API docs, a link to #323 where a stored multiplier carries a history the code can't show, and corrections to comments of #324 that were false, or that this branch's changes made false. The reasons the cut comments gave are here. - Message-position injections are priced at the select, at the multiplier the window is priced at, so the sample counts them as the request does. ContextManager.compile passes them in SelectOptions.messageInjections. - A sample is declared when the caller passes an overhead other than null or undefined: {} declares that the request sent nothing besides the window and the injections. - A run's samples agree with its first, not each with the one before, so a drifting run (a growing unpriced part) isn't learned as a constant. The same agreement decides whether two declared samples move a record whose multiplier wasn't learned from declared samples. - A consistent run replaces a multiplier outright because its samples agree on the multiplier, and stepping toward it would take several more applied changes, each a reprice of every message and summary. - The two-sample move. A record no declared sample has set holds the overhead as a rate. Inside the band, the first declared sample on it is held and the next decides: within the agreement, their mean replaces the multiplier instead of several EMA steps, and otherwise the EMA steps. Two, not one, because the move reaches every store with a learned multiplier whose caller declares its overhead, those already right included, and one wild sample shouldn't set their ruler alone. The held sample is in memory only, so a restart or a branch switch between the two costs one more sample. An undeclared sample between them leaves it, since an implied multiplier doesn't depend on the multiplier in force. A declared sample that settles the multiplier another way (a provisional replacement, a run) or meets a marked record ends it. Once the move has run, the record is marked declaredOverhead and later declared samples step by the EMA. An undeclared sample that moves the multiplier clears the mark, and so does dropping a provisional multiplier to 1, since the mark was about the multiplier dropped. - No record, or none with a multiplier to use, starts from a provisional 1 (#328). Learned, the store with the least evidence would take the slowest path: nine applied EMA steps to a true 1.4 for an undeclared caller, two samples, the first held, for a declared one. The load writes nothing, since with no record the next load decides the same. A record from a newer epoch, readable or not, and a read that throws aren't this case: each may hold what a later binary or the store keeps, which a provisional 1's first sample would write over even when it only confirms the 1. - Tests seed a record where they need a learned multiplier, since a store with no record now starts provisional. The drop test seeds a record that is both provisional and marked, which only a hand-written record holds, to pin that memory agrees with the record the drop writes. Behaviour is unchanged. One condition is named (fromNewerEpoch), a test helper's parameter is renamed (impliedMultiplier), and what some cut test comments said is now in assertion messages and one test title. Co-Authored-By: Claude Opus 5.5 --- src/context-manager.ts | 3 +- src/strategies/autobiographical.ts | 80 ++++-------------------------- test/calibration-band.test.ts | 33 ++++-------- test/calibration-overhead.test.ts | 27 +++------- 4 files changed, 30 insertions(+), 113 deletions(-) diff --git a/src/context-manager.ts b/src/context-manager.ts index 04fca26e..5a288dee 100644 --- a/src/context-manager.ts +++ b/src/context-manager.ts @@ -1178,8 +1178,7 @@ export class ContextManager { const branch = branchRefOf(this.store); const view = this.strategyMessageView(); - // Get selected entries from strategy. The injections it will sit among - // go with it, for its calibration's count of the request (SelectOptions). + // Get selected entries from strategy const messageInjections = (injections ?? []) .filter((inj) => inj.position === 'beforeUser' || inj.position === 'afterUser') .map((inj) => inj.content); diff --git a/src/strategies/autobiographical.ts b/src/strategies/autobiographical.ts index bce8c041..13b28aa6 100644 --- a/src/strategies/autobiographical.ts +++ b/src/strategies/autobiographical.ts @@ -1857,9 +1857,7 @@ function imageGroup(block: ContentBlock): unknown { /** Out-of-band calibration samples in a row it takes to learn from them, and * how close each one's implied multiplier must be to the run's first to - * continue the run (see reportRealInputTokens). The agreement also decides - * whether two declared samples move a store whose multiplier wasn't - * learned from declared samples. */ + * continue the run (see reportRealInputTokens). */ const OUT_OF_BAND_RUN_LENGTH = 3; const OUT_OF_BAND_RUN_AGREEMENT = 0.1; @@ -13867,10 +13865,6 @@ export class AutobiographicalStrategy implements ResettableStrategy { this._lastCompileEstimate = rs.head.tokens + rs.tail.tokens + rs.middleRaw.tokens + rs.summaries.l1.tokens + rs.summaries.l2.tokens + rs.summaries.l3.tokens; - // What the compile inserts among these entries after the select (its - // message-position injections) is in the request and not in the - // window: priced here, at the multiplier the window was priced at, so - // the sample counts it as the request does (#323). let injected = 0; for (const content of opts?.messageInjections ?? []) { injected += store.estimateTokens({ content } as unknown as StoredMessage); @@ -13882,7 +13876,6 @@ export class AutobiographicalStrategy implements ResettableStrategy { } private _lastCompileEstimate = 0; - /** The armed compile's message-position injections, priced with its window. */ private _lastCompileInjected = 0; private _storeView: MessageStoreView | null = null; @@ -13916,8 +13909,6 @@ export class AutobiographicalStrategy implements ResettableStrategy { // exhausting the picker on every wake. One sample per compile, always. if (!this._calibrationArmed) return; this._calibrationArmed = false; - // Declared: the caller passed what the request sent besides the window - // and the compile's injections ({} included). A JS caller's null isn't. const declared = overhead != null; // The overhead in the window's units (current calibrated), so the implied @@ -13938,8 +13929,7 @@ export class AutobiographicalStrategy implements ResettableStrategy { const observed = ratio * current; // back out the multiplier already applied // SANITY BAND: a representative sample sits near 1. Anything wilder is a // structural mismatch (a request we didn't compile, a partial compile, a - // provider quirk) — never evidence about chars-per-token. Log, don't learn, - // except a declared consistent run (below). + // provider quirk) — never evidence about chars-per-token. Log, don't learn. // The band is also checked on the IMPLIED raw multiplier: when the // multiplier sits at a clamp edge because the per-class rates were wrong // (signed thinking at a flat 600) and the rates are then fixed, every @@ -13953,11 +13943,8 @@ export class AutobiographicalStrategy implements ResettableStrategy { // the same number, so a store whose true ratio is past the band (every // epoch discard restarts there) rejects every sample and never learns. // A single wild sample is still noise, but a run of them that agree with - // the run's first, within OUT_OF_BAND_RUN_AGREEMENT, is a ruler that's - // off, not a request we didn't compile: the third in a row is learned, - // outright, which lands it on the clamp edge. Agreement is with the first, - // not the one before, so a drifting run (a growing unpriced part) isn't - // learned as a constant. + // the run's first is a ruler that's off, not a request we didn't compile: + // the third in a row is learned, outright, which lands it on the clamp edge. // Only for samples whose caller declared what it sent besides the window: // undeclared, a consistent ratio past the band is what tool schemas and a // system prompt look like, and learning it would fold the window to the @@ -13988,22 +13975,8 @@ export class AutobiographicalStrategy implements ResettableStrategy { this._outOfBandRun = null; } - // A multiplier learned before samples declared their overhead learned it - // as a rate (#323). Inside the band, the first declared sample on such a - // record is held, and the next declared sample decides: if the two agree - // within OUT_OF_BAND_RUN_AGREEMENT their mean replaces the multiplier, - // instead of about ten EMA steps, each a reprice of every message and - // summary; if not, the EMA steps as for any sample. Two, not one: this - // reaches every store that samples after the upgrade, those whose - // multiplier is already right included, and one wild sample shouldn't - // set their ruler alone. The held sample is in memory only, so a restart - // between them costs one more sample; an undeclared sample between them - // leaves it, since an implied multiplier doesn't depend on the multiplier - // in force. Either way the record is then marked (declaredOverhead): the - // move has run for it, and later declared samples step by the EMA; an - // undeclared sample that moves the multiplier takes the mark back. A - // provisional multiplier, or a consistent run (out of the band, three - // samples), is replaced by one sample as before. + // A multiplier learned from samples that didn't declare their overhead + // holds that overhead as a rate (#323). const markDeclared = declared && !this._calibrationDeclared; let replacement: number | null = null; if (markDeclared && !this._calibrationProvisional && !fromRun) { @@ -14021,10 +13994,6 @@ export class AutobiographicalStrategy implements ResettableStrategy { this._declaredCandidate = null; if (Math.abs(observed / candidate - 1) <= OUT_OF_BAND_RUN_AGREEMENT) replacement = (candidate + observed) / 2; } else if (declared) { - // A declared sample that decides the multiplier another way (a - // provisional replacement, a run) or meets a marked record ends any - // held sample: what it was held for, whether two declared samples - // should replace an unmarked multiplier, has been settled. this._declaredCandidate = null; } @@ -14033,9 +14002,7 @@ export class AutobiographicalStrategy implements ResettableStrategy { // A provisional multiplier is a guess (see CALIBRATION_PRICING_EPOCH). This // sample is the first evidence under this release's pricing, so it replaces // the guess outright; the EMA steps from here on. A consistent run replaces - // it outright too: its samples agree on the multiplier, and stepping toward - // it would take several more applied changes, each a reprice of every - // message and summary. + // it outright too. const provisional = this._calibrationProvisional; this._calibrationProvisional = null; const next = replacement ?? (provisional || fromRun ? observed : ema + alpha * (observed - ema)); @@ -14247,19 +14214,9 @@ export class AutobiographicalStrategy implements ResettableStrategy { private _calibrationEma: number | null = null; /** The out-of-band samples rejected in a row that agree with the first, * and the first one's implied multiplier (see reportRealInputTokens). In - * memory only: a restart, or a branch's re-initialisation - * (clearBranchMirrors), counts afresh. */ + * memory only: a restart counts afresh. */ private _outOfBandRun: { first: number; count: number } | null = null; - /** The implied multiplier of the first declared sample on a record whose - * multiplier wasn't learned from declared samples, held for the next - * declared sample (see reportRealInputTokens). In memory only. */ private _declaredCandidate: number | null = null; - /** Whether the two-sample move has run for the multiplier in force: set - * once declared samples have set, stepped or confirmed it (one sample for a - * provisional multiplier or a run, two for a learned one). Persisted as the - * record's declaredOverhead; an undeclared sample that moves the multiplier - * clears it, and so does dropping a provisional multiplier to 1 (see - * reportRealInputTokens). */ private _calibrationDeclared = false; private _calibrationLoaded = false; /** Set when an older-epoch multiplier was discarded; persisted with the record. */ @@ -14469,6 +14426,7 @@ export class AutobiographicalStrategy implements ResettableStrategy { try { const saved = this.store?.getStateJson(this.calibrationStateId) as { multiplier?: number; pricing?: number; reset?: CalibrationReset; provisional?: boolean; declaredOverhead?: boolean } | null; + const fromNewerEpoch = typeof saved?.pricing === 'number' && saved.pricing > AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH; if (saved && Number.isFinite(saved.multiplier)) { const current = AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH; const stamped = typeof saved.pricing === 'number' && Number.isFinite(saved.pricing); @@ -14584,24 +14542,10 @@ export class AutobiographicalStrategy implements ResettableStrategy { // This binary's learner still writes its own record over it on its // first applied sample, or when two declared samples confirm and // mark the 1 (see CALIBRATION_PRICING_EPOCH). - } else if (!(typeof saved?.pricing === 'number' && saved.pricing > AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH)) { - // No record, or none with a multiplier to use: nothing this store - // kept says what its multiplier is, so its 1 is a guess, as a - // discard's restart is, and the first sample the band accepts sets - // it (#328). Learned, it would take the slowest path for the store - // with the least evidence: for an undeclared caller nine applied - // EMA steps to reach a true 1.4, each a reprice of every message and - // summary, and for a declared one two samples, the first held. - // Nothing is - // written: with no record, the next load decides the same, and the - // first sample's write gives the stamp. A record from a newer epoch, - // with a multiplier this binary can read or not, and a read that - // throws (below) aren't this case: each may hold what a later binary - // or the store keeps, which a provisional 1's first sample would - // write over even when it only confirms the 1. + } else if (!fromNewerEpoch) { this._calibrationProvisional = { fromEpoch: null }; } - } catch { /* a record the store can't read loads 1, not provisional: unlike no record, it may hold what the store keeps (see the no-record branch above) */ } + } catch {} } // A dry select changes the store's multiplier only where select() read it // first and can put it back (see MessageStoreView). @@ -14629,8 +14573,6 @@ export class AutobiographicalStrategy implements ResettableStrategy { const reset: CalibrationReset = { discardedMultiplier: discarded, fromEpoch: provisional.fromEpoch, at: Date.now() }; this._calibration = 1; this._calibrationReset = reset; - // The mark was about the multiplier dropped, and goes with it, as the - // record below is written without it. this._calibrationDeclared = false; this.applyCalibration(); if (dryRun) return true; diff --git a/test/calibration-band.test.ts b/test/calibration-band.test.ts index 0325ef91..21e9740b 100644 --- a/test/calibration-band.test.ts +++ b/test/calibration-band.test.ts @@ -95,8 +95,6 @@ async function openManager(name: string, opts: { signedThinking?: boolean; stamp describe('estimator calibration band', () => { it('a multiplier pinned at the ceiling comes back down once the raw estimate is honest', async () => { - // A learned 1.8: a store with no record would be provisional, and set - // outright by the first sample (#328). const first = await openManager('pinned'); await first.cm.compile(BUDGET); first.internals.store!.setStateJson(first.internals.calibrationStateId, @@ -104,7 +102,7 @@ describe('estimator calibration band', () => { first.cm.close(); const { cm, strategy, internals } = await openManager('pinned'); await cm.compile(BUDGET); // arms one sample; estimate is in calibrated units (raw × 1.8) - assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1.8, null], 'learned'); + assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1.8, null], 'premise: the seeded record loads as learned'); const est = internals._lastCompileEstimate; assert.ok(est > 0); // The provider bills exactly the raw estimate: real/est = 1/1.8 ≈ 0.56. @@ -125,7 +123,6 @@ describe('estimator calibration band', () => { }); it('an in-band ratio is learned from as before', async () => { - // A learned 1: a store with no record would be set outright (#328). const first = await openManager('inband'); await first.cm.compile(BUDGET); first.internals.store!.setStateJson(first.internals.calibrationStateId, @@ -133,7 +130,7 @@ describe('estimator calibration band', () => { first.cm.close(); const { cm, strategy, internals } = await openManager('inband'); await cm.compile(BUDGET); - assert.equal(internals._calibrationProvisional, null, 'learned'); + assert.equal(internals._calibrationProvisional, null, 'premise: the seeded record loads as learned'); const est = internals._lastCompileEstimate; strategy.reportRealInputTokens(est * 1.5); assert.ok(internals._calibration > 1.0 && internals._calibration < 1.5, `got ${internals._calibration}`); @@ -225,11 +222,10 @@ describe('estimator calibration band', () => { const { cm, strategy, internals } = await openManager('newer-epoch-unreadable'); await cm.compile(BUDGET); assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1, null]); - // A first sample that only confirms the 1 writes nothing over it. strategy.reportRealInputTokens(internals._lastCompileEstimate * 1.01); const ema = (internals as unknown as { _calibrationEma: number | null })._calibrationEma; assert.ok(ema !== null && Math.abs(ema - 1.002) < 0.001, `premise: the sample reached the EMA, got ${ema}`); - assert.deepStrictEqual(internals.store!.getStateJson(internals.calibrationStateId), future); + assert.deepStrictEqual(internals.store!.getStateJson(internals.calibrationStateId), future, 'a first sample that only confirms the 1 writes nothing over it'); cm.close(); }); @@ -439,16 +435,13 @@ describe('estimator calibration band', () => { }); it('a store with no calibration record loads a provisional 1: the first sample the band accepts sets it, and stamps the record', async () => { - // The store with the least evidence: learned, its 1 would step a fifth of - // the way per sample, nine applied changes to reach a true 1.4 (#328). const epoch = AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH; const { cm, strategy, internals } = await openManager('fresh-provisional'); await cm.compile(BUDGET); assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1, { fromEpoch: null }]); assert.equal(internals.store!.getStateJson(internals.calibrationStateId), null, 'the load writes nothing'); - // A sample the band rejects is no evidence: the 1 stays provisional. strategy.reportRealInputTokens(internals._lastCompileEstimate * 3); - assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1, { fromEpoch: null }]); + assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1, { fromEpoch: null }], 'a sample the band rejects leaves it provisional'); await cm.compile(BUDGET); strategy.reportRealInputTokens(internals._lastCompileEstimate * 1.4); assert.ok(Math.abs(internals._calibration - 1.4) < 1e-9, `set by the sample, got ${internals._calibration}`); @@ -456,9 +449,8 @@ describe('estimator calibration band', () => { const saved = internals.store!.getStateJson(internals.calibrationStateId) as { multiplier?: number; pricing?: number; provisional?: boolean; reset?: unknown }; assert.deepStrictEqual([saved.multiplier, saved.pricing, saved.provisional, saved.reset], [1.4, epoch, undefined, undefined]); - // From here the EMA steps as it always did. await cm.compile(BUDGET); - strategy.reportRealInputTokens(internals._lastCompileEstimate * 1.2); // implies 1.68 + strategy.reportRealInputTokens(internals._lastCompileEstimate * 1.2); assert.ok(internals._calibration > 1.4 && internals._calibration < 1.6, `stepped, not replaced: ${internals._calibration}`); cm.close(); }); @@ -748,11 +740,9 @@ describe('estimator calibration band', () => { belowReopened.cm.close(); }); - it('dropping a provisional multiplier to 1 drops its declared mark in memory as on disk', async () => { - // An operator following the unwedge advice may seed provisional over a - // record a declared caller had marked. + it('dropping a provisional multiplier to 1 drops its declared mark in memory as on disk, for a seeded record that carries both', async () => { const epoch = AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH; - const tight = await tightBudget('drop-declared-probe', 1.2); // fits at 1.0, not at 1.45 + const tight = await tightBudget('drop-declared-probe', 1.2); const first = await openManager('drop-declared'); await first.cm.compile(BUDGET); first.internals.store!.setStateJson(first.internals.calibrationStateId, @@ -760,9 +750,8 @@ describe('estimator calibration band', () => { first.cm.close(); const { cm, strategy, internals } = await openManager('drop-declared'); const declared = () => (internals as unknown as { _calibrationDeclared: boolean })._calibrationDeclared; - await cm.compile(BUDGET); // loads the seed, which fits this budget + await cm.compile(BUDGET); assert.deepStrictEqual([internals._calibration, declared()], [1.45, true], 'premise: loaded, marked'); - // A dry compile drops it too, in the dry compile only: select() puts the mark back. const dry = await calibrationWarnings(() => cm.compile(tight, undefined, { dryRun: true })); assert.equal(dry.result.renderStats!.calibration!.multiplier, 1, 'premise: the dry compile dropped it'); assert.deepStrictEqual([internals._calibration, declared()], [1.45, true], 'put back after the dry compile that dropped it'); @@ -772,10 +761,9 @@ describe('estimator calibration band', () => { assert.equal(declared(), false); const dropped = internals.store!.getStateJson(internals.calibrationStateId) as { multiplier?: number; provisional?: boolean; declaredOverhead?: boolean }; assert.deepStrictEqual([dropped.multiplier, dropped.provisional, dropped.declaredOverhead], [1, true, undefined], 'the record the drop writes'); - // An undeclared sample confirming the 1 writes no mark either. strategy.reportRealInputTokens(internals._lastCompileEstimate * 1.01); const saved = internals.store!.getStateJson(internals.calibrationStateId) as { multiplier?: number; declaredOverhead?: boolean }; - assert.deepStrictEqual([saved.multiplier, saved.declaredOverhead], [1, undefined]); + assert.deepStrictEqual([saved.multiplier, saved.declaredOverhead], [1, undefined], 'an undeclared sample confirming the 1 writes no mark either'); cm.close(); }); @@ -1308,7 +1296,6 @@ describe('estimator calibration band', () => { }); it('a drift inside 2% is not applied; once the EMA passes 2% it is', async () => { - // A learned 1: a store with no record would be set by its first sample (#328). const first = await openManager('deadband'); await first.cm.compile(BUDGET); first.internals.store!.setStateJson(first.internals.calibrationStateId, @@ -1316,7 +1303,7 @@ describe('estimator calibration band', () => { first.cm.close(); const { cm, strategy, internals } = await openManager('deadband'); await cm.compile(BUDGET); - assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1, null], 'learned'); + assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1, null], 'premise: the seeded record loads as learned'); // real/est = 1.01 every compile: the EMA creeps toward 1.01 and never // leaves the band, so nothing is repriced between compiles. for (let i = 0; i < 12; i++) { diff --git a/test/calibration-overhead.test.ts b/test/calibration-overhead.test.ts index 9841942f..d94e8fbf 100644 --- a/test/calibration-overhead.test.ts +++ b/test/calibration-overhead.test.ts @@ -84,12 +84,11 @@ describe('estimator calibration and request overhead (#323)', () => { const overhead = overheadPrice(cm, SYSTEM, tools); assert.ok(overhead > window * 0.4 && overhead < window * 0.7, `overhead ${overhead} vs window ${window}`); // Undeclared, this sample reads real/est ≈ 1.5: in band, so it would be - // learned as a per-token rate 50% above the truth. The first sample sets - // a store with no record; the second steps it. + // learned as a per-token rate 50% above the truth. strategy.reportRealInputTokens(window + overhead, { system: SYSTEM, tools }); await cm.compile(BUDGET); strategy.reportRealInputTokens(internals._lastCompileEstimate + overhead, { system: SYSTEM, tools }); - assert.equal(internals._calibration, 1); + assert.equal(internals._calibration, 1, 'neither the first sample, which sets a store with no record, nor the second, which steps it, moves it from 1'); cm.close(); }); @@ -200,12 +199,10 @@ describe('estimator calibration and request overhead (#323)', () => { }); }); -/** The store's price of some content, at its current multiplier. */ function priceOf(cm: ContextManager, content: ContentBlock[]): number { return (cm as unknown as { messageStore: { estimateTokens(m: StoredMessage): number } }).messageStore.estimateTokens({ content } as unknown as StoredMessage); } -/** An injection at a position, of about `tokens` estimated tokens. */ function injectionOf(position: 'beforeUser' | 'afterUser' | 'system', tokens: number): ContextInjection { return { namespace: 'probe', position, content: [{ type: 'text', text: 'Retrieved knowledge. '.repeat(Math.ceil(tokens / 4)) }] }; } @@ -223,21 +220,19 @@ async function seeded(name: string, record: Record) { return { ...opened, internals: opened.internals as DeclaredInternals }; } -/** Compile, then one sample at implied multiplier k, with system and tools declared or not. */ -async function sampleAt(cm: ContextManager, strategy: AutobiographicalStrategy, internals: Internals, k: number, declared: boolean): Promise { +async function sampleAt(cm: ContextManager, strategy: AutobiographicalStrategy, internals: Internals, impliedMultiplier: number, declared: boolean): Promise { await cm.compile(BUDGET); const m = internals._calibration; const tools = toolsOf(internals._lastCompileEstimate / m / 2); const rawWindow = internals._lastCompileEstimate / m; const rawOverhead = overheadPrice(cm, SYSTEM, tools) / m; - strategy.reportRealInputTokens(k * (rawWindow + (declared ? rawOverhead : 0)), declared ? { system: SYSTEM, tools } : undefined); + strategy.reportRealInputTokens(impliedMultiplier * (rawWindow + (declared ? rawOverhead : 0)), declared ? { system: SYSTEM, tools } : undefined); } const declaredOf = (internals: Internals) => (internals.store!.getStateJson(internals.calibrationStateId) as { declaredOverhead?: boolean } | null)?.declaredOverhead; describe('estimator calibration: what the compile adds among the messages', () => { - /** Two compiles with the injection, each sampled with system, tools and the injection in the provider's count. */ async function twoSamples(name: string, injection: (window: number) => ContextInjection) { const { cm, strategy, internals } = await openManager(name); await cm.compile(BUDGET); @@ -254,8 +249,8 @@ describe('estimator calibration: what the compile adds among the messages', () = it('a message-position injection is priced into the sample: in band, it is not learned as a per-token rate', async () => { const { cm, internals, window, inj } = await twoSamples('injected-in-band', (w) => injectionOf('afterUser', w / 2)); - // Premise: priced as the window and the declared head alone, these samples read about 1.33. - assert.ok(priceOf(cm, inj.content) > window * 0.4, `injection ${priceOf(cm, inj.content)} vs window ${window}`); + assert.ok(priceOf(cm, inj.content) > window * 0.4, + `premise: priced as the window and the declared head alone, these samples would read about 1.33; injection ${priceOf(cm, inj.content)} vs window ${window}`); assert.equal(internals._calibration, 1); assert.equal(declaredOf(internals), true, 'the samples were taken'); cm.close(); @@ -315,7 +310,6 @@ describe('estimator calibration: what the compile adds among the messages', () = const tools = toolsOf(window / 2); for (let i = 0; i < 2; i++) { const result = await cm.compile(BUDGET, [injection]); - // The caller appends systemInjections to its system prompt, and declares that. const system = `${SYSTEM}\n${result.systemInjections.map((b) => (b as { text: string }).text).join('\n')}`; strategy.reportRealInputTokens(internals._lastCompileEstimate + overheadPrice(cm, system, tools), { system, tools }); } @@ -361,7 +355,6 @@ describe('estimator calibration: what the compile adds among the messages', () = const tools = toolsOf(internals._lastCompileEstimate / 2); const injection = injectionOf('afterUser', internals._lastCompileEstimate / 2); await cm.compile(BUDGET, [injection]); - // A provider at k = 1.5: the window, the head and the injection, each at the store's price. strategy.reportRealInputTokens(internals._lastCompileEstimate + overheadPrice(cm, SYSTEM, tools) + priceOf(cm, injection.content), { system: SYSTEM, tools }); assert.equal(internals._calibration, 1.5); assert.equal((internals as unknown as { _calibrationEma: number | null })._calibrationEma, 1.5, 'the sample was taken'); @@ -385,9 +378,8 @@ describe('estimator calibration: what the compile adds among the messages', () = await cm.compile(BUDGET); const window = internals._lastCompileEstimate; const system = `You are a resident. ${'Keep to your purpose. '.repeat(Math.ceil(window / 14))}`; - // Premise: left out, the samples read 1.25 to 1.7, inside the band, where they'd be learned. const share = overheadPrice(cm, system, []) / window; - assert.ok(share > 0.25 && share < 0.7, `the system prompt is ${share} of the window`); + assert.ok(share > 0.25 && share < 0.7, `premise: left out, the samples would read 1.25 to 1.7, in band; the system prompt is ${share} of the window`); for (let i = 0; i < 2; i++) { await cm.compile(BUDGET); strategy.reportRealInputTokens(internals._lastCompileEstimate + overheadPrice(cm, system, []), { system }); @@ -399,16 +391,13 @@ describe('estimator calibration: what the compile adds among the messages', () = }); describe('estimator calibration: consistent out-of-band runs', () => { - /** Compile, then one declared sample at `ratio` of the compile's estimate. */ async function oob(cm: ContextManager, strategy: AutobiographicalStrategy, internals: Internals, ratio: number, declared = true) { await cm.compile(BUDGET); - const nothingBesides: RequestOverhead = {}; // the package's type, from its root + const nothingBesides: RequestOverhead = {}; strategy.reportRealInputTokens(internals._lastCompileEstimate * ratio, declared ? nothingBesides : undefined); } it('a consistent run on a learned record without the flag is learned at the clamp, not held', async () => { - // The upgrade case: a learned multiplier no declared sample has confirmed. - // A store with no record is provisional, which skips the hold anyway. const { cm, strategy, internals } = await seeded('run-on-learned', { multiplier: 1, at: 1, pricing: epoch }); assert.equal(internals._calibrationProvisional, null, 'learned'); for (let i = 0; i < 3; i++) await oob(cm, strategy, internals, 2.1); From f95dfa0e24d57896206b5dd1dd432f96d846421d Mon Sep 17 00:00:00 2001 From: Esther-2584 Date: Sun, 25 Oct 2026 19:51:44 -0400 Subject: [PATCH 8/8] fix(calibration): answer our internal review of the comment pass - The two-declared-sample move's agreement is its own constant, DECLARED_PAIR_AGREEMENT, where it borrowed OUT_OF_BAND_RUN_AGREEMENT, so tuning the run's tolerance no longer retunes the pair without a word. Both are 0.1, as before. - markDeclared is renamed declaredOnUnmarked: it names the condition, a declared sample on a record not yet marked, and the block it guards holds the first such sample without marking. - The sanity-band comment keeps "except a declared consistent run (below)". #324's "Log, don't learn." was already contradicted by its own run code, so the clause corrects a false comment, which the comment pass keeps. - CALIBRATION_PRICING_EPOCH's doc drops the reason after "It writes nothing until its first sample"; it is in the previous commit's message. - The throwing read's test title says it loads 1, not provisional, as RenderStats.calibration's doc has it, not "a learned 1". - The drop test's last assertion also checks that the record is no longer provisional, which only the confirming sample's write does. - The first overhead test's message says what its samples do: the first confirms the provisional 1, the second is an EMA step. Behaviour is unchanged. Co-Authored-By: Claude Opus 5.5 --- src/strategies/autobiographical.ts | 21 +++++++++++---------- test/calibration-band.test.ts | 7 ++++--- test/calibration-overhead.test.ts | 2 +- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/strategies/autobiographical.ts b/src/strategies/autobiographical.ts index 13b28aa6..8ae95b0a 100644 --- a/src/strategies/autobiographical.ts +++ b/src/strategies/autobiographical.ts @@ -1860,6 +1860,7 @@ function imageGroup(block: ContentBlock): unknown { * continue the run (see reportRealInputTokens). */ const OUT_OF_BAND_RUN_LENGTH = 3; const OUT_OF_BAND_RUN_AGREEMENT = 0.1; +const DECLARED_PAIR_AGREEMENT = 0.1; /** * Autobiographical chunking strategy. @@ -13929,7 +13930,8 @@ export class AutobiographicalStrategy implements ResettableStrategy { const observed = ratio * current; // back out the multiplier already applied // SANITY BAND: a representative sample sits near 1. Anything wilder is a // structural mismatch (a request we didn't compile, a partial compile, a - // provider quirk) — never evidence about chars-per-token. Log, don't learn. + // provider quirk) — never evidence about chars-per-token. Log, don't learn, + // except a declared consistent run (below). // The band is also checked on the IMPLIED raw multiplier: when the // multiplier sits at a clamp edge because the per-class rates were wrong // (signed thinking at a flat 600) and the rates are then fixed, every @@ -13977,22 +13979,22 @@ export class AutobiographicalStrategy implements ResettableStrategy { // A multiplier learned from samples that didn't declare their overhead // holds that overhead as a rate (#323). - const markDeclared = declared && !this._calibrationDeclared; + const declaredOnUnmarked = declared && !this._calibrationDeclared; let replacement: number | null = null; - if (markDeclared && !this._calibrationProvisional && !fromRun) { + if (declaredOnUnmarked && !this._calibrationProvisional && !fromRun) { const candidate = this._declaredCandidate; if (candidate === null) { this._declaredCandidate = observed; this.diag.error( `[estimator-calibration] real/est=${ratio.toFixed(2)} implies ${observed.toFixed(2)}: the first sample ` + `with its overhead declared, held for the next; if they agree within ` + - `${OUT_OF_BAND_RUN_AGREEMENT * 100}% their mean replaces the multiplier, and if not it steps; ` + + `${DECLARED_PAIR_AGREEMENT * 100}% their mean replaces the multiplier, and if not it steps; ` + `multiplier stays ${current.toFixed(2)} ${counts}`, ); return; } this._declaredCandidate = null; - if (Math.abs(observed / candidate - 1) <= OUT_OF_BAND_RUN_AGREEMENT) replacement = (candidate + observed) / 2; + if (Math.abs(observed / candidate - 1) <= DECLARED_PAIR_AGREEMENT) replacement = (candidate + observed) / 2; } else if (declared) { this._declaredCandidate = null; } @@ -14026,7 +14028,7 @@ export class AutobiographicalStrategy implements ResettableStrategy { ); if (declared) this._calibrationDeclared = true; this.persistCalibration(); - } else if (markDeclared) { + } else if (declaredOnUnmarked) { this._calibrationDeclared = true; this.persistCalibration(); } @@ -14284,10 +14286,9 @@ export class AutobiographicalStrategy implements ResettableStrategy { * sample doesn't turn the guess into a learned value, and a later epoch's * load re-stamps it, still provisional, rather than discarding it again * over the reset it carries. A store with no record starts from a - * provisional 1 too (#328). It writes nothing until its first sample: with - * no record, each load decides the same. A dry run under a - * provisional multiplier above 1 takes the same fallback at the picker's - * refusal, which it otherwise only reports. + * provisional 1 too (#328). It writes nothing until its first sample. A dry + * run under a provisional multiplier above 1 takes the same fallback at the + * picker's refusal, which it otherwise only reports. * * A binary from before a bump still learns after a rollback, and its first * applied sample, or two declared samples that confirm and mark its 1, diff --git a/test/calibration-band.test.ts b/test/calibration-band.test.ts index 21e9740b..38d4e6b9 100644 --- a/test/calibration-band.test.ts +++ b/test/calibration-band.test.ts @@ -493,7 +493,7 @@ describe('estimator calibration band', () => { } }); - it('a calibration read that throws loads a learned 1, as before: the record it could not read may hold one worth keeping', async () => { + it('a calibration read that throws loads 1, not provisional, as before: the record it could not read may hold one worth keeping', async () => { const { cm, internals } = await openManager('fresh-unreadable'); const store = internals.store as unknown as { getStateJson(id: string): unknown }; const read = store.getStateJson; @@ -762,8 +762,9 @@ describe('estimator calibration band', () => { const dropped = internals.store!.getStateJson(internals.calibrationStateId) as { multiplier?: number; provisional?: boolean; declaredOverhead?: boolean }; assert.deepStrictEqual([dropped.multiplier, dropped.provisional, dropped.declaredOverhead], [1, true, undefined], 'the record the drop writes'); strategy.reportRealInputTokens(internals._lastCompileEstimate * 1.01); - const saved = internals.store!.getStateJson(internals.calibrationStateId) as { multiplier?: number; declaredOverhead?: boolean }; - assert.deepStrictEqual([saved.multiplier, saved.declaredOverhead], [1, undefined], 'an undeclared sample confirming the 1 writes no mark either'); + const saved = internals.store!.getStateJson(internals.calibrationStateId) as { multiplier?: number; provisional?: boolean; declaredOverhead?: boolean }; + assert.deepStrictEqual([saved.multiplier, saved.provisional, saved.declaredOverhead], [1, undefined, undefined], + 'an undeclared sample confirming the 1 writes the record, with no mark either'); cm.close(); }); diff --git a/test/calibration-overhead.test.ts b/test/calibration-overhead.test.ts index d94e8fbf..98149747 100644 --- a/test/calibration-overhead.test.ts +++ b/test/calibration-overhead.test.ts @@ -88,7 +88,7 @@ describe('estimator calibration and request overhead (#323)', () => { strategy.reportRealInputTokens(window + overhead, { system: SYSTEM, tools }); await cm.compile(BUDGET); strategy.reportRealInputTokens(internals._lastCompileEstimate + overhead, { system: SYSTEM, tools }); - assert.equal(internals._calibration, 1, 'neither the first sample, which sets a store with no record, nor the second, which steps it, moves it from 1'); + assert.equal(internals._calibration, 1, 'neither sample moves it from 1: the first confirms the provisional 1 of a store with no record, and the second is an EMA step'); cm.close(); });