diff --git a/changelog.d/323-calibration-request-overhead.fixed.md b/changelog.d/323-calibration-request-overhead.fixed.md index 7c06e8fb..68050e01 100644 --- a/changelog.d/323-calibration-request-overhead.fixed.md +++ b/changelog.d/323-calibration-request-overhead.fixed.md @@ -5,16 +5,34 @@ estimate doesn't, so a tool-heavy agent's multiplier learned its schemas as a per-token rate and folded history the budget had room for, or sat at 1.00 rejecting every sample as out of band. A caller that passes nothing - gets the old comparison. + gets the old comparison, plus the compile's message-position injections + (below). - With the overhead declared, three out-of-band calibration samples in a row - whose implied multipliers agree within 10% are learned, outright, at the - clamp edge, instead of being rejected for good. At 1.00 the ratio and the - implied multiplier are the same number, so a store whose true ratio was - past 1.8, where every pricing-epoch discard restarts, could never learn. - Undeclared samples are rejected as before, and the rejection line says the - caller may be leaving out the system prompt and tools it sends. + whose implied multipliers agree within 10% of the first are learned, + outright, at the clamp edge, instead of being rejected for good. At 1.00 + the ratio and the implied multiplier are the same number, so a store whose + true ratio was past 1.8, where every pricing-epoch discard restarts, could + never learn. Undeclared samples are rejected as before, and the rejection + line says the caller may be leaving out the system prompt and tools it + sends. +- The sample also counts the compile's message-position injections + (`beforeUser` and `afterUser`), which the request carries and the window + estimate doesn't: `ContextManager.compile` passes their content to the + select as `SelectOptions.messageInjections`, and the select prices it with + the window. Left out, an agent whose injections are large beside its window + learned them as a per-token rate, or, with the overhead declared, at the + clamp. - **Host operators:** a multiplier learned before the caller declared its - overhead holds that overhead, and steps back down over about ten compiles - once samples declare it. To relearn in one sample, seed it as provisional - on the stopped store: `setStateJson('/autobio:calibration', - { multiplier, at, pricing, provisional: true })`. + overhead holds that overhead. Inside the calibration band, the first two + samples that declare it replace the multiplier with their mean when they + agree within 10%, and the EMA steps as before when they don't; either way + the record then carries `declaredOverhead: true`, so later declared + samples step by the EMA. (Out of the band, the three-sample run above + applies.) The first of the two is held in memory, so a restart between + them costs one more sample. An undeclared sample that moves the multiplier + writes the record without the flag. A store whose multiplier, above 1, + puts every compile over budget gets no sample to move it, as before: to + unwedge it, seed the record as provisional on the stopped store, and the + refused compile drops it to 1: `setStateJson('/autobio:calibration', + { multiplier, at, pricing: , provisional: true })`. + `RequestOverhead` is exported from the package. diff --git a/changelog.d/328-fresh-calibration-provisional.fixed.md b/changelog.d/328-fresh-calibration-provisional.fixed.md new file mode 100644 index 00000000..55485a4b --- /dev/null +++ b/changelog.d/328-fresh-calibration-provisional.fixed.md @@ -0,0 +1,10 @@ +- A store with no calibration record, or a record with no finite + multiplier, starts from a provisional 1, as a discard's restart does + (#328). Its first sample inside the calibration band sets the multiplier, + declared or not: one sample, without the two-sample hold. Learned, a 1 + stepped a fifth of the way per undeclared sample (nine applied changes, + each a reprice of every message and summary, to reach a true 1.4), and + took two declared samples, the first held. Until that + sample, render stats report `calibration.provisional: true`. A record + from a newer pricing epoch, or a read of the record that throws, still + loads 1 as before, not provisional. diff --git a/src/context-manager.ts b/src/context-manager.ts index ea298d8c..5a288dee 100644 --- a/src/context-manager.ts +++ b/src/context-manager.ts @@ -1179,11 +1179,14 @@ export class ContextManager { const view = this.strategyMessageView(); // Get selected entries from strategy + const messageInjections = (injections ?? []) + .filter((inj) => inj.position === 'beforeUser' || inj.position === 'afterUser') + .map((inj) => inj.content); const entries = this.strategy.select( view, this.contextLog.createView(), effectiveBudget, - opts + messageInjections.length > 0 ? { ...opts, messageInjections } : opts ); const selectionCause = this.strategy.takeSelectionCause?.(); // Read now, before anything else selects: these are this select's own. diff --git a/src/index.ts b/src/index.ts index af88b229..82687b0b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -81,6 +81,7 @@ export type { RenderedLayout, LayoutUnit, // Strategy types + RequestOverhead, MessageStoreView, ContextLogView, StrategyContext, diff --git a/src/strategies/autobiographical.ts b/src/strategies/autobiographical.ts index 615138a7..8ae95b0a 100644 --- a/src/strategies/autobiographical.ts +++ b/src/strategies/autobiographical.ts @@ -1856,10 +1856,11 @@ function imageGroup(block: ContentBlock): unknown { } /** Out-of-band calibration samples in a row it takes to learn from them, and - * how close each one's implied multiplier must be to the one before it to + * how close each one's implied multiplier must be to the run's first to * continue the run (see reportRealInputTokens). */ const OUT_OF_BAND_RUN_LENGTH = 3; const OUT_OF_BAND_RUN_AGREEMENT = 0.1; +const DECLARED_PAIR_AGREEMENT = 0.1; /** * Autobiographical chunking strategy. @@ -2852,6 +2853,7 @@ export class AutobiographicalStrategy implements ResettableStrategy { this._lastRenderStats = null; this._lastSelectStats = null; this._lastCompileEstimate = 0; + this._lastCompileInjected = 0; this._storeView = null; // Prices memoized over the view just cleared: the next one may estimate otherwise. this._answerCostCache.clear(); @@ -2861,7 +2863,9 @@ export class AutobiographicalStrategy implements ResettableStrategy { this._calibrationLoaded = false; this._calibrationReset = null; this._calibrationProvisional = null; + this._calibrationDeclared = false; this._outOfBandRun = null; + this._declaredCandidate = null; this._imageStripReplayMemo = null; this._lastKvStable = null; this._storeOrder = new Map(); @@ -13862,23 +13866,31 @@ export class AutobiographicalStrategy implements ResettableStrategy { this._lastCompileEstimate = rs.head.tokens + rs.tail.tokens + rs.middleRaw.tokens + rs.summaries.l1.tokens + rs.summaries.l2.tokens + rs.summaries.l3.tokens; + let injected = 0; + for (const content of opts?.messageInjections ?? []) { + injected += store.estimateTokens({ content } as unknown as StoredMessage); + } + this._lastCompileInjected = injected; this._calibrationArmed = true; // exactly one sample per compile } return merged; } private _lastCompileEstimate = 0; + private _lastCompileInjected = 0; private _storeView: MessageStoreView | null = null; /** * Closed-loop estimator calibration (2026-07-12). Feed the REAL input total * for a request built from the latest compile (fresh + cache_read + * cache_creation, as the provider reports it), and what the request sent - * besides the window: its system prompt and tool definitions (#323). Those - * are priced as the store prices the window and added to the estimate, so - * the sample compares the whole request with the whole request; left out, - * the multiplier learns them as a per-token rate, which no value of it can - * be (see RequestOverhead). Maintains an EMA of real/estimated and applies + * besides the window and the compile's message-position injections: its + * system prompt and tool definitions (#323). The injections are priced at + * the compile (SelectOptions.messageInjections) and the overhead here, as + * the store prices the window, and both are added to the estimate, so the + * sample compares the whole request with the whole request; left out, the + * multiplier learns them as a per-token rate, which no value of it can be + * (see RequestOverhead). Maintains an EMA of real/estimated and applies * it as the store's global multiplier, persisted across restarts. The * per-class rates carry the shape; this carries the residual level. */ @@ -13898,21 +13910,28 @@ export class AutobiographicalStrategy implements ResettableStrategy { // exhausting the picker on every wake. One sample per compile, always. if (!this._calibrationArmed) return; this._calibrationArmed = false; + const declared = overhead != null; // The overhead in the window's units (current calibrated), so the implied // multiplier below is real / raw(window + overhead) and doesn't depend on // the multiplier it's measured at: one sample states it. - const overheadTokens = overhead ? this.priceRequestOverhead(overhead) : 0; - const est = window + overheadTokens; + const overheadTokens = declared ? this.priceRequestOverhead(overhead!) : 0; + const injected = this._lastCompileInjected; + const est = window + injected + overheadTokens; + const parts = [ + ...(overheadTokens > 0 ? [`${Math.round(overheadTokens / 1000)}k system+tools`] : []), + ...(injected > 0 ? [`${Math.round(injected / 1000)}k injected`] : []), + ]; const counts = `(est=${Math.round(est / 1000)}k` + - `${overheadTokens > 0 ? ` incl. ${Math.round(overheadTokens / 1000)}k system+tools` : ''}` + + `${parts.length > 0 ? ` incl. ${parts.join(', ')}` : ''}` + ` real=${Math.round(realTotal / 1000)}k)`; const ratio = realTotal / est; const current = this._calibration; const observed = ratio * current; // back out the multiplier already applied // SANITY BAND: a representative sample sits near 1. Anything wilder is a // structural mismatch (a request we didn't compile, a partial compile, a - // provider quirk) — never evidence about chars-per-token. Log, don't learn. + // provider quirk) — never evidence about chars-per-token. Log, don't learn, + // except a declared consistent run (below). // The band is also checked on the IMPLIED raw multiplier: when the // multiplier sits at a clamp edge because the per-class rates were wrong // (signed thinking at a flat 600) and the rates are then fixed, every @@ -13926,8 +13945,8 @@ export class AutobiographicalStrategy implements ResettableStrategy { // the same number, so a store whose true ratio is past the band (every // epoch discard restarts there) rejects every sample and never learns. // A single wild sample is still noise, but a run of them that agree with - // each other is a ruler that's off, not a request we didn't compile: the - // third in a row is learned, outright, which lands it on the clamp edge. + // the run's first is a ruler that's off, not a request we didn't compile: + // the third in a row is learned, outright, which lands it on the clamp edge. // Only for samples whose caller declared what it sent besides the window: // undeclared, a consistent ratio past the band is what tool schemas and a // system prompt look like, and learning it would fold the window to the @@ -13935,19 +13954,20 @@ export class AutobiographicalStrategy implements ResettableStrategy { let fromRun = false; if (!ratioInBand && !observedInBand) { const run = this._outOfBandRun; - this._outOfBandRun = !overhead ? null - : run && Math.abs(observed / run.observed - 1) <= OUT_OF_BAND_RUN_AGREEMENT - ? { observed, count: run.count + 1 } - : { observed, count: 1 }; + this._outOfBandRun = !declared ? null + : run && Math.abs(observed / run.first - 1) <= OUT_OF_BAND_RUN_AGREEMENT + ? { first: run.first, count: run.count + 1 } + : { first: observed, count: 1 }; const count = this._outOfBandRun?.count ?? 0; if (count < OUT_OF_BAND_RUN_LENGTH) { this.diag.error( `[estimator-calibration] REJECTED out-of-band sample real/est=${ratio.toFixed(2)} ${counts} — ` + - `not a window-shaped request; multiplier stays ${this._calibration.toFixed(2)}` + - (overhead - ? ` (${count} of ${OUT_OF_BAND_RUN_LENGTH} consistent samples to learn from)` - : `; if every sample reads about the same, the caller may be leaving out the system prompt ` + - `and tools it sends (reportRealInputTokens' second argument)`), + (declared + ? `multiplier stays ${this._calibration.toFixed(2)} until ${OUT_OF_BAND_RUN_LENGTH} samples in a row ` + + `agree within ${OUT_OF_BAND_RUN_AGREEMENT * 100}% (${count} of ${OUT_OF_BAND_RUN_LENGTH})` + : `not a window-shaped request; multiplier stays ${this._calibration.toFixed(2)}; if every sample ` + + `reads about the same, the caller may be leaving out the system prompt and tools it sends ` + + `(reportRealInputTokens' second argument)`), ); return; } @@ -13957,16 +13977,37 @@ export class AutobiographicalStrategy implements ResettableStrategy { this._outOfBandRun = null; } + // A multiplier learned from samples that didn't declare their overhead + // holds that overhead as a rate (#323). + const declaredOnUnmarked = declared && !this._calibrationDeclared; + let replacement: number | null = null; + if (declaredOnUnmarked && !this._calibrationProvisional && !fromRun) { + const candidate = this._declaredCandidate; + if (candidate === null) { + this._declaredCandidate = observed; + this.diag.error( + `[estimator-calibration] real/est=${ratio.toFixed(2)} implies ${observed.toFixed(2)}: the first sample ` + + `with its overhead declared, held for the next; if they agree within ` + + `${DECLARED_PAIR_AGREEMENT * 100}% their mean replaces the multiplier, and if not it steps; ` + + `multiplier stays ${current.toFixed(2)} ${counts}`, + ); + return; + } + this._declaredCandidate = null; + if (Math.abs(observed / candidate - 1) <= DECLARED_PAIR_AGREEMENT) replacement = (candidate + observed) / 2; + } else if (declared) { + this._declaredCandidate = null; + } + const alpha = 0.2; // slow EMA: one wild request shouldn't yank the ruler const ema = this._calibrationEma ?? current; // A provisional multiplier is a guess (see CALIBRATION_PRICING_EPOCH). This // sample is the first evidence under this release's pricing, so it replaces // the guess outright; the EMA steps from here on. A consistent run replaces - // it outright too: an EMA step from 1.00 toward a ratio past the band - // would leave the next sample just as far out of band. + // it outright too. const provisional = this._calibrationProvisional; this._calibrationProvisional = null; - const next = provisional || fromRun ? observed : ema + alpha * (observed - ema); + const next = replacement ?? (provisional || fromRun ? observed : ema + alpha * (observed - ema)); const clamped = Math.min(1.8, Math.max(0.6, next)); this._calibrationEma = clamped; // DEAD BAND. The multiplier is applied only once the EMA has moved more @@ -13985,6 +14026,10 @@ export class AutobiographicalStrategy implements ResettableStrategy { `[estimator-calibration] real/est=${ratio.toFixed(2)} confirms the provisional multiplier ` + `${current.toFixed(2)} ${counts}; stamped`, ); + if (declared) this._calibrationDeclared = true; + this.persistCalibration(); + } else if (declaredOnUnmarked) { + this._calibrationDeclared = true; this.persistCalibration(); } return; @@ -13992,10 +14037,11 @@ export class AutobiographicalStrategy implements ResettableStrategy { this.diag.error( `[estimator-calibration] real/est=${ratio.toFixed(2)} ` + `multiplier ${current.toFixed(2)} -> ${clamped.toFixed(2)}` + - `${fromRun ? ` (${OUT_OF_BAND_RUN_LENGTH} consistent out-of-band samples: learned at the clamp)` : provisional ? ' (provisional: replaced, not stepped)' : ''} ` + + `${fromRun ? ` (${OUT_OF_BAND_RUN_LENGTH} consistent out-of-band samples: learned at the clamp)` : provisional ? ' (provisional: replaced, not stepped)' : replacement !== null ? ' (two samples with their overhead declared agree: replaced by their mean, not stepped)' : ''} ` + counts, ); this._calibration = clamped; + this._calibrationDeclared = declared; this.applyCalibration(); this.persistCalibration(); } @@ -14024,6 +14070,7 @@ export class AutobiographicalStrategy implements ResettableStrategy { at: Date.now(), pricing: AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH, ...(this._calibrationReset ? { reset: this._calibrationReset } : {}), + ...(this._calibrationDeclared ? { declaredOverhead: true } : {}), }); } catch { /* persistence is best-effort */ } } @@ -14167,17 +14214,19 @@ export class AutobiographicalStrategy implements ResettableStrategy { private _calibration = 1; /** The EMA itself; `_calibration` is its last applied value (see the dead band). */ private _calibrationEma: number | null = null; - /** The out-of-band samples rejected in a row that agree with each other, - * and the last one's implied multiplier (see reportRealInputTokens). In + /** The out-of-band samples rejected in a row that agree with the first, + * and the first one's implied multiplier (see reportRealInputTokens). In * memory only: a restart counts afresh. */ - private _outOfBandRun: { observed: number; count: number } | null = null; + private _outOfBandRun: { first: number; count: number } | null = null; + private _declaredCandidate: number | null = null; + private _calibrationDeclared = false; private _calibrationLoaded = false; /** Set when an older-epoch multiplier was discarded; persisted with the record. */ private _calibrationReset: CalibrationReset | null = null; /** Set while the multiplier is provisional (see CALIBRATION_PRICING_EPOCH): * the first sample the band accepts replaces it outright, and one above 1 * that puts a compile over budget is dropped to 1. `fromEpoch` is the - * record's it came from, null for one without a stamp. */ + * record's it came from, null for one without a stamp or for no record. */ private _calibrationProvisional: { fromEpoch: number | null } | null = null; protected applyCalibration(): void { @@ -14236,12 +14285,14 @@ export class AutobiographicalStrategy implements ResettableStrategy { * is persisted as such (`provisional: true`), so a restart before that * sample doesn't turn the guess into a learned value, and a later epoch's * load re-stamps it, still provisional, rather than discarding it again - * over the reset it carries. A dry run under a - * provisional multiplier above 1 takes the same fallback at the picker's - * refusal, which it otherwise only reports. + * over the reset it carries. A store with no record starts from a + * provisional 1 too (#328). It writes nothing until its first sample. A dry + * run under a provisional multiplier above 1 takes the same fallback at the + * picker's refusal, which it otherwise only reports. * * A binary from before a bump still learns after a rollback, and its first - * applied sample overwrites the newer epoch's record with its own stamp + * applied sample, or two declared samples that confirm and mark its 1, + * overwrite the newer epoch's record with its own stamp * (reportRealInputTokens); rolling forward again treats that as an older * record. */ @@ -14372,9 +14423,11 @@ export class AutobiographicalStrategy implements ResettableStrategy { this._calibration = 1; this._calibrationReset = null; this._calibrationProvisional = null; + this._calibrationDeclared = false; try { const saved = this.store?.getStateJson(this.calibrationStateId) as - { multiplier?: number; pricing?: number; reset?: CalibrationReset; provisional?: boolean } | null; + { multiplier?: number; pricing?: number; reset?: CalibrationReset; provisional?: boolean; declaredOverhead?: boolean } | null; + const fromNewerEpoch = typeof saved?.pricing === 'number' && saved.pricing > AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH; if (saved && Number.isFinite(saved.multiplier)) { const current = AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH; const stamped = typeof saved.pricing === 'number' && Number.isFinite(saved.pricing); @@ -14389,6 +14442,7 @@ export class AutobiographicalStrategy implements ResettableStrategy { if (epoch === current) { this._calibration = Math.min(1.8, Math.max(0.6, saved.multiplier!)); this._calibrationReset = isCalibrationReset(saved.reset) ? saved.reset : null; + this._calibrationDeclared = saved.declaredOverhead === true; // A discard's 1 that no sample has confirmed yet was persisted as // provisional, and stays a guess across the restart: the first // sample still sets it, instead of stepping toward it. @@ -14409,6 +14463,7 @@ export class AutobiographicalStrategy implements ResettableStrategy { // still a guess. this._calibration = Math.min(1.8, Math.max(0.6, saved.multiplier!)); this._calibrationReset = isCalibrationReset(saved.reset) ? saved.reset : null; + this._calibrationDeclared = saved.declaredOverhead === true; if (saved.provisional === true) { this._calibrationProvisional = { fromEpoch: this._calibrationReset?.fromEpoch ?? null }; } @@ -14429,6 +14484,7 @@ export class AutobiographicalStrategy implements ResettableStrategy { pricing: current, ...(this._calibrationReset ? { reset: this._calibrationReset } : {}), ...(this._calibrationProvisional ? { provisional: true } : {}), + ...(this._calibrationDeclared ? { declaredOverhead: true } : {}), }); } catch { /* persistence is best-effort */ } } @@ -14485,9 +14541,12 @@ export class AutobiographicalStrategy implements ResettableStrategy { // leaves it on disk and doesn't use it, and logs nothing, since // re-stamping it as the discard warning advises would destroy it. // This binary's learner still writes its own record over it on its - // first applied sample (see CALIBRATION_PRICING_EPOCH). + // first applied sample, or when two declared samples confirm and + // mark the 1 (see CALIBRATION_PRICING_EPOCH). + } else if (!fromNewerEpoch) { + this._calibrationProvisional = { fromEpoch: null }; } - } catch { /* absent slot is fine */ } + } catch {} } // A dry select changes the store's multiplier only where select() read it // first and can put it back (see MessageStoreView). @@ -14515,6 +14574,7 @@ export class AutobiographicalStrategy implements ResettableStrategy { const reset: CalibrationReset = { discardedMultiplier: discarded, fromEpoch: provisional.fromEpoch, at: Date.now() }; this._calibration = 1; this._calibrationReset = reset; + this._calibrationDeclared = false; this.applyCalibration(); if (dryRun) return true; this.diag.warn( @@ -19286,11 +19346,13 @@ export class AutobiographicalStrategy implements ResettableStrategy { transitionBlocked: this.transitionBlocked, lastRenderStats: this._lastRenderStats, lastCompileEstimate: this._lastCompileEstimate, + lastCompileInjected: this._lastCompileInjected, calibrationArmed: this._calibrationArmed, calibrationLoaded: this._calibrationLoaded, calibration: this._calibration, calibrationReset: this._calibrationReset, calibrationProvisional: this._calibrationProvisional, + calibrationDeclared: this._calibrationDeclared, storeCalibration: store.getTokenCalibration?.(), storeView: this._storeView, storeOrder: this._storeOrder, @@ -19313,11 +19375,13 @@ export class AutobiographicalStrategy implements ResettableStrategy { this.transitionBlocked = saved.transitionBlocked; this._lastRenderStats = saved.lastRenderStats; this._lastCompileEstimate = saved.lastCompileEstimate; + this._lastCompileInjected = saved.lastCompileInjected; this._calibrationArmed = saved.calibrationArmed; this._calibrationLoaded = saved.calibrationLoaded; this._calibration = saved.calibration; this._calibrationReset = saved.calibrationReset; this._calibrationProvisional = saved.calibrationProvisional; + this._calibrationDeclared = saved.calibrationDeclared; if (saved.storeCalibration !== undefined) store.setTokenCalibration?.(saved.storeCalibration); this._storeView = saved.storeView; this._storeOrder = saved.storeOrder; diff --git a/src/types/strategy.ts b/src/types/strategy.ts index 45b643a8..0ac87ed1 100644 --- a/src/types/strategy.ts +++ b/src/types/strategy.ts @@ -370,6 +370,14 @@ export interface SelectOptions { multiplier: number; expiresAt: number; }; + /** The content of each injection the compile inserts among the selected + * entries after the select (`beforeUser` and `afterUser`), which the + * request carries and the select's window doesn't. A strategy whose + * calibration compares the provider's input total with the compile prices + * it there (AutobiographicalStrategy.reportRealInputTokens). + * ContextManager.compile fills it from its injections; `system` ones go + * into the caller's system prompt instead. */ + messageInjections?: ReadonlyArray; } /** @@ -667,10 +675,13 @@ export function isSummaryOverviewStrategy(s: ContextStrategy): s is SummaryOverv } /** - * What a request carried besides the compiled window, as the caller sent it - * (see `AutobiographicalStrategy.reportRealInputTokens`). The provider counts - * it in the input total and the compile estimate doesn't, so a calibration - * sample that leaves it out learns it as a per-token rate. + * What a request carried besides the compiled window and the compile's + * message-position injections (which the compile prices itself), as the + * caller sent it (see `AutobiographicalStrategy.reportRealInputTokens`). The + * provider counts it in the input total and the compile estimate doesn't, so + * a calibration sample that leaves it out learns it as a per-token rate. + * Passing it says this is everything the request sent besides those; a + * caller that can't see the request's head passes nothing. */ export interface RequestOverhead { /** The system prompt the request sent. */ @@ -731,11 +742,16 @@ export interface RenderStats { * that isn't provisional was learned under that epoch's pricing, or under * an older epoch's and kept, re-stamped, because nothing this store holds * was repriced since; either way it's reported under the epoch in force. + * The exception is the 1 loaded beside a record from a newer pricing + * epoch, or a record the store can't read: neither is used, and that 1 + * is neither learned nor provisional. * `provisional` is true while the multiplier is a guess, which the first * provider sample the calibration's band accepts replaces outright, rather * than stepping toward it (see * `AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH`): one kept from a - * record without a stamp, or the 1 a discard restarts from. That 1 stays + * record without a stamp, the 1 a discard restarts from, or the 1 a store + * with no calibration record, or none with a multiplier to use other than + * a newer epoch's, starts from. A discard's 1 stays * provisional through a later epoch's load, which re-stamps it whatever * that epoch repriced. * `reset` is present once a multiplier from an older or unknown epoch has diff --git a/test/calibration-band.test.ts b/test/calibration-band.test.ts index 0c108c8c..38d4e6b9 100644 --- a/test/calibration-band.test.ts +++ b/test/calibration-band.test.ts @@ -95,11 +95,14 @@ async function openManager(name: string, opts: { signedThinking?: boolean; stamp describe('estimator calibration band', () => { it('a multiplier pinned at the ceiling comes back down once the raw estimate is honest', async () => { + const first = await openManager('pinned'); + await first.cm.compile(BUDGET); + first.internals.store!.setStateJson(first.internals.calibrationStateId, + { multiplier: 1.8, at: 1, pricing: AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH }); + first.cm.close(); const { cm, strategy, internals } = await openManager('pinned'); - await cm.compile(BUDGET); - internals._calibration = 1.8; - internals.applyCalibration(); await cm.compile(BUDGET); // arms one sample; estimate is in calibrated units (raw × 1.8) + assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1.8, null], 'premise: the seeded record loads as learned'); const est = internals._lastCompileEstimate; assert.ok(est > 0); // The provider bills exactly the raw estimate: real/est = 1/1.8 ≈ 0.56. @@ -120,8 +123,14 @@ describe('estimator calibration band', () => { }); it('an in-band ratio is learned from as before', async () => { + const first = await openManager('inband'); + await first.cm.compile(BUDGET); + first.internals.store!.setStateJson(first.internals.calibrationStateId, + { multiplier: 1, at: 1, pricing: AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH }); + first.cm.close(); const { cm, strategy, internals } = await openManager('inband'); await cm.compile(BUDGET); + assert.equal(internals._calibrationProvisional, null, 'premise: the seeded record loads as learned'); const est = internals._lastCompileEstimate; strategy.reportRealInputTokens(est * 1.5); assert.ok(internals._calibration > 1.0 && internals._calibration < 1.5, `got ${internals._calibration}`); @@ -204,6 +213,22 @@ describe('estimator calibration band', () => { assert.deepEqual(warned.filter((l) => l.includes('[estimator-calibration]')), []); }); + it('a record from a newer pricing epoch with no multiplier this binary can read is left alone too', async () => { + const first = await openManager('newer-epoch-unreadable'); + await first.cm.compile(BUDGET); + const future = { multipliers: { default: 1.4 }, at: 123, pricing: AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH + 1 }; + first.internals.store!.setStateJson(first.internals.calibrationStateId, future); + first.cm.close(); + const { cm, strategy, internals } = await openManager('newer-epoch-unreadable'); + await cm.compile(BUDGET); + assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1, null]); + strategy.reportRealInputTokens(internals._lastCompileEstimate * 1.01); + const ema = (internals as unknown as { _calibrationEma: number | null })._calibrationEma; + assert.ok(ema !== null && Math.abs(ema - 1.002) < 0.001, `premise: the sample reached the EMA, got ${ema}`); + assert.deepStrictEqual(internals.store!.getStateJson(internals.calibrationStateId), future, 'a first sample that only confirms the 1 writes nothing over it'); + cm.close(); + }); + it('a multiplier learned under the current pricing epoch survives reopen', async () => { const first = await openManager('current-epoch'); await first.cm.compile(BUDGET); @@ -401,10 +426,89 @@ describe('estimator calibration band', () => { reopened.cm.close(); }); - it('a store with no calibration history reports its multiplier and no reset', async () => { + it('a store with no calibration history reports a provisional 1 and no reset', async () => { const { cm } = await openManager('no-history'); await cm.compile(BUDGET); - assert.deepEqual(cm.getRenderStats()!.calibration, { multiplier: 1, pricingEpoch: AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH }); + assert.deepStrictEqual(cm.getRenderStats()!.calibration, + { multiplier: 1, pricingEpoch: AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH, provisional: true }); + cm.close(); + }); + + it('a store with no calibration record loads a provisional 1: the first sample the band accepts sets it, and stamps the record', async () => { + const epoch = AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH; + const { cm, strategy, internals } = await openManager('fresh-provisional'); + await cm.compile(BUDGET); + assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1, { fromEpoch: null }]); + assert.equal(internals.store!.getStateJson(internals.calibrationStateId), null, 'the load writes nothing'); + strategy.reportRealInputTokens(internals._lastCompileEstimate * 3); + assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1, { fromEpoch: null }], 'a sample the band rejects leaves it provisional'); + await cm.compile(BUDGET); + strategy.reportRealInputTokens(internals._lastCompileEstimate * 1.4); + assert.ok(Math.abs(internals._calibration - 1.4) < 1e-9, `set by the sample, got ${internals._calibration}`); + assert.equal(internals._calibrationProvisional, null); + const saved = internals.store!.getStateJson(internals.calibrationStateId) as + { multiplier?: number; pricing?: number; provisional?: boolean; reset?: unknown }; + assert.deepStrictEqual([saved.multiplier, saved.pricing, saved.provisional, saved.reset], [1.4, epoch, undefined, undefined]); + await cm.compile(BUDGET); + strategy.reportRealInputTokens(internals._lastCompileEstimate * 1.2); + assert.ok(internals._calibration > 1.4 && internals._calibration < 1.6, `stepped, not replaced: ${internals._calibration}`); + cm.close(); + }); + + it("a store with no calibration record whose first sample falls inside the dead band confirms its 1, and stamps the record", async () => { + const { cm, strategy, internals } = await openManager('fresh-confirm'); + await cm.compile(BUDGET); + strategy.reportRealInputTokens(internals._lastCompileEstimate * 1.01); + assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1, null]); + const saved = internals.store!.getStateJson(internals.calibrationStateId) as { multiplier?: number; pricing?: number; provisional?: boolean }; + assert.deepStrictEqual([saved.multiplier, saved.pricing, saved.provisional], [1, AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH, undefined]); + cm.close(); + }); + + it('a store with no calibration record is still provisional after a restart before its first sample', async () => { + const first = await openManager('fresh-restart'); + await first.cm.compile(BUDGET); + first.cm.close(); + const { cm, strategy, internals } = await openManager('fresh-restart'); + await cm.compile(BUDGET); + assert.deepStrictEqual(internals._calibrationProvisional, { fromEpoch: null }); + assert.equal(internals.store!.getStateJson(internals.calibrationStateId), null); + strategy.reportRealInputTokens(internals._lastCompileEstimate * 0.8); + assert.ok(Math.abs(internals._calibration - 0.8) < 1e-9, `set by the sample, got ${internals._calibration}`); + cm.close(); + }); + + it('a record with no multiplier to use loads as no record: a provisional 1', async () => { + const epoch = AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH; + for (const [i, record] of ([{ at: 1, pricing: epoch }, { multiplier: '1.4', at: 1, pricing: epoch }, { multiplier: null, at: 1, pricing: epoch }, 7] as unknown[]).entries()) { + const name = `fresh-unusable-${i}`; + const first = await openManager(name); + await first.cm.compile(BUDGET); + first.internals.store!.setStateJson(first.internals.calibrationStateId, record); + first.cm.close(); + const { cm, internals } = await openManager(name); + await cm.compile(BUDGET); + assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1, { fromEpoch: null }], JSON.stringify(record)); + cm.close(); + } + }); + + it('a calibration read that throws loads 1, not provisional, as before: the record it could not read may hold one worth keeping', async () => { + const { cm, internals } = await openManager('fresh-unreadable'); + const store = internals.store as unknown as { getStateJson(id: string): unknown }; + const read = store.getStateJson; + let refused = 0; + store.getStateJson = (id: string) => { + if (id === internals.calibrationStateId) { refused++; throw new Error('unreadable'); } + return read.call(store, id); + }; + try { + await cm.compile(BUDGET); + } finally { + store.getStateJson = read; + } + assert.ok(refused > 0, 'premise: the load read the record, and the read threw'); + assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1, null]); cm.close(); }); @@ -636,6 +740,34 @@ describe('estimator calibration band', () => { belowReopened.cm.close(); }); + it('dropping a provisional multiplier to 1 drops its declared mark in memory as on disk, for a seeded record that carries both', async () => { + const epoch = AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH; + const tight = await tightBudget('drop-declared-probe', 1.2); + const first = await openManager('drop-declared'); + await first.cm.compile(BUDGET); + first.internals.store!.setStateJson(first.internals.calibrationStateId, + { multiplier: 1.45, at: 1, pricing: epoch, provisional: true, declaredOverhead: true }); + first.cm.close(); + const { cm, strategy, internals } = await openManager('drop-declared'); + const declared = () => (internals as unknown as { _calibrationDeclared: boolean })._calibrationDeclared; + await cm.compile(BUDGET); + assert.deepStrictEqual([internals._calibration, declared()], [1.45, true], 'premise: loaded, marked'); + const dry = await calibrationWarnings(() => cm.compile(tight, undefined, { dryRun: true })); + assert.equal(dry.result.renderStats!.calibration!.multiplier, 1, 'premise: the dry compile dropped it'); + assert.deepStrictEqual([internals._calibration, declared()], [1.45, true], 'put back after the dry compile that dropped it'); + await calibrationWarnings(() => cm.compile(tight)); + assert.equal(internals._calibration, 1); + assert.equal(cm.getRenderStats()!.calibration!.reset?.discardedMultiplier, 1.45, 'premise: the seeded 1.45 was loaded and dropped'); + assert.equal(declared(), false); + const dropped = internals.store!.getStateJson(internals.calibrationStateId) as { multiplier?: number; provisional?: boolean; declaredOverhead?: boolean }; + assert.deepStrictEqual([dropped.multiplier, dropped.provisional, dropped.declaredOverhead], [1, true, undefined], 'the record the drop writes'); + strategy.reportRealInputTokens(internals._lastCompileEstimate * 1.01); + const saved = internals.store!.getStateJson(internals.calibrationStateId) as { multiplier?: number; provisional?: boolean; declaredOverhead?: boolean }; + assert.deepStrictEqual([saved.multiplier, saved.provisional, saved.declaredOverhead], [1, undefined, undefined], + 'an undeclared sample confirming the 1 writes the record, with no mark either'); + cm.close(); + }); + it('a discard\'s provisional 1 survives a restart until a sample confirms it', async () => { // Both discard writers (the load's and the drop's) persist the 1 as // provisional; a reopen before the first sample must not read it as @@ -1165,9 +1297,14 @@ describe('estimator calibration band', () => { }); it('a drift inside 2% is not applied; once the EMA passes 2% it is', async () => { + const first = await openManager('deadband'); + await first.cm.compile(BUDGET); + first.internals.store!.setStateJson(first.internals.calibrationStateId, + { multiplier: 1, at: 1, pricing: AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH }); + first.cm.close(); const { cm, strategy, internals } = await openManager('deadband'); await cm.compile(BUDGET); - assert.equal(internals._calibration, 1); + assert.deepStrictEqual([internals._calibration, internals._calibrationProvisional], [1, null], 'premise: the seeded record loads as learned'); // real/est = 1.01 every compile: the EMA creeps toward 1.01 and never // leaves the band, so nothing is repriced between compiles. for (let i = 0; i < 12; i++) { diff --git a/test/calibration-overhead.test.ts b/test/calibration-overhead.test.ts index 9bd493ca..98149747 100644 --- a/test/calibration-overhead.test.ts +++ b/test/calibration-overhead.test.ts @@ -16,6 +16,8 @@ import { mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { Membrane, MockAdapter, NativeFormatter, type ContentBlock, type ToolDefinition } from '@animalabs/membrane'; +import type { ContextInjection } from '../src/types/index.js'; +import type { RequestOverhead } from '../src/index.js'; import { ContextManager, AutobiographicalStrategy } from '../src/index.js'; import type { StoredMessage } from '../src/types/index.js'; @@ -84,7 +86,9 @@ describe('estimator calibration and request overhead (#323)', () => { // Undeclared, this sample reads real/est ≈ 1.5: in band, so it would be // learned as a per-token rate 50% above the truth. strategy.reportRealInputTokens(window + overhead, { system: SYSTEM, tools }); - assert.equal(internals._calibration, 1); + await cm.compile(BUDGET); + strategy.reportRealInputTokens(internals._lastCompileEstimate + overhead, { system: SYSTEM, tools }); + assert.equal(internals._calibration, 1, 'neither sample moves it from 1: the first confirms the provisional 1 of a store with no record, and the second is an EMA step'); cm.close(); }); @@ -194,3 +198,458 @@ describe('estimator calibration and request overhead (#323)', () => { cm.close(); }); }); + +function priceOf(cm: ContextManager, content: ContentBlock[]): number { + return (cm as unknown as { messageStore: { estimateTokens(m: StoredMessage): number } }).messageStore.estimateTokens({ content } as unknown as StoredMessage); +} + +function injectionOf(position: 'beforeUser' | 'afterUser' | 'system', tokens: number): ContextInjection { + return { namespace: 'probe', position, content: [{ type: 'text', text: 'Retrieved knowledge. '.repeat(Math.ceil(tokens / 4)) }] }; +} + +type DeclaredInternals = Internals & { _calibrationDeclared: boolean; _calibrationArmed: boolean }; +const epoch = AutobiographicalStrategy.CALIBRATION_PRICING_EPOCH; + +async function seeded(name: string, record: Record) { + const first = await openManager(name); + await first.cm.compile(BUDGET); + first.internals.store!.setStateJson(first.internals.calibrationStateId, record); + first.cm.close(); + const opened = await openManager(name); + await opened.cm.compile(BUDGET); + return { ...opened, internals: opened.internals as DeclaredInternals }; +} + +async function sampleAt(cm: ContextManager, strategy: AutobiographicalStrategy, internals: Internals, impliedMultiplier: number, declared: boolean): Promise { + await cm.compile(BUDGET); + const m = internals._calibration; + const tools = toolsOf(internals._lastCompileEstimate / m / 2); + const rawWindow = internals._lastCompileEstimate / m; + const rawOverhead = overheadPrice(cm, SYSTEM, tools) / m; + strategy.reportRealInputTokens(impliedMultiplier * (rawWindow + (declared ? rawOverhead : 0)), declared ? { system: SYSTEM, tools } : undefined); +} + +const declaredOf = (internals: Internals) => + (internals.store!.getStateJson(internals.calibrationStateId) as { declaredOverhead?: boolean } | null)?.declaredOverhead; + +describe('estimator calibration: what the compile adds among the messages', () => { + async function twoSamples(name: string, injection: (window: number) => ContextInjection) { + const { cm, strategy, internals } = await openManager(name); + await cm.compile(BUDGET); + const window = internals._lastCompileEstimate; + const inj = injection(window); + const tools = toolsOf(window / 2); + for (let i = 0; i < 2; i++) { + await cm.compile(BUDGET, [inj]); + assert.equal(internals._lastCompileEstimate, window, 'the window estimate leaves the injection out'); + strategy.reportRealInputTokens(window + overheadPrice(cm, SYSTEM, tools) + priceOf(cm, inj.content), { system: SYSTEM, tools }); + } + return { cm, internals, window, inj }; + } + + it('a message-position injection is priced into the sample: in band, it is not learned as a per-token rate', async () => { + const { cm, internals, window, inj } = await twoSamples('injected-in-band', (w) => injectionOf('afterUser', w / 2)); + assert.ok(priceOf(cm, inj.content) > window * 0.4, + `premise: priced as the window and the declared head alone, these samples would read about 1.33; injection ${priceOf(cm, inj.content)} vs window ${window}`); + assert.equal(internals._calibration, 1); + assert.equal(declaredOf(internals), true, 'the samples were taken'); + cm.close(); + }); + + it('a large injection on a small window is not learned at the clamp', async () => { + const { cm, strategy, internals } = await openManager('injected-out-of-band'); + await cm.compile(BUDGET); + const window = internals._lastCompileEstimate; + const injection = injectionOf('beforeUser', window * 3); + const tools = toolsOf(window / 2); + for (let i = 0; i < 4; i++) { + await cm.compile(BUDGET, [injection]); + strategy.reportRealInputTokens(internals._lastCompileEstimate + overheadPrice(cm, SYSTEM, tools) + priceOf(cm, injection.content), { system: SYSTEM, tools }); + } + assert.equal(internals._calibration, 1); + assert.equal(declaredOf(internals), true, 'the samples were taken in band'); + cm.close(); + }); + + it('a dry compile between the compile and its sample leaves the compile\'s injection priced', async () => { + const { cm, strategy, internals } = await openManager('injected-dry'); + await cm.compile(BUDGET); + const window = internals._lastCompileEstimate; + const injection = injectionOf('afterUser', window / 2); + const tools = toolsOf(window / 2); + for (let i = 0; i < 2; i++) { + await cm.compile(BUDGET, [injection]); + await cm.compile(BUDGET, undefined, { dryRun: true }); + strategy.reportRealInputTokens(window + overheadPrice(cm, SYSTEM, tools) + priceOf(cm, injection.content), { system: SYSTEM, tools }); + } + assert.equal(internals._calibration, 1); + assert.equal(declaredOf(internals), true, 'the samples were taken'); + cm.close(); + }); + + it('a compile without injections prices none, after one with', async () => { + const { cm, strategy, internals } = await openManager('injected-then-none'); + await cm.compile(BUDGET); + const window = internals._lastCompileEstimate; + const injection = injectionOf('afterUser', window / 2); + const tools = toolsOf(window / 2); + await cm.compile(BUDGET, [injection]); + strategy.reportRealInputTokens(window + overheadPrice(cm, SYSTEM, tools) + priceOf(cm, injection.content), { system: SYSTEM, tools }); + await cm.compile(BUDGET); + strategy.reportRealInputTokens(internals._lastCompileEstimate + overheadPrice(cm, SYSTEM, tools), { system: SYSTEM, tools }); + assert.equal(internals._calibration, 1); + assert.equal(declaredOf(internals), true, 'the samples were taken'); + cm.close(); + }); + + it('a system-position injection is priced once, as part of the declared system prompt', async () => { + const { cm, strategy, internals } = await openManager('injected-system'); + await cm.compile(BUDGET); + const window = internals._lastCompileEstimate; + const injection = injectionOf('system', window / 2); + const tools = toolsOf(window / 2); + for (let i = 0; i < 2; i++) { + const result = await cm.compile(BUDGET, [injection]); + const system = `${SYSTEM}\n${result.systemInjections.map((b) => (b as { text: string }).text).join('\n')}`; + strategy.reportRealInputTokens(internals._lastCompileEstimate + overheadPrice(cm, system, tools), { system, tools }); + } + assert.equal(internals._calibration, 1); + assert.equal(declaredOf(internals), true, 'the samples were taken'); + cm.close(); + }); + + it('a dry compile with injections leaves the committing compile\'s count', async () => { + const { cm, strategy, internals } = await openManager('dry-with-injections'); + await cm.compile(BUDGET); + const window = internals._lastCompileEstimate; + const tools = toolsOf(window / 2); + for (let i = 0; i < 2; i++) { + await cm.compile(BUDGET); + await cm.compile(BUDGET, [injectionOf('afterUser', window / 2)], { dryRun: true }); + strategy.reportRealInputTokens(internals._lastCompileEstimate + overheadPrice(cm, SYSTEM, tools), { system: SYSTEM, tools }); + } + assert.equal(internals._calibration, 1); + assert.equal(declaredOf(internals), true, 'the samples were taken'); + cm.close(); + }); + + it('several message-position injections are each priced', async () => { + const { cm, strategy, internals } = await openManager('two-injections'); + await cm.compile(BUDGET); + const window = internals._lastCompileEstimate; + const injections = [injectionOf('beforeUser', window / 4), injectionOf('afterUser', window / 4)]; + const tools = toolsOf(window / 2); + for (let i = 0; i < 2; i++) { + await cm.compile(BUDGET, injections); + const injected = injections.reduce((sum, inj) => sum + priceOf(cm, inj.content), 0); + strategy.reportRealInputTokens(internals._lastCompileEstimate + overheadPrice(cm, SYSTEM, tools) + injected, { system: SYSTEM, tools }); + } + assert.equal(internals._calibration, 1); + assert.equal(declaredOf(internals), true, 'the samples were taken'); + cm.close(); + }); + + it('an injection is priced at the multiplier in force, as the window is', async () => { + const { cm, strategy, internals } = await seeded('injected-calibrated', { multiplier: 1.5, at: 1, pricing: epoch, declaredOverhead: true }); + assert.equal(internals._calibration, 1.5); + const tools = toolsOf(internals._lastCompileEstimate / 2); + const injection = injectionOf('afterUser', internals._lastCompileEstimate / 2); + await cm.compile(BUDGET, [injection]); + strategy.reportRealInputTokens(internals._lastCompileEstimate + overheadPrice(cm, SYSTEM, tools) + priceOf(cm, injection.content), { system: SYSTEM, tools }); + assert.equal(internals._calibration, 1.5); + assert.equal((internals as unknown as { _calibrationEma: number | null })._calibrationEma, 1.5, 'the sample was taken'); + cm.close(); + }); + + it('an undeclared sample counts the injections too', async () => { + const { cm, strategy, internals } = await openManager('injected-undeclared'); + await cm.compile(BUDGET); + const window = internals._lastCompileEstimate; + const injection = injectionOf('afterUser', window / 2); + await cm.compile(BUDGET, [injection]); + strategy.reportRealInputTokens(window + priceOf(cm, injection.content)); + assert.equal(internals._calibration, 1); + assert.equal((internals as unknown as { _calibrationEma: number | null })._calibrationEma, 1, 'the sample was taken'); + cm.close(); + }); + + it('a system prompt alone is priced into the sample', async () => { + const { cm, strategy, internals } = await openManager('system-only'); + await cm.compile(BUDGET); + const window = internals._lastCompileEstimate; + const system = `You are a resident. ${'Keep to your purpose. '.repeat(Math.ceil(window / 14))}`; + const share = overheadPrice(cm, system, []) / window; + assert.ok(share > 0.25 && share < 0.7, `premise: left out, the samples would read 1.25 to 1.7, in band; the system prompt is ${share} of the window`); + for (let i = 0; i < 2; i++) { + await cm.compile(BUDGET); + strategy.reportRealInputTokens(internals._lastCompileEstimate + overheadPrice(cm, system, []), { system }); + } + assert.equal(internals._calibration, 1); + assert.equal(declaredOf(internals), true, 'the samples were taken'); + cm.close(); + }); +}); + +describe('estimator calibration: consistent out-of-band runs', () => { + async function oob(cm: ContextManager, strategy: AutobiographicalStrategy, internals: Internals, ratio: number, declared = true) { + await cm.compile(BUDGET); + const nothingBesides: RequestOverhead = {}; + strategy.reportRealInputTokens(internals._lastCompileEstimate * ratio, declared ? nothingBesides : undefined); + } + + it('a consistent run on a learned record without the flag is learned at the clamp, not held', async () => { + const { cm, strategy, internals } = await seeded('run-on-learned', { multiplier: 1, at: 1, pricing: epoch }); + assert.equal(internals._calibrationProvisional, null, 'learned'); + for (let i = 0; i < 3; i++) await oob(cm, strategy, internals, 2.1); + assert.equal(internals._calibration, 1.8); + assert.equal(declaredOf(internals), true); + cm.close(); + }); + + it('an undeclared out-of-band sample ends a declared run', async () => { + const { cm, strategy, internals } = await openManager('undeclared-ends-run'); + await oob(cm, strategy, internals, 2.1); + await oob(cm, strategy, internals, 2.1); + await oob(cm, strategy, internals, 2.1, false); + await oob(cm, strategy, internals, 2.1); + assert.equal(internals._calibration, 1); + await oob(cm, strategy, internals, 2.1); + await oob(cm, strategy, internals, 2.1); + assert.equal(internals._calibration, 1.8, 'a fresh run of three is learned'); + cm.close(); + }); + + it('a run agrees with its first sample: within 10% it is learned, and a drift past 10% of the first is not', async () => { + const drift = await openManager('run-drift'); + for (const ratio of [2.0, 2.18, 2.38]) await oob(drift.cm, drift.strategy, drift.internals, ratio); + assert.equal(drift.internals._calibration, 1, 'each within 10% of the one before, the third 19% past the first'); + drift.cm.close(); + const near = await openManager('run-near'); + for (const ratio of [2.0, 2.18, 2.08]) await oob(near.cm, near.strategy, near.internals, ratio); + assert.equal(near.internals._calibration, 1.8, 'each within 10% of the first'); + near.cm.close(); + }); + + it('a consistent run below the band is learned at the floor', async () => { + const { cm, strategy, internals } = await openManager('run-low'); + for (let i = 0; i < 3; i++) await oob(cm, strategy, internals, 0.3); + assert.equal(internals._calibration, 0.6); + cm.close(); + }); + + it('one compile counts once toward a run, however many reports follow it', async () => { + const { cm, strategy, internals } = await openManager('run-one-arm'); + await cm.compile(BUDGET); + for (let i = 0; i < 3; i++) strategy.reportRealInputTokens(internals._lastCompileEstimate * 2.1, {}); + assert.equal(internals._calibration, 1, 'three reports of one compile'); + await oob(cm, strategy, internals, 2.1); + assert.equal(internals._calibration, 1, 'the second compile is the run\'s second sample'); + await oob(cm, strategy, internals, 2.1); + assert.equal(internals._calibration, 1.8, 'the third compile is its third'); + cm.close(); + }); + + it('a branch switch starts a run afresh', async () => { + const { cm, strategy, internals } = await openManager('run-branch'); + await oob(cm, strategy, internals, 2.1); + await oob(cm, strategy, internals, 2.1); + await cm.fork('other'); + await oob(cm, strategy, internals, 2.1); + assert.equal(internals._calibration, 1); + assert.equal((internals as unknown as { _outOfBandRun: { count: number } | null })._outOfBandRun?.count, 1, 'the sample on the new branch starts a run'); + cm.close(); + }); +}); + +describe('estimator calibration: a store that learned the overhead as a rate', () => { + it('two declared samples that agree replace a multiplier learned without them, by their mean', async () => { + const { cm, strategy, internals } = await seeded('learned-undeclared', { multiplier: 1.8, at: 1, pricing: epoch }); + assert.equal(internals._calibration, 1.8); + await sampleAt(cm, strategy, internals, 1, true); + assert.equal(internals._calibration, 1.8, 'the first is held'); + assert.equal(declaredOf(internals), undefined); + await sampleAt(cm, strategy, internals, 1.04, true); + assert.ok(Math.abs(internals._calibration - 1.02) <= 0.01, `replaced by the mean, got ${internals._calibration}`); + assert.equal(declaredOf(internals), true, 'and the record says its samples declare their overhead'); + cm.close(); + }); + + it('two declared samples that disagree step by the EMA, and mark the record declared', async () => { + const { cm, strategy, internals } = await seeded('learned-disagree', { multiplier: 1.8, at: 1, pricing: epoch }); + await sampleAt(cm, strategy, internals, 1, true); + await sampleAt(cm, strategy, internals, 1.5, true); + assert.ok(Math.abs(internals._calibration - 1.74) <= 0.01, `stepped toward the second, got ${internals._calibration}`); + assert.equal(declaredOf(internals), true); + await sampleAt(cm, strategy, internals, 1, true); + assert.ok(Math.abs(internals._calibration - 1.592) <= 0.01, `then steps by the EMA, got ${internals._calibration}`); + cm.close(); + }); + + it('a branch switch between the two loses the held sample', async () => { + const { cm, strategy, internals } = await seeded('held-branch', { multiplier: 1.8, at: 1, pricing: epoch }); + await sampleAt(cm, strategy, internals, 1, true); + await cm.fork('other'); + await cm.compile(BUDGET); + assert.equal(internals._calibration, 1.8, 'the fork has the record'); + await sampleAt(cm, strategy, internals, 1, true); + assert.equal(internals._calibration, 1.8, 'held, not paired with the other branch\'s sample'); + assert.notEqual((internals as unknown as { _declaredCandidate: number | null })._declaredCandidate, null, 'and held afresh'); + cm.close(); + }); + + it('an undeclared sample between the two leaves the held sample, and the pair still replaces', async () => { + const { cm, strategy, internals } = await seeded('held-undeclared', { multiplier: 1.8, at: 1, pricing: epoch }); + await sampleAt(cm, strategy, internals, 1, true); + assert.equal(internals._calibration, 1.8, 'the first is held'); + await sampleAt(cm, strategy, internals, 1, false); + assert.ok(Math.abs(internals._calibration - 1.64) <= 0.01, `the undeclared sample steps, got ${internals._calibration}`); + await sampleAt(cm, strategy, internals, 1, true); + assert.ok(Math.abs(internals._calibration - 1) <= 0.01, `paired with the held sample and replaced, got ${internals._calibration}`); + assert.equal(declaredOf(internals), true); + cm.close(); + }); + + it('a consistent run between the two ends the held sample', async () => { + const { cm, strategy, internals } = await seeded('held-run', { multiplier: 1, at: 1, pricing: epoch }); + await sampleAt(cm, strategy, internals, 1, true); + for (let i = 0; i < 3; i++) { + await cm.compile(BUDGET); + strategy.reportRealInputTokens(internals._lastCompileEstimate * 2.1, {}); + } + assert.equal(internals._calibration, 1.8, 'the run is learned'); + await sampleAt(cm, strategy, internals, 1.5, false); + assert.ok(Math.abs(internals._calibration - 1.74) <= 0.01, `an undeclared sample steps and unmarks, got ${internals._calibration}`); + await sampleAt(cm, strategy, internals, 1, true); + assert.ok(Math.abs(internals._calibration - 1.74) <= 0.01, `held, not paired with the sample from before the run, got ${internals._calibration}`); + cm.close(); + }); + + it('a restart between the two loses the held sample: it starts over, and the record is not marked', async () => { + const first = await seeded('held-restart', { multiplier: 1.8, at: 1, pricing: epoch }); + await sampleAt(first.cm, first.strategy, first.internals, 1, true); + first.cm.close(); + const { cm, strategy, internals } = await openManager('held-restart'); + await cm.compile(BUDGET); + const typed = internals as DeclaredInternals; + assert.deepEqual([internals._calibration, typed._calibrationDeclared, declaredOf(internals)], [1.8, false, undefined]); + await sampleAt(cm, strategy, internals, 1, true); + assert.equal(internals._calibration, 1.8, 'held again'); + await sampleAt(cm, strategy, internals, 1, true); + assert.ok(Math.abs(internals._calibration - 1) <= 0.02, `then replaced, got ${internals._calibration}`); + cm.close(); + }); + + it('a multiplier learned from declared samples steps by the EMA', async () => { + const { cm, strategy, internals } = await seeded('learned-declared', { multiplier: 1.8, at: 1, pricing: epoch, declaredOverhead: true }); + await sampleAt(cm, strategy, internals, 1, true); + assert.ok(Math.abs(internals._calibration - 1.64) <= 0.01, `stepped, got ${internals._calibration}`); + cm.close(); + }); + + it('after the replacement, a restart steps by the EMA', async () => { + const first = await seeded('declared-restart', { multiplier: 1.8, at: 1, pricing: epoch }); + await sampleAt(first.cm, first.strategy, first.internals, 1, true); + await sampleAt(first.cm, first.strategy, first.internals, 1, true); + first.cm.close(); + const { cm, strategy, internals } = await openManager('declared-restart'); + await cm.compile(BUDGET); + await sampleAt(cm, strategy, internals, 1.3, true); + assert.ok(Math.abs(internals._calibration - 1.06) <= 0.01, `stepped, got ${internals._calibration}`); + cm.close(); + }); + + it('an undeclared sample that moves the multiplier unmarks the record, and the next two declared replace it', async () => { + const { cm, strategy, internals } = await seeded('undeclared-after-declared', { multiplier: 1, at: 1, pricing: epoch, declaredOverhead: true }); + await sampleAt(cm, strategy, internals, 1.5, false); + assert.ok(internals._calibration > 1.05, `the undeclared sample stepped, got ${internals._calibration}`); + assert.equal(declaredOf(internals), undefined); + await sampleAt(cm, strategy, internals, 1, true); + await sampleAt(cm, strategy, internals, 1, true); + assert.ok(Math.abs(internals._calibration - 1) <= 0.02, `replaced, got ${internals._calibration}`); + cm.close(); + }); + + it('two declared samples that confirm the multiplier mark the record declared', async () => { + const { cm, strategy, internals } = await seeded('confirmed-declared', { multiplier: 1, at: 1, pricing: epoch }); + await sampleAt(cm, strategy, internals, 1, true); + await sampleAt(cm, strategy, internals, 1, true); + assert.equal(internals._calibration, 1); + assert.equal(declaredOf(internals), true); + await sampleAt(cm, strategy, internals, 1.5, true); + assert.ok(Math.abs(internals._calibration - 1.1) <= 0.01, `stepped, got ${internals._calibration}`); + cm.close(); + }); + + it('a store with no record is set by its first declared sample, with nothing held, and marked declared', async () => { + const { cm, strategy, internals } = await openManager('fresh-declared'); + await cm.compile(BUDGET); + assert.deepEqual(internals._calibrationProvisional, { fromEpoch: null }); + await sampleAt(cm, strategy, internals as DeclaredInternals, 1.4, true); + assert.ok(Math.abs(internals._calibration - 1.4) <= 0.01, `set by one sample, got ${internals._calibration}`); + assert.equal((internals as unknown as { _declaredCandidate: number | null })._declaredCandidate, null, 'nothing held'); + assert.equal(declaredOf(internals), true); + cm.close(); + }); + + it('a provisional multiplier is replaced by one declared sample, and marked declared', async () => { + const { cm, strategy, internals } = await seeded('provisional-declared', { multiplier: 1, at: 1, pricing: epoch, provisional: true }); + await sampleAt(cm, strategy, internals, 1.4, true); + assert.ok(Math.abs(internals._calibration - 1.4) <= 0.01, `replaced, not stepped, got ${internals._calibration}`); + assert.equal(internals._calibrationProvisional, null); + assert.equal(declaredOf(internals), true); + cm.close(); + }); + + it('a dry compile after the load keeps a record\'s declared overhead', async () => { + const { cm, strategy, internals } = await seeded('declared-dry', { multiplier: 1.8, at: 1, pricing: epoch, declaredOverhead: true }); + await cm.compile(BUDGET, undefined, { dryRun: true }); + assert.equal(internals._calibrationDeclared, true); + await sampleAt(cm, strategy, internals, 1, true); + assert.ok(Math.abs(internals._calibration - 1.64) <= 0.01, `stepped, not held, got ${internals._calibration}`); + cm.close(); + }); + + it('an undeclared sample inside the dead band leaves a record marked, and a rejected declared one leaves it unmarked', async () => { + const marked = await seeded('deadband-undeclared', { multiplier: 1, at: 1, pricing: epoch, declaredOverhead: true }); + await sampleAt(marked.cm, marked.strategy, marked.internals, 1.01, false); + const ema = (marked.internals as unknown as { _calibrationEma: number | null })._calibrationEma; + assert.ok(ema !== null && Math.abs(ema - 1.002) < 0.001, `premise: the sample reached the EMA, got ${ema}`); + assert.deepStrictEqual([marked.internals._calibration, marked.internals._calibrationDeclared, declaredOf(marked.internals)], [1, true, true]); + marked.cm.close(); + const unmarked = await seeded('rejected-declared', { multiplier: 1, at: 1, pricing: epoch }); + await unmarked.cm.compile(BUDGET); + unmarked.strategy.reportRealInputTokens(unmarked.internals._lastCompileEstimate * 2.1, {}); + assert.equal((unmarked.internals as unknown as { _outOfBandRun: { count: number } | null })._outOfBandRun?.count, 1, 'premise: the sample was rejected as the start of a run'); + assert.deepStrictEqual([unmarked.internals._calibration, unmarked.internals._calibrationDeclared, declaredOf(unmarked.internals)], [1, false, undefined]); + unmarked.cm.close(); + }); + + it('any overhead a JS caller passes but null or undefined counts as declared, in the run as in the mark', async () => { + const { cm, strategy, internals } = await seeded('falsy-overhead', { multiplier: 1, at: 1, pricing: epoch }); + for (let i = 0; i < 3; i++) { + await cm.compile(BUDGET); + strategy.reportRealInputTokens(internals._lastCompileEstimate * 2.1, 0 as unknown as RequestOverhead); + } + assert.equal(internals._calibration, 1.8, 'learned as a declared run'); + assert.equal(declaredOf(internals), true); + cm.close(); + }); + + it('a null overhead from a JS caller counts as undeclared', async () => { + const { cm, strategy, internals } = await seeded('null-overhead', { multiplier: 1, at: 1, pricing: epoch }); + await cm.compile(BUDGET); + strategy.reportRealInputTokens(internals._lastCompileEstimate * 1.5, null as unknown as RequestOverhead); + assert.ok(Math.abs(internals._calibration - 1.1) <= 0.01, `stepped as undeclared, not held, got ${internals._calibration}`); + assert.deepEqual([internals._calibrationDeclared, declaredOf(internals)], [false, undefined]); + cm.close(); + }); + + it('a re-stamp keeps a record\'s declared overhead', async () => { + const { cm, internals } = await seeded('restamp-declared', { multiplier: 1.2, at: 1, pricing: epoch - 1, declaredOverhead: true }); + const saved = internals.store!.getStateJson(internals.calibrationStateId) as { pricing?: number; declaredOverhead?: boolean }; + assert.equal(saved.pricing, epoch, 're-stamped'); + assert.equal(saved.declaredOverhead, true); + assert.equal(internals._calibrationDeclared, true); + cm.close(); + }); +});