From 922fdb611ab7f0cc64819a39c2a9fd381a6c33fb Mon Sep 17 00:00:00 2001 From: Homen Shum Date: Wed, 14 Oct 2026 12:10:25 -0700 Subject: [PATCH 01/11] chore(models): refresh stale model defaults (NodeRoom) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move NodeRoom's default model ids off retired or deprecated ones, per model-defaults SPEC Part A section A.2.9: - one DEFAULT_AGENT_MODEL (gemini-3.8-flash) in convex/modelDefaults.ts replaces the gemini-3.5-flash literals (Google deprecated it 2026-10-08); env chains unchanged - top_paid default anthropic/claude-sonnet-4 -> anthropic/claude-sonnet-5.5; Haiku defaults -> claude-haiku-5-5; pi-ai default gemini-2.5-flash -> DEFAULT_AGENT_MODEL - TTS gemini-3.1-flash-tts-preview -> gemini-3.8-flash-tts (shutdown 2026-11-17) - catalog: 5.5 and gemini pricing rows, tier allowlists, fallback chains pruned; the dotted claude-id regex becomes an explicit table (claude-opus-4.7 now maps to claude-opus-4-7) - Claude 5.5 request shape: forced tool_choice downgraded to auto for Sonnet/Opus 5.5, explicit effort on the AI-SDK path, 2048 max_tokens floor, @ai-sdk/anthropic >= 3.0.120 - judge scripts pinned to gemini-3.6-flash with temperature removed - ladder: --real needs a dollar cap, per-dispatch cost reservation, receipt carries the tested commit, budget and dispatch policy Merge is gated on the L1-L4 ladder receipt (not run here: paid). The adaptiveRouter fast lane stays on gemini-3.5-flash until that receipt exists. Spec: model-defaults SPEC Part A §A.2 (NodeRoom) Co-Authored-By: Claude Opus 5.5 --- .tours/01-primary-user-flow.tour | 4 +- .tours/03-debug-and-recovery.tour | 6 +- convex/agent.ts | 11 +- convex/agentJobRunner.ts | 4 +- convex/agentJobs.ts | 6 +- convex/http.ts | 5 +- convex/modelDefaults.ts | 16 + convex/streaming.ts | 3 +- evals/ladder.ts | 180 ++++++++++-- evals/ladderBudget.test.ts | 251 ++++++++++++++++ package-lock.json | 37 ++- package.json | 4 +- packages/walkthrough-review-cli/README.md | 2 +- packages/walkthrough-review-cli/src/core.ts | 2 +- scripts/benchmark/probe.ts | 13 +- scripts/design-qa/rubric-panel.ts | 4 +- scripts/eval/painJudge.ts | 4 +- scripts/eval/providerParseSmoke.ts | 4 +- scripts/gemini-demo-media-judge.ts | 3 +- scripts/gemini-mp4-workflow-review.ts | 3 +- scripts/gemini-ui-review.ts | 2 +- scripts/gemini-visual-polish.ts | 4 +- scripts/judge-demo-gif.ts | 8 +- scripts/judge-state-captures.ts | 8 +- .../spreadsheetbench-chart-visual-grade.ts | 3 +- scripts/startup-diligence-provider-eval.ts | 5 +- scripts/walkthroughs/judge-video.ts | 6 +- skills/seo-qa/scripts/judge-video-gemini.ts | 3 +- src/eval/spreadsheetBenchChartVisualProbe.ts | 2 +- src/nodeagent/capture/reasoning.ts | 2 +- src/nodeagent/core/adaptiveRouter.ts | 5 +- src/nodeagent/integrations/langchain.ts | 8 +- src/nodeagent/models/adapter.ts | 170 ++++++++++- src/nodeagent/models/convexModel.ts | 31 +- src/nodeagent/models/modelCatalog.ts | 189 ++++++++---- src/nodeagent/models/openAiTokenLimit.ts | 2 +- src/nodeagent/models/outputTokens.ts | 20 ++ src/nodeagent/models/piAiAdapter.ts | 3 +- src/nodeagent/models/providerParserLive.ts | 8 +- src/ui/Chat.tsx | 3 +- tests/agentJobsRuntime.test.ts | 28 ++ tests/agentJobsSource.test.ts | 4 +- .../modelAdapterProviderNeutralRoute.test.ts | 10 +- tests/modelAliasClaude.test.ts | 17 +- tests/modelCatalogRefresh.test.ts | 108 +++++++ tests/modelPricingAgreement.test.ts | 86 ++++++ tests/modelRefreshRequestShape.test.ts | 277 ++++++++++++++++++ tests/openAiTokenLimit.test.ts | 6 + tests/openRouterFreeModels.test.ts | 4 +- tests/providerParserLive.test.ts | 6 +- tests/qaMatrix.test.ts | 8 +- tests/voiceHostedTtsProviders.test.ts | 3 +- vitest.config.ts | 2 +- walkthrough-review.config.json | 2 +- 54 files changed, 1408 insertions(+), 197 deletions(-) create mode 100644 convex/modelDefaults.ts create mode 100644 evals/ladderBudget.test.ts create mode 100644 src/nodeagent/models/outputTokens.ts create mode 100644 tests/modelCatalogRefresh.test.ts create mode 100644 tests/modelPricingAgreement.test.ts create mode 100644 tests/modelRefreshRequestShape.test.ts diff --git a/.tours/01-primary-user-flow.tour b/.tours/01-primary-user-flow.tour index cdd9f11d..d3c512cc 100644 --- a/.tours/01-primary-user-flow.tour +++ b/.tours/01-primary-user-flow.tour @@ -27,14 +27,14 @@ }, { "file": "src/ui/Chat.tsx", - "line": 1878, + "line": 1879, "pattern": "const send = \\(raw\\?: string", "title": "The primary user action", "description": "`send` is what the Send button calls. Two separate things happen and it matters that they are separate:\n\n1. The message is posted to the room, with a client-generated `clientMsgId` — that id is how a retry updates the same message instead of double-posting it.\n2. Only then is the text inspected to see whether it was addressed to the assistant." }, { "file": "src/ui/Chat.tsx", - "line": 1166, + "line": 1167, "pattern": "function parsePublicNodeAgentRequest", "title": "Where text becomes intent", "description": "The assistant that can edit the shared sheet is addressable ONLY as a leading `@nodeagent …` directive. Mentioning it mid-sentence does not summon it.\n\nThis decides intent. It does not decide permission — that happens on the server, in the next step but one." diff --git a/.tours/03-debug-and-recovery.tour b/.tours/03-debug-and-recovery.tour index 83e77502..2aa1316b 100644 --- a/.tours/03-debug-and-recovery.tour +++ b/.tours/03-debug-and-recovery.tour @@ -27,21 +27,21 @@ }, { "file": "src/ui/Chat.tsx", - "line": 269, + "line": 270, "pattern": "const MAX_FAILED_SENDS = 50;", "title": "Failure 3 — a message fails to post, and the list is bounded", "description": "`MAX_FAILED_SENDS = 50`, FIFO. Every in-memory collection on an agent path needs a cap and an eviction rule; agents loop, and an unbounded list fills memory in minutes rather than months." }, { "file": "src/ui/Chat.tsx", - "line": 1943, + "line": 1944, "pattern": "const retrySend = \\(cid: string, text: string\\)", "title": "Retry that cannot double-post", "description": "`retrySend` re-sends with the SAME `clientMsgId`. That id is what makes the retry idempotent at the store, so a user hammering Retry on a flaky connection does not produce five copies of their message." }, { "file": "src/ui/Chat.tsx", - "line": 935, + "line": 936, "pattern": "function buildAgentFailureNotice\\(", "title": "Failure 4 — the agent fails", "description": "`buildAgentFailureNotice` turns a provider error into a sentence a non-engineer can act on, carrying which model selection was used, what was asked, and which job it was. A stack trace on this surface is a defect, not a debug aid — this is a surface where a user decides whether to trust the assistant." diff --git a/convex/agent.ts b/convex/agent.ts index 657a22fd..e2d8b6a9 100644 --- a/convex/agent.ts +++ b/convex/agent.ts @@ -19,6 +19,7 @@ import { v } from "convex/values"; import { makeFunctionReference } from "convex/server"; +import { DEFAULT_AGENT_MODEL, DEFAULT_FILE_EGRESS_MODEL } from "./modelDefaults"; import { action } from "./_generated/server"; import type { Id } from "./_generated/dataModel"; import { actorProofV } from "./lib"; @@ -62,7 +63,6 @@ const MIN_ACTION_RESERVE_MS = 10_000; const ACTION_SAFETY_MARGIN_MS = 15_000; const DEFAULT_CONTEXT_MAX_CHARS = 24_000; const DEFAULT_CONTEXT_KEEP_RECENT = 10; -const DEFAULT_FILE_EGRESS_MODEL = "z-ai/glm-4.7-flash"; const roomsFullRef = makeFunctionReference<"query">("rooms:full"); const agentJobsCreateOrReuseRef = makeFunctionReference<"mutation">("agentJobs:createOrReuse") as any; const agentJobsFinishInteractiveRef = makeFunctionReference<"mutation">("agentJobs:finishInteractive") as any; @@ -338,12 +338,13 @@ export const runRoomAgent = action({ // - research (background synthesis): deepseek-v4-flash — first route to clear the v3 composite- // synthesis benchmark 9/9 at $0.0034/run (docs/eval/results.json; ~300x cheaper than gemini-3.5-flash's // measured $1.10/task). Evidence covers the fetch->synthesize->write shape ONLY. - // - interactive collaboration (lock/CAS/draft): stays on gemini-3.5-flash, the only route with a - // recorded L1-L4 collaboration-ladder pass. flash gets this lane only after it passes the ladder. + // - interactive collaboration (lock/CAS/draft): DEFAULT_AGENT_MODEL (gemini-3.8-flash). The recorded + // L1-L4 collaboration-ladder pass is for gemini-3.5-flash (Google now serves it as 3.6); this default + // merges only with docs/eval/ladder/model-refresh-noderoom.json (3.6 + 3.8, 8/8 pass). const egressArtifacts = providerEgressArtifactsFromRoomState(roomState); const requestedModelName = a.mode === "research" ? (process.env.AGENT_RESEARCH_MODEL ?? "minimax/minimax-m3") - : (process.env.AGENT_MODEL ?? "gemini-3.5-flash"); + : (process.env.AGENT_MODEL ?? DEFAULT_AGENT_MODEL); const model = agentModel(modelNameForEgress(requestedModelName, "public_ask", egressArtifacts), { entrypoint: "public_ask" }); const egressDecision = providerEgressDecision({ model: model.name, @@ -780,7 +781,7 @@ export const runPrivateAgent = action({ const requester = roomState.members.find((m: { id: unknown }) => String(m.id) === a.requester.actor.id) as { id: unknown; name: string } | undefined; if (!requester) throw new Error("member_required"); const egressArtifacts = providerEgressArtifactsFromRoomState(roomState); - const model = agentModel(modelNameForEgress(process.env.AGENT_MODEL ?? "gemini-3.5-flash", "private_agent", egressArtifacts), { entrypoint: "private_agent" }); + const model = agentModel(modelNameForEgress(process.env.AGENT_MODEL ?? DEFAULT_AGENT_MODEL, "private_agent", egressArtifacts), { entrypoint: "private_agent" }); const egressDecision = providerEgressDecision({ model: model.name, entrypoint: "private_agent", diff --git a/convex/agentJobRunner.ts b/convex/agentJobRunner.ts index 259cbdb6..741510d8 100644 --- a/convex/agentJobRunner.ts +++ b/convex/agentJobRunner.ts @@ -43,6 +43,7 @@ import { } from "../src/nodeagent/guardrails/egressPolicy"; import { makeConvexStepJournal } from "./agentStepJournalClient"; import { createVerifiedWorkbookWorkflowHook } from "../src/nodeagent/guardrails/workbookWorkflow"; +import { DEFAULT_AGENT_MODEL, DEFAULT_FILE_EGRESS_MODEL } from "./modelDefaults"; const CONVEX_ACTION_LIMIT_MS = 10 * 60_000; const DEFAULT_SLICE_BUDGET_MS = 7 * 60_000; @@ -53,7 +54,6 @@ const DEFAULT_LEASE_EXTRA_MS = 60_000; const DEFAULT_RESUME_DELAY_MS = 5_000; const DEFAULT_CONTEXT_MAX_CHARS = 24_000; const DEFAULT_CONTEXT_KEEP_RECENT = 10; -const DEFAULT_FILE_EGRESS_MODEL = "z-ai/glm-4.7-flash"; const RESUME_CHECKPOINT_PREFIX = "RESUME CHECKPOINT:"; const agentJobsClaimSliceRef = makeFunctionReference<"mutation">("agentJobs:claimSlice") as any; const agentJobsFinishSliceRef = makeFunctionReference<"mutation">("agentJobs:finishSlice") as any; @@ -509,7 +509,7 @@ export const runFreeAutoJobSlice = internalAction({ const roomArtifacts = await ctx.runQuery(artifactsListForRoomRef, { roomId: claimed.roomId }) as Array<{ title?: string; kind?: string; meta?: unknown; visibility?: string }>; const egressArtifacts = providerEgressArtifactsForClaimedJob(roomArtifacts, claimed); let entrypoint = runnerEntrypoint(claimed); - const modelPolicy = claimed.modelPolicy || (entrypoint === "free" ? "openrouter/free-auto" : process.env.AGENT_MODEL ?? "gemini-3.5-flash"); + const modelPolicy = claimed.modelPolicy || (entrypoint === "free" ? "openrouter/free-auto" : process.env.AGENT_MODEL ?? DEFAULT_AGENT_MODEL); let resolvedModelPolicy = modelPolicy === "openrouter/free-auto" ? process.env.FREE_AUTO_JOB_MODEL ?? modelPolicy : modelPolicy; diff --git a/convex/agentJobs.ts b/convex/agentJobs.ts index a9178743..01b7d046 100644 --- a/convex/agentJobs.ts +++ b/convex/agentJobs.ts @@ -28,13 +28,13 @@ import { } from "../src/nodeagent/guardrails/egressPolicy"; import { parseBulkCompanyIngest } from "../src/nodeagent/skills/finance/bulkIngest"; import { finalizePublicAgentJobStreamAfterTerminal } from "./agentStreamTerminal"; +import { DEFAULT_AGENT_MODEL, DEFAULT_FILE_EGRESS_MODEL } from "./modelDefaults"; // BOUND: cap a single bulk-diligence fan-out so one command can't enqueue unbounded jobs. const MAX_BULK_COMPANIES = 50; const MAX_INVESTIGATION_ELEMENTS = 8_192; const MAX_INVESTIGATION_WRITE_IDS = 8_192; const LEASE_EXPIRY_GRACE_MS = 30_000; -const DEFAULT_FILE_EGRESS_MODEL = "z-ai/glm-4.7-flash"; const INVESTIGATION_RESEARCH_GOAL = "Research every pending or stale company in the exact authorized dataset: claim only its editable research fields, retrieve public sources, write evidence-bearing summary/funding/headcount/recent_signal plus source/source2/last_researched/status, and release every managed lock."; const INVESTIGATION_WRITABLE_FIELD_IDS = [ "summary", @@ -2098,9 +2098,9 @@ function defaultModelPolicyForRoute(args: { routePolicy: RoutePolicy; entrypoint if (args.modelPolicy) return args.modelPolicy; if (args.routePolicy === "explicit") throw new Error("explicit_route_requires_modelPolicy"); if (args.routePolicy === "free_auto" || args.entrypoint === "free") return "openrouter/free-auto"; - if (args.routePolicy === "top_paid") return process.env.AGENT_TOP_PAID_MODEL ?? process.env.AGENT_MODEL ?? "anthropic/claude-sonnet-4"; + if (args.routePolicy === "top_paid") return process.env.AGENT_TOP_PAID_MODEL ?? process.env.AGENT_MODEL ?? "anthropic/claude-sonnet-5.5"; if (args.mode === "research") return process.env.AGENT_RESEARCH_MODEL ?? process.env.AGENT_WORKER_MODEL ?? "minimax/minimax-m3"; - return process.env.AGENT_ORCHESTRATOR_MODEL ?? process.env.AGENT_MODEL ?? "gemini-3.5-flash"; + return process.env.AGENT_ORCHESTRATOR_MODEL ?? process.env.AGENT_MODEL ?? DEFAULT_AGENT_MODEL; } function configuredFileEgressModel() { diff --git a/convex/http.ts b/convex/http.ts index 7403d7b4..26f00878 100644 --- a/convex/http.ts +++ b/convex/http.ts @@ -15,6 +15,7 @@ import type { StreamId } from "@convex-dev/persistent-text-streaming"; import { streamingComponent } from "./streaming"; import { streamPrivateReplyText } from "./streamingModel"; import { privateAgentSystemPrompt } from "./agentPrompts"; +import { DEFAULT_AGENT_MODEL } from "./modelDefaults"; import { auth } from "./auth"; const http = httpRouter(); @@ -32,7 +33,7 @@ const NVIDIA_STT_MODEL = "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning"; const NVIDIA_AUDIO_MAX_BYTES = 5 * 1024 * 1024; const CLOUDFLARE_AI_MODEL_TTS = "@cf/myshell-ai/melotts"; const GEMINI_TTS_BASE_URL = "https://generativelanguage.googleapis.com/v1beta/models"; -const GEMINI_TTS_MODEL = "gemini-3.1-flash-tts-preview"; +const GEMINI_TTS_MODEL = "gemini-3.8-flash-tts"; const GOOGLE_TTS_URL = "https://texttospeech.googleapis.com/v1/text:synthesize"; type VoiceSttProvider = "openai" | "nvidia"; type VoiceTtsProvider = "openai" | "cloudflare" | "gemini" | "google" | "disabled"; @@ -172,7 +173,7 @@ http.route({ const userMsg = `ROOM CONTEXT\n${meta.roomContext}\n\n${meta.requesterName} asks: ${meta.goal}`; let answer = ""; try { - answer = await streamPrivateReplyText(process.env.AGENT_MODEL ?? "gemini-3.5-flash", system, userMsg, append); + answer = await streamPrivateReplyText(process.env.AGENT_MODEL ?? DEFAULT_AGENT_MODEL, system, userMsg, append); } catch (error) { // HONEST_STATUS: the partial text persists, the error is visible text, never a silent 2xx void. const msg = `(private agent error: ${error instanceof Error ? error.message.slice(0, 160) : "model call failed"})`; diff --git a/convex/modelDefaults.ts b/convex/modelDefaults.ts new file mode 100644 index 00000000..61fca9af --- /dev/null +++ b/convex/modelDefaults.ts @@ -0,0 +1,16 @@ +/** + * Convex-safe default model ids (no imports, no Node APIs). One literal per default so a model + * refresh edits one line. Env chains (AGENT_MODEL ?? ...) stay at the call sites. + * + * Spec: model-defaults SPEC Part A §A.2 (NodeRoom). + */ + +/** + * Default agent model. Was "gemini-3.5-flash": Google deprecated it 2026-10-08 and auto-routes it + * to gemini-3.6-flash. Merge is gated on the L1-L4 ladder receipt in + * docs/eval/ladder/model-refresh-noderoom.json (see CHANGELOG/qa/model-refresh-noderoom.md). + */ +export const DEFAULT_AGENT_MODEL = "gemini-3.8-flash"; + +/** File-egress fallback (OpenRouter). Listed, no expiry; unchanged by the refresh. */ +export const DEFAULT_FILE_EGRESS_MODEL = "z-ai/glm-4.7-flash"; diff --git a/convex/streaming.ts b/convex/streaming.ts index b8d10b62..a3ebf6b8 100644 --- a/convex/streaming.ts +++ b/convex/streaming.ts @@ -25,6 +25,7 @@ import { actorProofV, actorV, requireActiveAgentJobLease, requireActorCanUseChan import { summarizeRoomForPrivate } from "./agentPrompts"; import { assertProviderEgressAllowed } from "../src/nodeagent/guardrails/egressPolicy"; import { streamingComponent } from "./streamingComponent"; +import { DEFAULT_AGENT_MODEL } from "./modelDefaults"; export { streamingComponent } from "./streamingComponent"; const roomsFullRef = makeFunctionReference<"query">("rooms:full"); @@ -65,7 +66,7 @@ export const createPrivateReplyStream = mutation({ const roomState = await ctx.runQuery(roomsFullRef, { roomId: a.roomId, requester: a.requester }); if (!roomState) throw new Error("room_not_found"); assertProviderEgressAllowed({ - model: process.env.AGENT_MODEL ?? "gemini-3.5-flash", + model: process.env.AGENT_MODEL ?? DEFAULT_AGENT_MODEL, entrypoint: "private_agent", artifacts: roomState.artifacts.map((art: { title: string; kind: string; meta?: unknown }) => ({ title: art.title, diff --git a/evals/ladder.ts b/evals/ladder.ts index bb17f2b5..47c082cf 100644 --- a/evals/ladder.ts +++ b/evals/ladder.ts @@ -6,8 +6,10 @@ * npm run ladder:real -- gpt-5.4-nano,gemini-3.1-flash-lite */ import "../scripts/benchmark/loadEnv"; -import { mkdirSync, writeFileSync } from "node:fs"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { execSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { pathToFileURL } from "node:url"; import { appendEvalRuns, computeCaseSetHash, DEFAULT_STORE, runKey, type EvalRunRecord } from "./evalStore"; import { dirname, join, relative, resolve } from "node:path"; import { z } from "zod"; @@ -20,7 +22,17 @@ import { runAgent } from "../src/nodeagent/core/runtime"; import type { AgentMessage, AgentModel, AgentResult, AgentTool, AgentTraceEvent, RoomTools } from "../src/nodeagent/core/types"; import { scriptedModel, type Planner } from "../src/nodeagent/models/scripted"; import { recomputeVariancePlan } from "../src/nodeagent/core/plans"; -import { model as realModel, priceRun } from "../src/nodeagent/models/adapter"; +import { + createSpendMeter, + model as realModel, + priceRun, + priceRunStrict, + requestInputTokenUpperBound, + type DispatchPolicy, + type SpendMeter, +} from "../src/nodeagent/models/adapter"; +import { getModelPricing, resolveModelAlias } from "../src/nodeagent/models/modelCatalog"; +import { modelMaxOutputTokens } from "../src/nodeagent/models/outputTokens"; import { selectOpenRouterFreeModels } from "../src/nodeagent/models/openRouterFreeModels"; import { stableJournalHash } from "../src/nodeagent/core/journal"; @@ -97,6 +109,19 @@ interface Rung { interface RuntimeBudget { rungTimeoutMs?: number; reserveMs: number; + /** Dollar cap for a --real run (LADDER_MAX_COST_USD / --max-cost-usd). Mandatory when any provider is called. */ + maxCostUsd?: number; +} + +/** Every --real run: no adapter retries, no cross-model fallback, so one step = at most one reserved request. */ +export const RECEIPT_DISPATCH_POLICY: DispatchPolicy = { maxRetries: 0, crossModelFallback: false }; + +interface LadderBudgetReport { + maxUsd: number; + estimateUsd: number; + spentUsd: number; + requests: number; + stoppedBy: SpendMeter["stoppedBy"]; } interface RungResult { @@ -562,7 +587,7 @@ function checkFailurePointers(rung: Rung, result: AgentResult, checks: Record AgentModel, modelName: string, budget: RuntimeBudget): Promise { +async function runRung(rung: Rung, maker: (rung: Rung) => AgentModel, modelName: string, budget: RuntimeBudget, meter?: SpendMeter): Promise { const env = (rung.makeEnv ?? demoEnv)(); rung.setup?.(env); const rt = new CountingRoomTools(env.engine, env.roomId, env.artifactId, env.actor, env.sessionId, env.stats); @@ -593,6 +618,8 @@ async function runRung(rung: Rung, maker: (rung: Rung) => AgentModel, modelName: deadlineAt, reserveMs: budget.reserveMs, onTrace: (event) => rung.onTrace?.(event, env), + // A metered run hands the runtime the dollars still left job-wide and the strict (throwing) price. + ...(meter ? { spendLimits: { maxCostUsd: Math.max(0, meter.maxUsd - meter.spentUsd) }, priceStep: priceRunStrict } : {}), }); let result: AgentResult; let slice1: AgentResult | undefined; @@ -625,8 +652,9 @@ async function runRung(rung: Rung, maker: (rung: Rung) => AgentModel, modelName: const combinedTrace = slice1 ? [...slice1.trace, { step: -1, tool: "slice_boundary", args: { forced: true, slice1Exhausted: slice1.exhausted }, result: { handoff: slice1.handoff?.summary ?? null }, ms: 0 } as AgentTraceEvent, ...result.trace] : result.trace; - const cost = (result.usage ? priceRun(resolvedModel, result.usage.inputTokens, result.usage.outputTokens) : 0) - + (slice1?.usage ? priceRun(resolvedModel, slice1.usage.inputTokens, slice1.usage.outputTokens) : 0); + const price = meter ? priceRunStrict : priceRun; // a paid run never costs a guessed $1/$5 row + const cost = (result.usage ? price(resolvedModel, result.usage.inputTokens, result.usage.outputTokens) : 0) + + (slice1?.usage ? price(resolvedModel, slice1.usage.inputTokens, slice1.usage.outputTokens) : 0); return { pass, checkDetails, @@ -691,6 +719,7 @@ function parseRuntimeBudget(): RuntimeBudget { return { rungTimeoutMs: optionNumber("--rung-timeout-ms", "LADDER_RUNG_TIMEOUT_MS"), reserveMs: optionNumber("--reserve-ms", "LADDER_RESERVE_MS") ?? 5_000, + maxCostUsd: optionNumber("--max-cost-usd", "LADDER_MAX_COST_USD"), }; } @@ -737,17 +766,34 @@ async function parseModelRoutes(): Promise { return routes; } -function writeJsonReport(results: RungResult[], selectedRungs: Rung[], requestedRoutes: string[]): void { +/** What makes a --real report a receipt (SPEC A.0.3): the tested commit, the constant it tested, the spend, the policy. */ +interface ReportReceipt { + startedAt: string; + budget: LadderBudgetReport | null; + dispatchPolicy: DispatchPolicy | null; +} + +function sha256OfFile(path: string): string | null { + try { return createHash("sha256").update(readFileSync(path)).digest("hex"); } catch { return null; } +} + +function writeJsonReport(results: RungResult[], selectedRungs: Rung[], requestedRoutes: string[], receipt: ReportReceipt): void { const outPath = optionValue("--json-out"); if (!outPath) return; const absolute = resolve(process.cwd(), outPath); mkdirSync(dirname(absolute), { recursive: true }); + const real = receipt.budget !== null; + const identity = real ? readGitIdentity() : undefined; writeFileSync( absolute, JSON.stringify({ generatedAt: new Date().toISOString(), + startedAt: receipt.startedAt, requestedRoutes, rungs: selectedRungs.map((r) => ({ id: r.id, level: r.level, label: r.label })), + ...(identity ? { testedCommit: identity.commitSha, gitDirty: identity.gitDirty, constantSha256: sha256OfFile("convex/modelDefaults.ts") } : {}), + budget: receipt.budget, + dispatchPolicy: receipt.dispatchPolicy, results: results.map(publicResult), }, null, 2), ); @@ -817,7 +863,8 @@ function readGitIdentity(): GitIdentity { let commitSha = "nocommit"; let status = ""; let diff = ""; - try { commitSha = execSync("git rev-parse --short HEAD", { stdio: ["ignore", "pipe", "ignore"] }).toString().trim(); } catch { /* not a git repo */ } + // Full SHA: the receipt's testedCommit must be an ancestor-checkable 40-hex id (SPEC A.0.3), not a short prefix. + try { commitSha = execSync("git rev-parse HEAD", { stdio: ["ignore", "pipe", "ignore"] }).toString().trim(); } catch { /* not a git repo */ } try { status = execSync("git status --porcelain=v1", { stdio: ["ignore", "pipe", "ignore"] }).toString(); } catch { /* not a git repo */ } try { diff = execSync("git diff --no-ext-diff --binary HEAD --", { stdio: ["ignore", "pipe", "ignore"], maxBuffer: 16 * 1024 * 1024 }).toString(); } catch { /* large or unavailable diff */ } const gitDirty = status.trim().length > 0; @@ -857,24 +904,117 @@ function normalizePath(value: string): string { return value.replace(/\\/g, "/"); } -async function main() { +/** Opening-context size and per-step growth of one rung, read from a scripted dry run (zero provider requests). */ +interface RungShape { + rung: Rung; + ctxTokens0: number; + growthTokens: number; + /** Provider-request upper bound for one model on this rung (resume rungs run two slices). */ + maxSteps: number; +} + +async function scriptedRungShape(rung: Rung, budget: RuntimeBudget): Promise { + const sizes: number[] = []; + const probe = (r: Rung): AgentModel => { + const base = r.scripted(); + return { + get name() { return base.name; }, + async next(input) { + sizes.push(requestInputTokenUpperBound(input.system, input.messages, input.tools)); + return base.next(input); + }, + }; + }; + await runRung(rung, probe, "scripted", { ...budget, maxCostUsd: undefined }); + const ctxTokens0 = sizes[0] ?? 0; + const growthTokens = sizes.length > 1 ? Math.max(0, ((sizes.at(-1) ?? 0) - ctxTokens0) / (sizes.length - 1)) : 0; + const maxSteps = (rung.maxSteps ?? 18) + (rung.resume?.slice1MaxSteps ?? 0); + return { rung, ctxTokens0, growthTokens, maxSteps }; +} + +/** + * Worst-case USD before the first paid request: + * Σ_models Σ_rungs Σ_{step=1..maxSteps} ((ctx0 + (step-1)·growth) · inPerM + maxOut · outPerM) / 1e6. + * ctx0 and growth come from the scripted dry run, so this is a pre-flight sanity bound ("can the cap cover a full + * grid at all?"), not the spend guard: the guard is the per-dispatch reservation, which prices each request's real + * input bytes. A free-auto route fans out over its candidates, so it multiplies by the candidate limit. The + * request count holds only under RECEIPT_DISPATCH_POLICY: with retries and fallback each step could issue several. + */ +export function estimateLadderUsd(routes: string[], shapes: RungShape[], freeAutoCandidates = 1): number { + let total = 0; + for (const route of routes) { + const pricing = getModelPricing(resolveModelAlias(route)); + if (!pricing) throw new Error(`no pricing row for "${route}"`); + const maxOut = modelMaxOutputTokens(resolveModelAlias(route)); + const fanOut = /free-auto$/i.test(route) ? Math.max(1, freeAutoCandidates) : 1; + for (const shape of shapes) { + for (let step = 1; step <= shape.maxSteps; step++) { + total += fanOut * (((shape.ctxTokens0 + (step - 1) * shape.growthTokens) * pricing.inputPer1M + maxOut * pricing.outputPer1M) / 1_000_000); + } + } + } + return total; +} + +export async function main() { const realModels = await parseModelRoutes(); const budget = parseRuntimeBudget(); - const models: { name: string; maker: (rung: Rung) => AgentModel }[] = realModels.length - ? realModels.map((id) => ({ name: id, maker: () => realModel(id) })) - : [{ name: "scripted", maker: (rung) => rung.scripted() }]; - + const startedAt = new Date().toISOString(); const rungFilter = parseRungFilter(); const selectedRungs = rungFilter ? RUNGS.filter((r) => rungFilter.has(r.id)) : RUNGS; + const real = realModels.length > 0; + + let meter: SpendMeter | undefined; + let estimateUsd = 0; + const budgetReport = (): LadderBudgetReport | null => (meter + ? { maxUsd: meter.maxUsd, estimateUsd, spentUsd: meter.spentUsd, requests: meter.requests, stoppedBy: meter.stoppedBy } + : null); + const refuse = (message: string) => { + console.error(`ladder: ${message}`); + writeJsonReport([], selectedRungs, realModels, { startedAt, budget: budgetReport(), dispatchPolicy: RECEIPT_DISPATCH_POLICY }); + process.exitCode = 1; + }; + + if (real) { + // A paid run needs a dollar cap, and every id it will call must have a verified price: no cap, no guessed row. + if (budget.maxCostUsd === undefined) { + console.error("ladder: --real needs a dollar cap (LADDER_MAX_COST_USD or --max-cost-usd). Refusing to start; no request was made."); + process.exitCode = 1; + return; + } + meter = createSpendMeter(budget.maxCostUsd); + const unpriced = realModels.find((id) => !getModelPricing(resolveModelAlias(id))); + if (unpriced) { + meter.stoppedBy = "unpriced_model"; + refuse(`no pricing row for "${unpriced}"; add a verified row (modelCatalog.modelPricing) before a paid run. No request was made.`); + return; + } + const shapes: RungShape[] = []; + for (const rung of selectedRungs) shapes.push(await scriptedRungShape(rung, budget)); + estimateUsd = estimateLadderUsd(realModels, shapes, optionNumber("--free-auto-top", "LADDER_FREE_AUTO_TOP") ?? 1); + console.log(`pre-flight: worst case $${estimateUsd.toFixed(4)} for ${realModels.length} route(s) x ${selectedRungs.length} rung(s); cap $${meter.maxUsd.toFixed(4)}`); + if (estimateUsd > meter.maxUsd) { + meter.stoppedBy = "spend_budget"; + refuse(`worst-case estimate $${estimateUsd.toFixed(4)} exceeds the $${meter.maxUsd.toFixed(4)} cap. Raise the cap or narrow --levels; no request was made.`); + return; + } + } + + const models: { name: string; maker: (rung: Rung) => AgentModel }[] = real + ? realModels.map((id) => ({ name: id, maker: () => realModel(id, { dispatch: RECEIPT_DISPATCH_POLICY, meter }) })) + : [{ name: "scripted", maker: (rung) => rung.scripted() }]; const grid: Record> = {}; const results: RungResult[] = []; for (const model of models) { grid[model.name] = {}; for (const rung of selectedRungs) { + if (meter?.stoppedBy) break; // the cap is reached: no further rung may spend process.stdout.write(` ${model.name} · ${rung.id} ... `); - const res = await runRung(rung, model.maker, model.name, budget); + const res = await runRung(rung, model.maker, model.name, budget, meter); results.push(res); + // The runtime's own pre-call gate can end a rung on the cap before the adapter is asked to dispatch. + if (meter && res.stopReason === "spend_budget") meter.stoppedBy ??= "spend_budget"; grid[model.name][rung.id] = res.pass; const stop = "stopReason" in res && res.stopReason !== "done" ? `, ${res.stopReason}` : ""; const resolved = res.resolvedModel !== model.name ? `, resolved=${res.resolvedModel}` : ""; @@ -891,13 +1031,17 @@ async function main() { } console.log("-".repeat(72)); console.log("legend: " + selectedRungs.map((r) => `L${r.level}=${r.label}`).join(" | ")); - writeJsonReport(results, selectedRungs, models.map((m) => m.name)); + if (meter) console.log(`spend: $${meter.spentUsd.toFixed(4)} of $${meter.maxUsd.toFixed(4)} over ${meter.requests} provider request(s)${meter.stoppedBy ? `; STOPPED by ${meter.stoppedBy} (not a receipt)` : ""}`); + writeJsonReport(results, selectedRungs, models.map((m) => m.name), { startedAt, budget: budgetReport(), dispatchPolicy: real ? RECEIPT_DISPATCH_POLICY : null }); recordRunsToStore(results); + if (meter?.stoppedBy) process.exitCode = 1; // a budget stop is an incomplete run, never a pass if (Object.values(grid).some((row) => Object.values(row).some((ok) => !ok))) process.exitCode = 1; } -main().catch((error) => { - console.error(error); - process.exit(1); -}); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error); + process.exit(1); + }); +} diff --git a/evals/ladderBudget.test.ts b/evals/ladderBudget.test.ts new file mode 100644 index 00000000..be16e8ae --- /dev/null +++ b/evals/ladderBudget.test.ts @@ -0,0 +1,251 @@ +/** + * Ladder spend budget (SPEC Part A §A.2.9 "Ladder budget", Codex R1, v4-new-2, v5-new-1). + * + * Tests the ACTUAL adapter path, not a fake AgentModel: `model(id, { dispatch, meter })` runs the real AI-SDK + * Google provider with `fetch` replaced by a fake transport that counts physical requests. The claims: + * - the meter moves at DISPATCHES (one reservation per physical request, retries and fallbacks included); + * - a request whose worst case would cross the cap is never issued; + * - the AI SDK's own retry loop adds no request of its own (pinned to 0); + * - an id the catalog cannot price is refused before any request, never reserved at a guessed price. + */ +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + createSpendMeter, + model, + priceRunStrict, + requestInputTokenUpperBound, + reserveDispatch, + SpendBudgetError, + UnpricedModelError, + type DispatchPolicy, +} from "../src/nodeagent/models/adapter"; +import { modelMaxOutputTokens } from "../src/nodeagent/models/outputTokens"; + +const PRICED = "gemini-3.8-flash"; +const UNPRICED = "gemini-9.9-unpriced-test"; // routes to the Google provider, has no pricing row +const RECEIPT: DispatchPolicy = { maxRetries: 0, crossModelFallback: false }; +const TURN = { system: "s", messages: [{ role: "user" as const, content: "hi" }], tools: [] }; +const BOUND = requestInputTokenUpperBound(TURN.system, TURN.messages, TURN.tools); +const RESERVE = reserveDispatch(PRICED, BOUND, Number.POSITIVE_INFINITY); + +type Call = { url: string; body: Record }; +let calls: Call[] = []; +let billedUsd = 0; + +function ok(inputTokens = 1_000, outputTokens = 200) { + return new Response(JSON.stringify({ + candidates: [{ content: { role: "model", parts: [{ text: "ok" }] }, finishReason: "STOP", index: 0 }], + usageMetadata: { promptTokenCount: inputTokens, candidatesTokenCount: outputTokens, totalTokenCount: inputTokens + outputTokens }, + }), { status: 200, headers: { "content-type": "application/json" } }); +} +const unavailable = () => new Response(JSON.stringify({ error: { code: 503, message: "Service unavailable", status: "UNAVAILABLE" } }), { status: 503, headers: { "content-type": "application/json" } }); + +function fakeTransport(answer: (n: number, call: Call) => Response | Promise) { + calls = []; + vi.stubGlobal("fetch", async (input: string | URL | Request, init?: RequestInit) => { + const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; + const call: Call = { url, body: JSON.parse(typeof init?.body === "string" ? init.body : "{}") }; + calls.push(call); + return answer(calls.length, call); + }); +} + +const SAVED = { key: process.env.GOOGLE_GENERATIVE_AI_API_KEY, fb: process.env.AGENT_FALLBACK_MODEL, cap: process.env.AGENT_MODEL_MAX_OUTPUT_TOKENS, ladderCap: process.env.LADDER_MAX_COST_USD }; +beforeEach(() => { + process.env.GOOGLE_GENERATIVE_AI_API_KEY = "test-google-key"; + delete process.env.AGENT_FALLBACK_MODEL; + delete process.env.AGENT_MODEL_MAX_OUTPUT_TOKENS; + delete process.env.LADDER_MAX_COST_USD; + billedUsd = 0; +}); +afterEach(() => { + vi.unstubAllGlobals(); + vi.useRealTimers(); + for (const [k, v] of [["GOOGLE_GENERATIVE_AI_API_KEY", SAVED.key], ["AGENT_FALLBACK_MODEL", SAVED.fb], ["AGENT_MODEL_MAX_OUTPUT_TOKENS", SAVED.cap], ["LADDER_MAX_COST_USD", SAVED.ladderCap]] as const) { + if (v === undefined) delete process.env[k]; else process.env[k] = v; + } +}); + +describe("meter moves at dispatches (adapter path, fake transport)", () => { + it("(a) a charged response then ECONNRESET under the receipt policy: exactly two dispatches, error propagates, failed reservation is kept", async () => { + fakeTransport((n) => { + if (n === 1) return ok(1_000, 200); + throw Object.assign(new Error("fetch failed: ECONNRESET"), { code: "ECONNRESET" }); + }); + const meter = createSpendMeter(10); + const m = model(PRICED, { dispatch: RECEIPT, meter }); + + await m.next(TURN); + expect(meter.requests).toBe(1); + expect(meter.spentUsd).toBeCloseTo(priceRunStrict(PRICED, 1_000, 200), 12); // reservation replaced by the priced usage + + await expect(m.next(TURN)).rejects.toThrow(/ECONNRESET/); + expect(calls).toHaveLength(2); // maxRetries 0: no retry, no SDK retry, no fallback + expect(meter.requests).toBe(2); + expect(meter.spentUsd).toBeCloseTo(priceRunStrict(PRICED, 1_000, 200) + RESERVE, 12); // the failed dispatch's reservation is retained + }); + + it("(b) the same outage under the default policy reserves once per attempt and never crosses the cap", async () => { + vi.useFakeTimers(); + fakeTransport(() => unavailable()); + const meter = createSpendMeter(RESERVE * 2.5); // room for two reservations, not three + const m = model(PRICED, { meter }); // default policy: 3 retries, fallback allowed (no AGENT_FALLBACK_MODEL set) + + const outcome = m.next(TURN).then(() => null, (e: unknown) => e); + await vi.advanceTimersByTimeAsync(120_000); + const error = await outcome; + + expect(error).toBeInstanceOf(SpendBudgetError); + expect((error as SpendBudgetError).receipt.stopReason).toBe("spend_budget"); + expect(calls).toHaveLength(2); // the attempt whose reservation would cross the cap is never issued + expect(meter.requests).toBe(2); + expect(meter.spentUsd).toBeLessThanOrEqual(meter.maxUsd); + expect(meter.stoppedBy).toBe("spend_budget"); + }); + + it("(c) a reservation above the remaining budget before the first dispatch throws spend_budget with zero fetch calls", async () => { + fakeTransport(() => ok()); + const meter = createSpendMeter(RESERVE / 2); + await expect(model(PRICED, { dispatch: RECEIPT, meter }).next(TURN)).rejects.toMatchObject({ receipt: { stopReason: "spend_budget" } }); + expect(calls).toHaveLength(0); + expect(meter.requests).toBe(0); + expect(meter.spentUsd).toBe(0); + }); + + it("(d) every issued request body carries the output cap the reservation priced", async () => { + fakeTransport(() => ok()); + const meter = createSpendMeter(10); + const m = model(PRICED, { dispatch: RECEIPT, meter }); + await m.next(TURN); + await m.next(TURN); + expect(calls.map((c) => c.body.generationConfig?.maxOutputTokens)).toEqual([modelMaxOutputTokens(), modelMaxOutputTokens()]); + process.env.AGENT_MODEL_MAX_OUTPUT_TOKENS = "2048"; + await m.next(TURN); + expect(calls.at(-1)!.body.generationConfig.maxOutputTokens).toBe(2048); + }); + + it("(f) retryable 503s under { maxRetries: 3 }: exactly four requests, four reservations; the SDK adds none", async () => { + vi.useFakeTimers(); + fakeTransport(() => unavailable()); + const meter = createSpendMeter(10); + const m = model(PRICED, { dispatch: { maxRetries: 3, crossModelFallback: false }, meter }); + const outcome = m.next(TURN).then(() => null, (e: unknown) => e); + await vi.advanceTimersByTimeAsync(120_000); + expect(await outcome).toBeInstanceOf(Error); + expect(calls).toHaveLength(4); + expect(meter.requests).toBe(4); + expect(meter.spentUsd).toBeCloseTo(RESERVE * 4, 12); + }); +}); + +describe("(g) zero requests for an unpriced id", () => { + it("model() with a $10 meter and at the $0.01 equality case: UnpricedModelError, zero fetch calls, meter untouched", async () => { + fakeTransport(() => ok()); // the transport would answer 200 if anything were sent + for (const maxUsd of [10, 0.01]) { + const meter = createSpendMeter(maxUsd); + const error = await model(UNPRICED, { dispatch: RECEIPT, meter }).next(TURN).then(() => null, (e: unknown) => e); + expect(error).toBeInstanceOf(UnpricedModelError); + expect((error as UnpricedModelError).receipt.stopReason).toBe("spend_budget"); + expect(meter.requests).toBe(0); + expect(meter.spentUsd).toBe(0); + expect(meter.stoppedBy).toBe("unpriced_model"); + } + expect(calls).toHaveLength(0); + }); + + it("a priced primary answering four 503s with an unpriced AGENT_FALLBACK_MODEL makes the primary's four requests and none for the fallback", async () => { + vi.useFakeTimers(); + process.env.AGENT_FALLBACK_MODEL = UNPRICED; + fakeTransport(() => unavailable()); + const meter = createSpendMeter(10); + const outcome = model(PRICED, { meter }).next(TURN).then(() => null, (e: unknown) => e); // default policy + await vi.advanceTimersByTimeAsync(120_000); + expect(await outcome).toBeInstanceOf(UnpricedModelError); + expect(calls).toHaveLength(4); + expect(calls.every((c) => c.url.includes(`/models/${PRICED}:`))).toBe(true); + expect(meter.requests).toBe(4); + }); + + it("an unmetered adapter is unchanged: no reservation, no refusal", async () => { + fakeTransport(() => ok()); + const step = await model(UNPRICED).next(TURN); // no meter: legacy callers are not budget-gated + expect(step.text).toBe("ok"); + expect(calls).toHaveLength(1); + }); +}); + +describe("main(): pre-flight and the grid", () => { + let dir: string; + const argvBefore = process.argv; + const exitBefore = process.exitCode; + beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "ladder-budget-")); vi.spyOn(console, "log").mockImplementation(() => {}); vi.spyOn(console, "error").mockImplementation(() => {}); }); + afterEach(() => { process.argv = argvBefore; process.exitCode = exitBefore; rmSync(dir, { recursive: true, force: true }); vi.restoreAllMocks(); }); + + async function runMain(args: string[]) { + const out = join(dir, "report.json"); + process.argv = ["node", "ladder.ts", ...args, "--json-out", out]; + process.exitCode = undefined; + const { main } = await import("./ladder"); + await main(); + const exitCode = process.exitCode; + process.exitCode = exitBefore; + let report: Record | undefined; + try { report = JSON.parse(readFileSync(out, "utf8")); } catch { /* refusal before any report */ } + return { exitCode, report }; + } + + it("refuses --real without a dollar cap: exit 1, no request", async () => { + fakeTransport(() => ok()); + const { exitCode } = await runMain(["--real", PRICED, "--levels=1-4"]); + expect(exitCode).toBe(1); + expect(calls).toHaveLength(0); + }); + + it("(g) --real with an unpriced id exits 1 at pre-flight: stoppedBy unpriced_model, no result rows, zero fetch calls", async () => { + fakeTransport(() => ok()); + const { exitCode, report } = await runMain(["--real", UNPRICED, "--levels=1-4", "--max-cost-usd=1"]); + expect(exitCode).toBe(1); + expect(report?.budget).toMatchObject({ stoppedBy: "unpriced_model", requests: 0, spentUsd: 0 }); + expect(report?.results).toEqual([]); + expect(report?.dispatchPolicy).toEqual(RECEIPT); + expect(calls).toHaveLength(0); + }); + + it("refuses to start when the worst-case estimate exceeds the cap (the human raises the cap, never the check)", async () => { + fakeTransport(() => ok()); + const { exitCode, report } = await runMain(["--real", PRICED, "--levels=1-4", "--max-cost-usd=0.0001"]); + expect(exitCode).toBe(1); + expect(report?.budget.estimateUsd).toBeGreaterThan(0.0001); + expect(report?.budget).toMatchObject({ stoppedBy: "spend_budget", requests: 0 }); + expect(calls).toHaveLength(0); + }); + + it("(e) a grid driven through the adapter stops before the cap; budget.requests equals the fetch count; a stop is not a receipt", async () => { + // Every answer is a valid tool call (list_artifacts takes no arguments, so the loop keeps going) that bills ~$2.25 + // of input (3M tokens at $0.75/M), far above its reservation: the cap is crossed by settlement, and every later + // dispatch must be refused instead of issued. + const cap = 5; + fakeTransport(() => { + billedUsd += 2.25; + return new Response(JSON.stringify({ + candidates: [{ content: { role: "model", parts: [{ functionCall: { name: "list_artifacts", args: {} } }] }, finishReason: "STOP", index: 0 }], + usageMetadata: { promptTokenCount: 3_000_000, candidatesTokenCount: 10, totalTokenCount: 3_000_010 }, + }), { status: 200, headers: { "content-type": "application/json" } }); + }); + let issuedAfterCap = 0; + const base = globalThis.fetch; + vi.stubGlobal("fetch", async (...args: Parameters) => { if (billedUsd >= cap) issuedAfterCap += 1; return base(...args); }); + + const { exitCode, report } = await runMain(["--real", PRICED, "--levels=1-4", `--max-cost-usd=${cap}`]); + expect(calls.length).toBeGreaterThan(2); // the cap is crossed by settlement, not before the first answers + expect(issuedAfterCap).toBe(0); + expect(report?.budget.requests).toBe(calls.length); + expect(report?.budget.stoppedBy).toBe("spend_budget"); + expect(report?.dispatchPolicy).toEqual(RECEIPT); + expect(report?.testedCommit).toMatch(/^[0-9a-f]{40}$/); + expect(exitCode).toBe(1); + }); +}); diff --git a/package-lock.json b/package-lock.json index 140d6c65..8f5db488 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "0.1.1", "license": "MIT", "dependencies": { - "@ai-sdk/anthropic": "^3.0.84", + "@ai-sdk/anthropic": "^3.0.120", "@ai-sdk/google": "^3.0.82", "@ai-sdk/openai": "^3.0.71", "@assistant-ui/react-o11y": "^0.0.25", @@ -99,13 +99,13 @@ } }, "node_modules/@ai-sdk/anthropic": { - "version": "3.0.117", - "resolved": "https://registry.npmjs.org/@ai-sdk/anthropic/-/anthropic-3.0.117.tgz", - "integrity": "sha512-ZdcpT3tWRifu93DPL5jSGvx9QmRi4zi0H7JiCKp76dH6dytQKYdg12Jpl1HHamCT3KgxSrNMOPKCv8q+gXqAVw==", + "version": "3.0.120", + "resolved": "https://registry.npmjs.org/@ai-sdk/anthropic/-/anthropic-3.0.120.tgz", + "integrity": "sha512-nDdCokLr0nexRUwUpWzzXzK73+KbejgmZMJdo8XthIeeeCXQvtKuhvpNs091N97NEaM3XHmOI55Z8/TWuNSWcQ==", "license": "Apache-2.0", "dependencies": { "@ai-sdk/provider": "3.0.16", - "@ai-sdk/provider-utils": "4.0.51" + "@ai-sdk/provider-utils": "4.0.52" }, "engines": { "node": ">=18" @@ -114,6 +114,33 @@ "zod": "^3.25.76 || ^4.1.8" } }, + "node_modules/@ai-sdk/anthropic/node_modules/@ai-sdk/provider-utils": { + "version": "4.0.52", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-4.0.52.tgz", + "integrity": "sha512-cUimyIz1jjwYoF9n6HO7sU+UBW7Nu3T4lW7562yRvEhM9/8JqlW4cecxSpkiafjRzAphZG2NfJ3/Ch5cHoCIhg==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.16", + "@standard-schema/spec": "^1.1.0", + "eventsource-parser": "^3.0.8", + "undici": "^6.28.0" + }, + "engines": { + "node": ">=18.17" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/anthropic/node_modules/undici": { + "version": "6.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.29.0.tgz", + "integrity": "sha512-R+RODBqp6i2pPflGdq+xIOUkl+RNfGgHwoinecKu/JCuf2uO06cOKoDbI2P7Dn6KcswdKwrczbU6IYJ6K8X+wg==", + "license": "MIT", + "engines": { + "node": ">=18.17" + } + }, "node_modules/@ai-sdk/gateway": { "version": "3.0.191", "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-3.0.191.tgz", diff --git a/package.json b/package.json index a0b382fd..4b866fac 100644 --- a/package.json +++ b/package.json @@ -250,7 +250,7 @@ "eval:credit": "tsx evals/creditEval.ts", "ladder": "tsx evals/ladder.ts", "ladder:real": "tsx evals/ladder.ts --real", - "ladder:free": "tsx evals/ladder.ts --real openrouter/free-auto --free-auto-top=5 --levels=1-4 --rung-timeout-ms=540000 --reserve-ms=30000 --json-out docs/eval/free-auto-router-ladder.json", + "ladder:free": "tsx evals/ladder.ts --real openrouter/free-auto --free-auto-top=5 --max-cost-usd=1 --levels=1-4 --rung-timeout-ms=540000 --reserve-ms=30000 --json-out docs/eval/free-auto-router-ladder.json", "eval:diff": "tsx evals/evalDiff.ts", "proofs:staleness": "tsx scripts/proof-staleness-check.ts", "proofs:publish": "tsx scripts/proof-release-publish.ts --write-readme", @@ -275,7 +275,7 @@ "voice:prod-deploy:proof": "tsx scripts/voice-prod-deploy-proof.ts" }, "dependencies": { - "@ai-sdk/anthropic": "^3.0.84", + "@ai-sdk/anthropic": "^3.0.120", "@ai-sdk/google": "^3.0.82", "@ai-sdk/openai": "^3.0.71", "@assistant-ui/react-o11y": "^0.0.25", diff --git a/packages/walkthrough-review-cli/README.md b/packages/walkthrough-review-cli/README.md index b5b03896..a5654cf6 100644 --- a/packages/walkthrough-review-cli/README.md +++ b/packages/walkthrough-review-cli/README.md @@ -22,7 +22,7 @@ Generic shape: walkthrough-review run \ --config walkthrough-review.config.json \ --base http://127.0.0.1:5178 \ - --model gemini-3.5-flash \ + --model gemini-3.6-flash \ --ui-review \ startup-diligence-war-room ``` diff --git a/packages/walkthrough-review-cli/src/core.ts b/packages/walkthrough-review-cli/src/core.ts index 4f70b55f..16e40ad0 100644 --- a/packages/walkthrough-review-cli/src/core.ts +++ b/packages/walkthrough-review-cli/src/core.ts @@ -75,7 +75,7 @@ type CommandVariables = { }; const DEFAULT_CONFIG = "walkthrough-review.config.json"; -const DEFAULT_MODEL = "gemini-3.5-flash"; +const DEFAULT_MODEL = "gemini-3.6-flash"; const DEFAULT_OUTPUT_ROOT = join("docs", "eval", "walkthrough-review"); const DEFAULT_DRY_OUTPUT_ROOT = join(".tmp-qa", "walkthrough-review"); const DEFAULT_MEDIA_ROOT = join("docs", "walkthroughs"); diff --git a/scripts/benchmark/probe.ts b/scripts/benchmark/probe.ts index 8bd2755a..0bc8ba7b 100644 --- a/scripts/benchmark/probe.ts +++ b/scripts/benchmark/probe.ts @@ -9,23 +9,28 @@ const { google } = await import("@ai-sdk/google"); const or = createOpenAI({ apiKey: process.env.OPENROUTER_API_KEY, baseURL: "https://openrouter.ai/api/v1" }); const cases: [string, any][] = [ - // Anthropic latest + // Anthropic latest (5.5 = the migration-guide successors; claude-opus-4-8 is still Active) + ["claude-haiku-5-5", anthropic("claude-haiku-5-5")], + ["claude-sonnet-5-5", anthropic("claude-sonnet-5-5")], + ["claude-opus-5-5", anthropic("claude-opus-5-5")], ["claude-opus-4-8", anthropic("claude-opus-4-8")], ["claude-sonnet-4-6", anthropic("claude-sonnet-4-6")], ["claude-haiku-4-5", anthropic("claude-haiku-4-5")], - // Google latest (gemini 3.x) + // Google latest (gemini 3.x). gemini-3.5-flash stays on purpose: Google auto-routes it to 3.6 since 2026-10-08. + ["gemini-3.8-flash", google("gemini-3.8-flash")], + ["gemini-3.5-flash-lite", google("gemini-3.5-flash-lite")], ["gemini-3.5-flash", google("gemini-3.5-flash")], ["gemini-3-flash-preview", google("gemini-3-flash-preview")], ["gemini-3.1-flash-lite", google("gemini-3.1-flash-lite")], ["gemini-2.5-flash", google("gemini-2.5-flash")], ["gemini-2.5-flash-lite", google("gemini-2.5-flash-lite")], - // OpenAI latest (gpt-5.x) + // OpenAI latest (gpt-5.x / gpt-6.x) + ["gpt-6.1-sol", openai("gpt-6.1-sol")], ["gpt-5.5", openai("gpt-5.5")], ["gpt-5.4", openai("gpt-5.4")], ["gpt-5.4-mini", openai("gpt-5.4-mini")], ["gpt-5.4-nano", openai("gpt-5.4-nano")], // OpenRouter (cheap/free) — simple-call reachability - ["OR nex-agi/nex-n2-pro:free", or.chat("nex-agi/nex-n2-pro:free")], ["OR ibm-granite/granite-4.1-8b", or.chat("ibm-granite/granite-4.1-8b")], ["OR deepseek/deepseek-v4-flash", or.chat("deepseek/deepseek-v4-flash")], ["OR z-ai/glm-4.7-flash", or.chat("z-ai/glm-4.7-flash")], diff --git a/scripts/design-qa/rubric-panel.ts b/scripts/design-qa/rubric-panel.ts index d4921683..1a7aae57 100644 --- a/scripts/design-qa/rubric-panel.ts +++ b/scripts/design-qa/rubric-panel.ts @@ -28,7 +28,7 @@ const opt = (n: string) => args.find((a) => a.startsWith(`--${n}=`))?.slice(n.le const mediaPath = opt("media"); const refPath = opt("reference"); const floorPath = opt("floor"); -const model = process.env.GEMINI_UI_REVIEW_MODEL ?? "gemini-3.5-flash"; +const model = process.env.GEMINI_UI_REVIEW_MODEL ?? "gemini-3.6-flash"; const outPath = opt("out") ?? join(process.cwd(), ".tmp-qa", "design-rubric.json"); if (!mediaPath) throw new Error("--media= required"); if (!process.env.GOOGLE_GENERATIVE_AI_API_KEY) throw new Error("GOOGLE_GENERATIVE_AI_API_KEY required"); @@ -72,7 +72,7 @@ async function judgeLens(lens: typeof LENSES[number]) { { type: "file", data: candidate, filename: "candidate.png", mediaType: "image/png" }, ]; if (reference) content.push({ type: "file", data: reference, filename: "reference.png", mediaType: "image/png" }); - const { object } = await generateObject({ model: google(model), schema: lensSchema(lens.dims), messages: [{ role: "user", content }], temperature: 0.2 }); + const { object } = await generateObject({ model: google(model), schema: lensSchema(lens.dims), messages: [{ role: "user", content }] }); return { lens: lens.key, ...object }; } diff --git a/scripts/eval/painJudge.ts b/scripts/eval/painJudge.ts index db3283f9..e7f2c04f 100644 --- a/scripts/eval/painJudge.ts +++ b/scripts/eval/painJudge.ts @@ -5,7 +5,7 @@ * point is addressed. This is the independent "does the live workflow truly relieve the * pain" check, before the Remotion demo. * - * npx tsx scripts/eval/painJudge.ts # gemini-3.5-flash + * npx tsx scripts/eval/painJudge.ts # gemini-3.6-flash * JUDGE_MODEL=gemini-3.1-flash-lite npx tsx scripts/eval/painJudge.ts */ import "../benchmark/loadEnv"; @@ -15,7 +15,7 @@ import { generateText } from "ai"; import { createGoogleGenerativeAI } from "@ai-sdk/google"; const ROOT = process.cwd(); -const MODEL = process.env.JUDGE_MODEL || "gemini-3.5-flash"; +const MODEL = process.env.JUDGE_MODEL || "gemini-3.6-flash"; const google = createGoogleGenerativeAI({ apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY }); const FRAMES: [string, string][] = [ diff --git a/scripts/eval/providerParseSmoke.ts b/scripts/eval/providerParseSmoke.ts index faac869d..889f3730 100644 --- a/scripts/eval/providerParseSmoke.ts +++ b/scripts/eval/providerParseSmoke.ts @@ -5,7 +5,7 @@ * Verifies the architecture invariant: canonical Convex storageId != disposable provider file id. * * npx tsx scripts/eval/providerParseSmoke.ts # gemini default - * npx tsx scripts/eval/providerParseSmoke.ts gemini-3.5-flash,gpt-5.4-mini,claude-haiku-4-5 + * npx tsx scripts/eval/providerParseSmoke.ts gemini-3.8-flash,gpt-5.4-mini,claude-haiku-5-5 */ import "../benchmark/loadEnv"; import { z } from "zod"; @@ -18,7 +18,7 @@ import { priceRun } from "../../src/nodeagent/models/adapter"; import { artifactsFromProviderExtraction, providerFileCacheMeta, type CanonicalFileRef, type ProviderExtraction, type ProviderUploadResult } from "../../src/app/providerParserAdapter"; import type { CellPayload, ProviderParser } from "../../src/engine/types"; -const MODELS = (process.argv[2] || "gemini-3.5-flash,gpt-5.4-mini,claude-haiku-4-5").split(",").map((s) => s.trim()); +const MODELS = (process.argv[2] || "gemini-3.8-flash,gpt-5.4-mini,claude-haiku-5-5").split(",").map((s) => s.trim()); // Synthetic messy diligence text — the ParselyFi/PitchBook archetype (unstructured → structured), no private data. const SAMPLE = `Diligence scratch notes (messy): diff --git a/scripts/gemini-demo-media-judge.ts b/scripts/gemini-demo-media-judge.ts index e1c209fc..74052e76 100644 --- a/scripts/gemini-demo-media-judge.ts +++ b/scripts/gemini-demo-media-judge.ts @@ -18,7 +18,7 @@ import { z } from "zod"; const ROOT = process.cwd(); const args = process.argv.slice(2); const invokedCommand = ["npm", "run", "media:gemini-judge", "--", ...args].map(quoteCommandArg).join(" "); -const model = optionValue("--model") ?? process.env.GEMINI_MEDIA_JUDGE_MODEL ?? "gemini-3.5-flash"; +const model = optionValue("--model") ?? process.env.GEMINI_MEDIA_JUDGE_MODEL ?? "gemini-3.6-flash"; const runId = optionValue("--run-id") ?? timestampId(new Date()); const only = optionValue("--only")?.toLowerCase(); const inputPaths = optionValues("--input"); @@ -238,7 +238,6 @@ async function judgeAsset(asset: Asset): Promise { model: google(model), schema: judgeSchema, messages, - temperature: 0.2, }); const judge = normalizeJudge(result.object); return { diff --git a/scripts/gemini-mp4-workflow-review.ts b/scripts/gemini-mp4-workflow-review.ts index ed6c8a44..56928617 100644 --- a/scripts/gemini-mp4-workflow-review.ts +++ b/scripts/gemini-mp4-workflow-review.ts @@ -12,7 +12,7 @@ import { z } from "zod"; const args = process.argv.slice(2); const mediaPath = optionValue("--media"); const outPath = optionValue("--out") ?? join(process.cwd(), "docs", "eval", "agent-improvement-loop", "mp4-workflow-review.json"); -const model = optionValue("--model") ?? process.env.GEMINI_UI_REVIEW_MODEL ?? "gemini-3.5-flash"; +const model = optionValue("--model") ?? process.env.GEMINI_UI_REVIEW_MODEL ?? "gemini-3.6-flash"; if (!mediaPath) throw new Error("--media=