diff --git a/.env.example b/.env.example index 5ee72f42..beb105a8 100644 --- a/.env.example +++ b/.env.example @@ -59,10 +59,13 @@ AGENT_QUALITY_CANDIDATE_TIMEOUT_MS=45000 OPENROUTER_FREE_MODEL_CACHE_MS=600000 OPENROUTER_REQUIRE_NO_TRAINING=1 OPENROUTER_FREE_ALLOW_FILE_EGRESS=0 -# Provider-neutral NodeAgent uses OpenRouter's governed free ladder first, then -# this direct Gemini route when free routes are unavailable. Receipts retain the -# catalog price because an account's free-tier entitlement cannot be asserted. -NODEAGENT_FREE_AUTO_GOOGLE_MODEL=gemini-3.5-flash +# Provider-neutral NodeAgent uses OpenRouter's governed free ladder only. This +# direct Gemini route is an opt-in paid leg: it is taken only when the caller sets +# allowPaidFallback AND passes a spend meter (adapter.ts), and no production caller +# does either, so by default a free-route outage is an error, not a paid request. +# Receipts retain the catalog price because an account's free-tier entitlement +# cannot be asserted. +NODEAGENT_FREE_AUTO_GOOGLE_MODEL=gemini-3.8-flash AGENT_MODEL=gpt-5.4-nano # Optional voice provider overrides. Browser TTS is the free client default; # Convex `/voice/*` endpoints are authenticated hosted fallbacks. @@ -81,7 +84,7 @@ VOICE_CLOUDFLARE_TTS_MODEL=@cf/myshell-ai/melotts VOICE_CLOUDFLARE_TTS_LANG=en GEMINI_API_KEY= VOICE_GEMINI_TTS_FREE_TIER_CONFIRMED=0 -VOICE_GEMINI_TTS_MODEL=gemini-3.1-flash-tts-preview +VOICE_GEMINI_TTS_MODEL=gemini-3.8-flash-tts VOICE_GEMINI_TTS_VOICE=Kore VOICE_GEMINI_TTS_INSTRUCTIONS=Read clearly and concisely for a collaborative work room. GOOGLE_CLOUD_TTS_API_KEY= @@ -113,7 +116,7 @@ SEO_QA_GOOGLE_TARGET_PHRASE=NodeRoom collaborative AI room GOOGLE_SEARCH_CONSOLE_SITE_URL=https://noderoom.live/ GOOGLE_SEARCH_CONSOLE_ACCESS_TOKEN= GOOGLE_SEARCH_CONSOLE_SERVICE_ACCOUNT_JSON= -GEMINI_SEO_VIDEO_JUDGE_MODEL=gemini-3.5-flash +GEMINI_SEO_VIDEO_JUDGE_MODEL=gemini-3.6-flash NEBIUS_SEO_JUDGE_MODEL=nebius/MiniMaxAI/MiniMax-M2.5 NEBIUS_SMOKE_MODEL=nebius/MiniMaxAI/MiniMax-M2.5 NEBIUS_ENABLE_VISION_JUDGE=0 @@ -162,8 +165,8 @@ FREE_JOB_GOAL= FREE_JOB_TIMEOUT_MS=900000 # Optional server-side parser model overrides. -PROVIDER_PARSER_GEMINI_MODEL=gemini-3.5-flash +PROVIDER_PARSER_GEMINI_MODEL=gemini-3.8-flash,gemini-3.6-flash PROVIDER_PARSER_OPENAI_MODEL=gpt-5.4-mini -PROVIDER_PARSER_ANTHROPIC_MODEL=claude-haiku-4-5 -PROVIDER_PARSER_OPENROUTER_MODEL=z-ai/glm-5.2,moonshotai/kimi-k2.7-code,cohere/north-mini-code:free,nvidia/nemotron-3-ultra-550b-a55b,nex-agi/nex-n2-pro +PROVIDER_PARSER_ANTHROPIC_MODEL=claude-haiku-5-5,claude-sonnet-5-5 +PROVIDER_PARSER_OPENROUTER_MODEL=z-ai/glm-5.2,moonshotai/kimi-k2.7-code,cohere/north-mini-code:free,nvidia/nemotron-3-ultra-550b-a55b PROVIDER_PARSER_ALLOW_FILE_EGRESS=0 diff --git a/.tours/01-primary-user-flow.tour b/.tours/01-primary-user-flow.tour index cdd9f11d..00eb3a0f 100644 --- a/.tours/01-primary-user-flow.tour +++ b/.tours/01-primary-user-flow.tour @@ -27,14 +27,14 @@ }, { "file": "src/ui/Chat.tsx", - "line": 1878, + "line": 1879, "pattern": "const send = \\(raw\\?: string", "title": "The primary user action", "description": "`send` is what the Send button calls. Two separate things happen and it matters that they are separate:\n\n1. The message is posted to the room, with a client-generated `clientMsgId` — that id is how a retry updates the same message instead of double-posting it.\n2. Only then is the text inspected to see whether it was addressed to the assistant." }, { "file": "src/ui/Chat.tsx", - "line": 1166, + "line": 1167, "pattern": "function parsePublicNodeAgentRequest", "title": "Where text becomes intent", "description": "The assistant that can edit the shared sheet is addressable ONLY as a leading `@nodeagent …` directive. Mentioning it mid-sentence does not summon it.\n\nThis decides intent. It does not decide permission — that happens on the server, in the next step but one." @@ -48,7 +48,7 @@ }, { "file": "convex/agentJobs.ts", - "line": 2782, + "line": 2799, "pattern": "export const startPublicAsk = mutation\\(", "title": "Where input becomes trusted (live tier)", "description": "This is the real trust boundary. Convex validators reject a malformed payload before a job row exists, and `requireActorProof` checks that this browser is actually a member of this room.\n\nThe client-side parse two steps back exists to decide what the user meant. This decides what they are allowed to do." diff --git a/.tours/03-debug-and-recovery.tour b/.tours/03-debug-and-recovery.tour index 83e77502..2aa1316b 100644 --- a/.tours/03-debug-and-recovery.tour +++ b/.tours/03-debug-and-recovery.tour @@ -27,21 +27,21 @@ }, { "file": "src/ui/Chat.tsx", - "line": 269, + "line": 270, "pattern": "const MAX_FAILED_SENDS = 50;", "title": "Failure 3 — a message fails to post, and the list is bounded", "description": "`MAX_FAILED_SENDS = 50`, FIFO. Every in-memory collection on an agent path needs a cap and an eviction rule; agents loop, and an unbounded list fills memory in minutes rather than months." }, { "file": "src/ui/Chat.tsx", - "line": 1943, + "line": 1944, "pattern": "const retrySend = \\(cid: string, text: string\\)", "title": "Retry that cannot double-post", "description": "`retrySend` re-sends with the SAME `clientMsgId`. That id is what makes the retry idempotent at the store, so a user hammering Retry on a flaky connection does not produce five copies of their message." }, { "file": "src/ui/Chat.tsx", - "line": 935, + "line": 936, "pattern": "function buildAgentFailureNotice\\(", "title": "Failure 4 — the agent fails", "description": "`buildAgentFailureNotice` turns a provider error into a sentence a non-engineer can act on, carrying which model selection was used, what was asked, and which job it was. A stack trace on this surface is a defect, not a debug aid — this is a surface where a user decides whether to trust the assistant." diff --git a/CHANGELOG/README.md b/CHANGELOG/README.md index c0a25e56..03bd9e91 100644 --- a/CHANGELOG/README.md +++ b/CHANGELOG/README.md @@ -23,6 +23,7 @@ entries have not been invented or backfilled. ## Quality assurance - [Benchmark timeout budgets](qa/benchmark-timeout-budgets.md) +- [Model default refresh](qa/model-refresh-noderoom.md) - [NodeKit note reference proof](qa/nodekit-note-reference-proof.md) See [TEMPLATE.md](TEMPLATE.md) before adding or editing a lane. diff --git a/CHANGELOG/qa/model-refresh-noderoom.md b/CHANGELOG/qa/model-refresh-noderoom.md new file mode 100644 index 00000000..ff1c27f6 --- /dev/null +++ b/CHANGELOG/qa/model-refresh-noderoom.md @@ -0,0 +1,202 @@ +# Model default refresh (NodeRoom) + +Append-only lane for the 2026-10 refresh of NodeRoom's default model ids +(model-defaults SPEC Part A, section A.2.9). + +## 2026-10-14 — Refresh stale model defaults and fix Claude 5.5 request shapes + +Google deprecated `gemini-3.5-flash` on 2026-10-08 and OpenRouter drops +`google/gemini-2.5-*` on 2026-10-20, so the agent default moves to one constant, +`DEFAULT_AGENT_MODEL` (`convex/modelDefaults.ts`, `gemini-3.8-flash`). Claude +defaults move to the 5.5 ids, the dotted-id regex in `resolveModelAlias` becomes +an explicit table, forced `tool_choice` is downgraded to `auto` for Sonnet/Opus +5.5, and `evals/ladder.ts` now refuses a `--real` run without a dollar cap and +reserves cost at every provider dispatch. Merge waits for the L1-L4 ladder +receipt `docs/eval/ladder/model-refresh-noderoom.json`; until it exists +`tests/qaMatrix.test.ts` stays red on purpose, and the fast lane in +`adaptiveRouter.ts` stays on `gemini-3.5-flash`. + +**Commit**: `922fdb6`. **Author**: Homen Shum. + +## 2026-10-14 — Review round 1: merge blockers, receipt fields, tiered pricing + +PR 261 is a draft and carries the `needs-live-eval` label; both stay until the three +live checks below exist (SPEC A.0.3: the label is a block, not a note): + +1. the L1-L4 ladder receipt `docs/eval/ladder/model-refresh-noderoom.json`; +2. the manual listen check of `gemini-3.8-flash-tts` (`convex/http.ts:36`, A.0.7); +3. the observed Environment smoke that OpenRouter accepts `anthropic/claude-sonnet-5.5` + for `top_paid` with `tool_choice: auto` and `strict: true` tool definitions. + +Fixes: the `--json-out` report now carries `command`, `candidateIds`, `echoedModels`, +per-dispatch `rows` (provider-echoed model, usage, cost, `fallbackUsed`), `environment` +and `budget.requestCeiling`, so a receipt can be judged against A.0.3. The echoed model is +read from the raw response body (`modelVersion` / `model`) because the AI SDK's +`response.modelId` is the requested id for Google. A refused reservation is terminal for the +meter: no retry, fallback model or candidate may dispatch after it, and the provider-neutral +Google fallback honours `crossModelFallback: false`. `claude-haiku-5-5` is priced +tier-aware (`longContext`, $0.50/$2.50 above 100K prompt tokens) in every cost and +reservation path. Sonnet/Opus 5.5 send `strict: true` tool definitions wherever a forced +`tool_choice` is downgraded to `auto`; the AI-SDK capture reasoner and provider parser send +the explicit §1.3 effort. `.env.example` copies of the changed defaults now match the code. + +**Commit**: the commit after `71a1f15` on this branch. **Author**: Homen Shum. + +## 2026-10-15 — Review round 2: per-request tiers, call-site tests, free-auto ceiling + +The long-context price tier is now chosen per provider request only (`priceRequest`, +`convexPriceRequest`, reservation, meter settlement, runtime `priceStep`). Usage summed over +several requests is priced at base-tier rates (`priceRun`, `convexPriceRun`, `creditModel`, +`proofloopModelTracking`), and the ladder sums settled dispatch costs, so two 60K requests are +no longer billed as one 120K request. `claude-haiku-5-5` carries its `$0.05` cache-read rate +above 100K. The five `DEFAULT_AGENT_MODEL` call sites (`convex/agent.ts` x2, `agentJobRunner.ts`, +`streaming.ts`, `Chat.tsx`) and the remaining judge/benchmark scripts now have tests that fail +on revert; `qaMatrix` checks both `AGENT_MODEL` fallbacks. The LangChain bridge sends +`strict: true` on the forced-choice downgrade, `probe.ts` and `providerParseSmoke.ts` send the +explicit effort, `providerParseSmoke.ts` now reaches Google for Gemini ids, and the ladder +pre-flight multiplies a free-auto route by `OPENROUTER_FREE_AUTO_LIMIT`. The merge blockers +above are unchanged. + +**Commit**: the commit after `0ff118a` on this branch. **Author**: Homen Shum. + +## 2026-10-15 — Review round 3: cost class preserved, unknown ids fail closed, honest failed-rung cost + +Closes the round-1 findings 1 and 2 that rounds 1 and 2 left open (SPEC A.0.10), plus four round-2 defects. + +- **Unknown ids fail closed.** `resolveModelAlias` now throws `UnknownModelError` for an id that is not priced, not + declared by the alias table and not free-looking, so `model()` and `convexModel()` (and the fallback lists they read) + throw before any request. An unknown `-free` / `:free` id resolves to `openrouter/free-auto`. Pricing and telemetry + readers (`priceRun`, `reserveDispatch`, `creditModel`, `proofloopModelTracking`, the egress policy, `benchmark/run.ts`, + the interop normalizer) read the lenient `lookupModelAlias`, because they must tolerate historical ids and never dispatch. +- **Free never becomes paid.** A free primary no longer falls back to a paid model: adapter `fallbackModelFor` and + `convexModel` drop the paid candidate unless the caller sets `allowPaidFallback: true` and there is a reservation + (adapter: a `SpendMeter`, reserved in `meteredDispatch`; Convex: a finite `maxCostUsd`, estimated per candidate by the + failover controller). Paid -> paid and paid -> free are unchanged. +- **Failed rung cost.** `runRung`'s catch branch reports the sum of its settled dispatch costs, not `0`. +- **Alias fan-out.** `ladderRequestCeiling` / `estimateLadderUsd` resolve the route before testing for free-auto fan-out, + so `free` and `openrouter-auto` count 8 requests per step, not 1. +- **Request shape + pricing agreement.** `probe.ts` `gpt-6.1-sol` is asserted on the wire (URL, `model`, no sampling). + `benchmark/run.ts` pricing moved to `scripts/benchmark/pricingAtRun.ts` (run.ts runs the benchmark on import) so + `tests/modelPricingAgreement.test.ts` runs it, and the `--dry-run` sweep script, on a dated OpenRouter `/models` + fixture; a x1e5 conversion now fails both. +- **Tests retargeted, not loosened.** `evals/ladderBudget.test.ts` (g), `tests/convexModelFreeAutoMode.test.ts` and + `tests/modelAliasClaude.test.ts` used never-heard-of ids that were dispatched; they now assert the refusal, and the + original invariants (zero requests, meter untouched, terminal stop, conservative reservation) are kept on the id the + alias table declares without a price row (`claude-sonnet-4.5`). +- **Spec discrepancies left for a human (SPEC section C), not decided here:** see the PR body. + +**Commit**: the commit after `55261e8` on this branch. **Author**: Homen Shum. + +## 2026-10-16 — Review round 5: known ids from the repo's lists, one guard per boundary, D2 for top_paid, REA/ORX + +Closes the round-3 review of the refreshed ids (SPEC-v9 D1, D2, D5; A.0.4, A.0.6, A.0.10). + +- **Known = named by the repo or by SPEC section 1.2 (D1).** `REPO_DECLARED_UNPRICED_IDS` now holds every + `OPENROUTER_TOP_PAID_AGENT_ROUTES` route and every section 1.2 wire id on every transport. Before this, 9 of the 19 + top-paid routes and 18 of the 32 section 1.2 ids threw `UnknownModelError`, so `model-eval-matrix --route-set top-paid` + broke for those routes. `tests/modelKnownIds.test.ts` now reads `resolveRouteSet(...)` for all six route sets, + `allAgentLlmRoutes()` and the section 1.2 list (typed in the test, because the spec lives outside this repo). +- **Eval scripts stay pass-through (D1).** `model(id, { operatorSelected: true })` keeps an operator- or live-list-selected + id as written: an unknown paid id is not refused and a concrete `:free` id is not rewritten to `openrouter/free-auto`. + Only `scripts/benchmark/run.ts`, `scripts/proofloop-free-model-gauge.ts`, `evals/ladder.ts` (`--real`, including + `ladder:free --free-auto-top`), `scripts/spreadsheetbench-run.ts` and `evals/chatIntakeRuntime.ts` set it. Server code + never does. The alias table still applies. +- **PlanPreview-blocked runs record again.** `runRoomAgent` records a blocked run with the marker `modelPolicy: "not_started"` + (no model runs). The boundary check rejected it, so the plan message and the credit settlement were never reached. The + marker is accepted only together with `initialStatus: "blocked"`. A convex-test drives `runRoomAgent`'s blocked branch. +- **Private stream and capture fail closed.** `streamPrivateReplyText` and `aiSdkReasoner` call `resolveModelAlias` first + (one guard each; `aiSdkReasoner` covers `capturesNode`, `captureSource` and the Firecrawl tool). Tests assert 0 fetch calls + for an unknown `AGENT_MODEL` and for `claude-not-in-catalog-9`, `gpt-not-in-catalog`, `o3-not-in-catalog`, `claude-opus-9`. +- **Call sites driven, not grepped.** `tests/modelRefreshAgentWire.test.ts` runs `runRoomAgent` (public_ask), + `runPrivateAgent` and `runFreeAutoJobSlice` (a claimed job with no `modelPolicy`) through a convex-test database and a + fake `fetch`, and asserts the model in the request URL is `gemini-3.8-flash` (or the `AGENT_MODEL` override) and that no + sampling key is sent. Reverting `modelNameForEgress` to always return the file-egress model, or the runner default to + `gemini-3.5-flash`, fails five of its six tests. +- **D2 for `top_paid`.** `convex/agentJobs.ts` `top_paid` defaulted to the retired `anthropic/claude-sonnet-4`; round 1 + moved it to `anthropic/claude-sonnet-5.5`. No OpenRouter key exists, so under D2 it is now + `DEFAULT_TOP_PAID_MODEL = nvidia/nemotron-3.5-super-vl:free` (`convex/modelDefaults.ts`), the newest `:free` row of the + public list `https://openrouter.ai/api/v1/models` (fetched 2026-10-16T11:32:04Z, recorded in + `docs/eval/openrouter-free-models-2026-10-16.json`) with `tools`, `tool_choice`, `structured_outputs`, a 1,000,000-token + context, text output and no `expiration_date`. Newer or equal rows were skipped on the stated criteria: + `dots-studio/dots-3-note-preview:free` and `poolside/laguna-s-2.1:free` carry an `expiration_date`; `liquid/lfm-2.5-2.6b:free` + has 65,536 context; `nvidia/nemotron-3.5-lightning:free` and the two `thinkingmachines` rows lack `structured_outputs` or + `tool_choice`. **Marked: live-unverified (no key, Homen 2026-10-16).** Behaviour change to expect: a free route cannot take + an uploaded file, so a `top_paid` job in a room with a file is refused at the free-file-egress gate (or promoted to + `z-ai/glm-4.7-flash` when the existing `FREE_AUTO_ALLOW_FILE_EGRESS_PROMOTION` opt-in is set) unless `AGENT_TOP_PAID_MODEL` or + `AGENT_MODEL` is set; those env vars still win. The `anthropic/claude-sonnet-5.5` request-shape handling (`auto` plus + `strict: true`) stays in `convexModel` and its test for an operator who sets that id. Merge blocker 3 (the Environment + smoke for `claude-sonnet-5.5`) therefore no longer applies to a code default. + - Other OpenRouter-routed rows: `providerParserLive.ts:113` only drops the unlisted `nex-agi/nex-n2-pro`; it picks no new + target, the remaining candidates are frozen paid routes with no vendor successor (D2a), so D2 does not apply. + `modelRegistry.ts:48` stays **defer** (`anthropic/claude-fable-5` is active, floor 2027-06-09), so D2 does not apply. +- **Docs.** The `.env.example` note on `NODEAGENT_FREE_AUTO_GOOGLE_MODEL` now says the direct Gemini leg is opt-in + (`allowPaidFallback` plus a meter, `adapter.ts`) and that no production caller sets either. + +### REA (how mature references do the same thing) and ORX (primary sources), fetched 2026-10-16 + +REA reads the mechanisms this PR touches: an explicit alias table instead of a regex, a known-id registry that fails +closed, tiered per-request pricing, a spend meter. Facts, with what was adopted or rejected. + +| Reference (fetched) | Fact observed | Adopted | Rejected | +|---|---|---|---| +| LiteLLM `model_prices_and_context_window.json` (raw.githubusercontent.com/BerriAI/litellm/main, 4,194 entries, 2026-10-16T14:44Z) | One flat map keyed by provider-prefixed wire id (`gemini/gemini-3.8-flash`, `vertex_ai/gemini-3.8-flash`, `claude-haiku-5-5`, `bedrock_mantle/anthropic.claude-haiku-5-5`). Tier prices are separate keys on the same row (`input_cost_per_token_above_100k_tokens`, `*_above_200k_tokens`, `cache_read_input_token_cost_above_100k_tokens`); `claude-haiku-5-5` carries 5e-07 / 2.5e-06 / 5e-08 above 100K. `deprecation_date` is per row and differs by transport (`azure_ai/claude-sonnet-5` 2027-06-30, `vertex_ai/claude-sonnet-5` 2026-12-24, native `claude-sonnet-5` null). `max_output_tokens` is 128000 for `claude-haiku-5-5`. | An id is looked up by its exact wire string per transport (the explicit table, no regex); the tier is a field of the price row and is chosen per request; a lifecycle date belongs to the transport, not to the model name (SPEC B.1). Our Haiku 5.5 tier ($0.50 / $2.50 / cache read $0.05 above 100K) equals LiteLLM's. | A single unified id space across transports; regional multipliers (`us.anthropic.*` rows at 1.1x) and batch keys, which no NodeRoom path uses. | +| models.dev `https://models.dev/api.json` (231 providers, 2026-10-16T14:44Z) | `anthropic/claude-haiku-5-5`: cost 0.1 / 0.5, `cache_read` 0.01, `cache_write` 0.125, one `tiers` entry `{type: "context", size: 100000}` at 0.5 / 2.5 / 0.05 / 0.625; `limit.output` 128000; `release_date` 2026-10-07; `tool_call` and `structured_output` true. Resellers re-price the same model (`poe/google/gemini-3.8-flash` input 0.7576 against 0.75 at the vendor). | The tier boundary (100K prompt tokens, per request) and the vendor rates cross-check our table; `release_date` agrees with the spec. | Reseller rows as a price source: prices come from the vendor page, never an aggregator. | +| OpenRouter `GET /api/v1/models` (public, no key; 465 rows, 2026-10-16T11:32:04Z; earlier reads 2026-10-10 and 2026-10-14 in the spec) | Each row has `id`, `created`, `context_length`, per-token string `pricing`, `supported_parameters` (`tools`, `tool_choice`, `structured_outputs`) and `expiration_date`; 18 of 465 rows carry an `expiration_date`, including two of the newest `:free` rows. | Capability selection from `supported_parameters`, the `expiration_date` exclusion and `created` ordering for D2; the per-token to per-1M conversion is already pinned by `tests/modelPricingAgreement.test.ts`. | Treating absence from the list as a vendor retirement (it is "unlisted on this transport", SPEC 1.1). | +| Vercel AI SDK `@ai-sdk/anthropic` 3.0.120 (installed source, `dist/index.mjs` `getModelCapabilities`, read 2026-10-16) | Capabilities are matched by substring. `claude-haiku-5-5` matches no 5.x branch (only `claude-opus-5-5`, `claude-opus-5` and `claude-sonnet-5` do), so it falls into the generic `claude-` branch: `maxOutputTokens` 128000, `isKnownModel: false`. The SDK reports the flag and still dispatches. | The `isKnownModel` idea, enforced instead of reported: `resolveModelAlias` throws `UnknownModelError`, and `modelMaxOutputTokens` keeps its own 2048 floor and caps for the 5.5 ids rather than trusting the SDK fallback. | Substring matching (it is how `claude-haiku-5-5` ended up in the fallback branch). | + +ORX, primary sources behind the ids, prices and dates in this PR (ids and prices were not taken from recall): + +- Claude ids, prices and floors: `frontier.json` (read 2026-10-10, which cites `platform.claude.com/docs/en/models/overview` + and the release notes); the Anthropic deprecations page re-read live 2026-10-14 (stored copy + `_scratch/anthropic-deprecations-2026-10-14-v8.html` in the model-defaults workspace): `claude-haiku-4-5-20251001` Active, + floor 2026-10-15; `claude-sonnet-4-5-20250929` deprecated, retires 2026-11-30. +- Gemini ids, prices and the `gemini-3.5-flash` deprecation (2026-10-08) and `gemini-3.1-flash-tts-preview` shutdown + (2026-11-17): `frontier.json` (read 2026-10-10), Google's deprecations table as quoted in SPEC 1.1. +- OpenAI ids and the 2026-10-14 GPT-4-series notice (removal 2027-01-14): OpenAI deprecations page re-read live 2026-10-14. +- OpenRouter ids, `expiration_date` and the D2 selection: the public list above, recorded in + `docs/eval/openrouter-free-models-2026-10-16.json` (all 15 `:free` rows with the criteria columns). +- Not verified by a request: no provider call was made. Every `:free` route and the Claude 5.5 ids over OpenRouter are + request-shape tested against a fake transport only. + +## 2026-10-16 — Review round 6: operator ids stay pass-through everywhere, free-looking unknown ids are not sent raw, env defaults checked at the boundary + +Closes the round-4 review (SPEC-v9 D1; A.0.4, A.0.5, A.0.6, A.0.10). No check, test or threshold was weakened; no provider call was made. + +- **Operator overrides stay pass-through at every CLI entry point (D1, A.0.4).** Round 5 set `operatorSelected` in five scripts and + missed the rest. It is now also set in `evals/financeModelLive.ts`, `evals/managedLockPerf.ts`, `evals/professionalRuntimeLive.ts`, + `scripts/halo-live-path-calibration.ts`, `scripts/sfn-nonbtb-nodeagent-runner.ts` (`SFN_MODEL_ID`, found by the same search), + `src/eval/bankerToolBenchNodeAgentGeneral.ts` (`--model-id`) and `scripts/startup-diligence-provider-eval.ts`. The last one + calls `convexModel()`, which had no such option: `ConvexModelOptions.operatorSelected` now has the same contract as + `ModelAdapterOptions.operatorSelected`, and `normalizeFallbackModelIds` no longer re-resolves a primary id its caller already + resolved. Server code never sets it. `tests/modelKnownIds.test.ts` replaced its five-file regex pin with a scan of every + resolver call (balanced parentheses, so multi-line calls count) in twelve entry points. +- **A free-looking unknown id is no longer sent under its raw name (A.0.10).** `aiSdkReasoner` and `streamPrivateReplyText` called + the resolver and threw the result away. Now the RESOLVED id is what dispatches. The streaming path routes `openrouter/free-auto` + to OpenRouter like any known free-auto input; the capture seam serves only Anthropic and OpenAI, so it refuses + (`capture_reasoner_route_unsupported`) instead of sending a free route to a provider that cannot serve it. +- **Env defaults and the file-egress promotion pass the public-boundary check (A.0.10).** `defaultModelPolicyForRoute` ran + `knownModelPolicy` only on an explicit `modelPolicy`; the `AGENT_ORCHESTRATOR_MODEL` / `AGENT_MODEL` / `AGENT_TOP_PAID_MODEL` / + `AGENT_RESEARCH_MODEL` defaults and `configuredFileEgressModel()` now pass it too. Override precedence is unchanged. + `.tours/01-primary-user-flow.tour` step 7 moved from `agentJobs.ts:2798` to `:2799` (one comment line was added above it). +- **Retained dotted ids keep the wire form main sent (A.0.5).** `claude-opus-4.1` and `claude-haiku-3.5` alias to + `claude-opus-4-1` and `claude-haiku-3-5` (the old regex did this for every dotted id), with hyphenated price rows at the + dotted rows' prices so they still price. `openrouter/auto`, the label `chooseNodeAgentRoute` gives the benchmark lane, is declared + known again (unpriced: a metered run still refuses it). +- **`gemini-3.6-flash` context window (A.0.6).** `frontier.json` marks it UNVERIFIED. The value 1048576 is `context_length` for + `google/gemini-3.6-flash` on the public OpenRouter list (`GET /api/v1/models`, read 2026-10-16), now cited in the row. + +## 2026-10-17 — Review round 7: an operator-picked model is the model measured, the private stream refuses the free sentinel, pricing source 6 is exercised + +- **The startup-diligence eval measures the model the operator named (D1, A.0.4).** `--model gemini-not-in-catalog` reached the provider, + but the contract eval then passed that id to `agentJobs.createOrReuse` as `modelPolicy`, whose public boundary refuses an id outside the + catalog (`unknown_model_policy`). The candidate loop swallowed the error, sent a second paid request to `gemini-3.8-flash` and reported + that model. The job now always carries the catalog free route (the runner is stubbed, it is never dispatched); the model measured is + recorded on the attempt rows as before. Test: one request, one reported model. +- **`judge()` takes `operatorSelected` (D1, A.0.4).** `evals/professionalLiveCatalog.ts` (`--model`) and `scripts/benchmark/run.ts` + (`JUDGE_MODEL`) pass it; `scripts/openrouter-free-discover.ts` keeps the strict resolver for its fixed id. Without the option nothing changes. +- **The private stream refuses `openrouter/free-auto` before any request (A.0.10).** The sentinel is not an OpenRouter model id (the public + `/api/v1/models` list read 2026-10-17 has `openrouter/free` and `openrouter/auto`), so posting it failed every time, and the comment that + said the egress policy refused it was false. `streamPrivateReplyText` now throws `private_stream_free_auto_unsupported`. +- **Tests.** Pricing source 6 (`proofloopModelTracking`) is called by the agreement test for every refresh row, the tier edge and the two retired + rows; the retired `claude-opus-4-1` / `claude-haiku-3-5` rows stay undated on purpose (main's prices, no vendor read). The Anthropic + `generateObject` parser tests assert no forced `tool_choice`, no `json` tool, native `output_config.format` and a `max_tokens` of at least 64K. diff --git a/convex/agent.ts b/convex/agent.ts index 657a22fd..bfa43ed9 100644 --- a/convex/agent.ts +++ b/convex/agent.ts @@ -19,6 +19,7 @@ import { v } from "convex/values"; import { makeFunctionReference } from "convex/server"; +import { DEFAULT_AGENT_MODEL, DEFAULT_FILE_EGRESS_MODEL } from "./modelDefaults"; import { action } from "./_generated/server"; import type { Id } from "./_generated/dataModel"; import { actorProofV } from "./lib"; @@ -38,7 +39,7 @@ import { AgentRunError, runAgent } from "../src/nodeagent/core/runtime"; import type { AgentTraceEvent } from "../src/nodeagent/core/types"; import { SERVER_PRODUCTION_ROOM_TOOLS as PRODUCTION_ROOM_TOOLS } from "../src/nodeagent/skills/server/productionTools"; import { MANAGED_LOCK_SYSTEM_PROMPT } from "../src/nodeagent/models/prompts/systemPrompt"; -import { convexModel as agentModel, convexPriceRun as priceRun } from "../src/nodeagent/models/convexModel"; +import { convexModel as agentModel, convexPriceRequest, convexPriceRun as priceRun } from "../src/nodeagent/models/convexModel"; import { buildResearchContext, buildNoteContext, buildWallContext } from "../src/nodeagent/core/worldModel"; import { injectMemoryIntoSystemPrompt } from "../src/nodemem/memoryContextBuilder"; import { nodeMemInjectionEnabled, nodeMemRoomConfigEnabled } from "./nodemem"; @@ -62,7 +63,6 @@ const MIN_ACTION_RESERVE_MS = 10_000; const ACTION_SAFETY_MARGIN_MS = 15_000; const DEFAULT_CONTEXT_MAX_CHARS = 24_000; const DEFAULT_CONTEXT_KEEP_RECENT = 10; -const DEFAULT_FILE_EGRESS_MODEL = "z-ai/glm-4.7-flash"; const roomsFullRef = makeFunctionReference<"query">("rooms:full"); const agentJobsCreateOrReuseRef = makeFunctionReference<"mutation">("agentJobs:createOrReuse") as any; const agentJobsFinishInteractiveRef = makeFunctionReference<"mutation">("agentJobs:finishInteractive") as any; @@ -338,12 +338,13 @@ export const runRoomAgent = action({ // - research (background synthesis): deepseek-v4-flash — first route to clear the v3 composite- // synthesis benchmark 9/9 at $0.0034/run (docs/eval/results.json; ~300x cheaper than gemini-3.5-flash's // measured $1.10/task). Evidence covers the fetch->synthesize->write shape ONLY. - // - interactive collaboration (lock/CAS/draft): stays on gemini-3.5-flash, the only route with a - // recorded L1-L4 collaboration-ladder pass. flash gets this lane only after it passes the ladder. + // - interactive collaboration (lock/CAS/draft): DEFAULT_AGENT_MODEL (gemini-3.8-flash). The recorded + // L1-L4 collaboration-ladder pass is for gemini-3.5-flash (Google now serves it as 3.6); this default + // merges only with docs/eval/ladder/model-refresh-noderoom.json (3.6 + 3.8, 8/8 pass). const egressArtifacts = providerEgressArtifactsFromRoomState(roomState); const requestedModelName = a.mode === "research" ? (process.env.AGENT_RESEARCH_MODEL ?? "minimax/minimax-m3") - : (process.env.AGENT_MODEL ?? "gemini-3.5-flash"); + : (process.env.AGENT_MODEL ?? DEFAULT_AGENT_MODEL); const model = agentModel(modelNameForEgress(requestedModelName, "public_ask", egressArtifacts), { entrypoint: "public_ask" }); const egressDecision = providerEgressDecision({ model: model.name, @@ -643,7 +644,7 @@ export const runRoomAgent = action({ maxTokens: envNumber("AGENT_MAX_TOKENS_PER_RUN", 250_000, 1_000, 4_000_000), maxCostUsd: envNumber("AGENT_MAX_USD_PER_RUN", 2, 0.01, 100), }, - priceStep: (modelName, inputTokens, outputTokens) => priceRun(modelName, inputTokens, outputTokens), + priceStep: convexPriceRequest, onTrace: (event) => { void recordLiveOperation({ kind: liveOperationKind(event), @@ -780,7 +781,7 @@ export const runPrivateAgent = action({ const requester = roomState.members.find((m: { id: unknown }) => String(m.id) === a.requester.actor.id) as { id: unknown; name: string } | undefined; if (!requester) throw new Error("member_required"); const egressArtifacts = providerEgressArtifactsFromRoomState(roomState); - const model = agentModel(modelNameForEgress(process.env.AGENT_MODEL ?? "gemini-3.5-flash", "private_agent", egressArtifacts), { entrypoint: "private_agent" }); + const model = agentModel(modelNameForEgress(process.env.AGENT_MODEL ?? DEFAULT_AGENT_MODEL, "private_agent", egressArtifacts), { entrypoint: "private_agent" }); const egressDecision = providerEgressDecision({ model: model.name, entrypoint: "private_agent", diff --git a/convex/agentJobRunner.ts b/convex/agentJobRunner.ts index 259cbdb6..56c49c92 100644 --- a/convex/agentJobRunner.ts +++ b/convex/agentJobRunner.ts @@ -19,7 +19,7 @@ import { SERVER_PRODUCTION_ROOM_TOOLS as PRODUCTION_ROOM_TOOLS } from "../src/no import { MANAGED_LOCK_SYSTEM_PROMPT } from "../src/nodeagent/models/prompts/systemPrompt"; import { injectMemoryIntoSystemPrompt } from "../src/nodemem/memoryContextBuilder"; import { nodeMemInjectionEnabled, nodeMemRecordingEnabled, nodeMemRoomConfigEnabled } from "./nodemem"; -import { configuredConvexModelFallbacks, convexModel as agentModel, convexPriceRun as priceRun } from "../src/nodeagent/models/convexModel"; +import { configuredConvexModelFallbacks, convexModel as agentModel, convexPriceRequest, convexPriceRun as priceRun } from "../src/nodeagent/models/convexModel"; import { qualityFailoverRetryAt } from "../src/nodeagent/models/qualityFailover"; import { authorizedModelForFramePhase } from "../src/nodeagent/models/phaseModel"; import { buildResearchContext, buildCompanyDeepDiveContext } from "../src/nodeagent/core/worldModel"; @@ -43,6 +43,7 @@ import { } from "../src/nodeagent/guardrails/egressPolicy"; import { makeConvexStepJournal } from "./agentStepJournalClient"; import { createVerifiedWorkbookWorkflowHook } from "../src/nodeagent/guardrails/workbookWorkflow"; +import { DEFAULT_AGENT_MODEL, DEFAULT_FILE_EGRESS_MODEL } from "./modelDefaults"; const CONVEX_ACTION_LIMIT_MS = 10 * 60_000; const DEFAULT_SLICE_BUDGET_MS = 7 * 60_000; @@ -53,7 +54,6 @@ const DEFAULT_LEASE_EXTRA_MS = 60_000; const DEFAULT_RESUME_DELAY_MS = 5_000; const DEFAULT_CONTEXT_MAX_CHARS = 24_000; const DEFAULT_CONTEXT_KEEP_RECENT = 10; -const DEFAULT_FILE_EGRESS_MODEL = "z-ai/glm-4.7-flash"; const RESUME_CHECKPOINT_PREFIX = "RESUME CHECKPOINT:"; const agentJobsClaimSliceRef = makeFunctionReference<"mutation">("agentJobs:claimSlice") as any; const agentJobsFinishSliceRef = makeFunctionReference<"mutation">("agentJobs:finishSlice") as any; @@ -509,7 +509,7 @@ export const runFreeAutoJobSlice = internalAction({ const roomArtifacts = await ctx.runQuery(artifactsListForRoomRef, { roomId: claimed.roomId }) as Array<{ title?: string; kind?: string; meta?: unknown; visibility?: string }>; const egressArtifacts = providerEgressArtifactsForClaimedJob(roomArtifacts, claimed); let entrypoint = runnerEntrypoint(claimed); - const modelPolicy = claimed.modelPolicy || (entrypoint === "free" ? "openrouter/free-auto" : process.env.AGENT_MODEL ?? "gemini-3.5-flash"); + const modelPolicy = claimed.modelPolicy || (entrypoint === "free" ? "openrouter/free-auto" : process.env.AGENT_MODEL ?? DEFAULT_AGENT_MODEL); let resolvedModelPolicy = modelPolicy === "openrouter/free-auto" ? process.env.FREE_AUTO_JOB_MODEL ?? modelPolicy : modelPolicy; @@ -928,7 +928,7 @@ export const runFreeAutoJobSlice = internalAction({ deadlineAt, reserveMs, spendLimits, - priceStep: (modelName: string, inputTokens: number, outputTokens: number) => priceRun(modelName, inputTokens, outputTokens), + priceStep: convexPriceRequest, onTextDelta: (delta, step) => onPublicTextDelta(delta, step), onStreamEvent: (event) => recordStreamEvent(event), onTrace: (event) => { @@ -961,7 +961,7 @@ export const runFreeAutoJobSlice = internalAction({ // USD half reachable (P0-4: without it the gate received costUsd=0 and maxCostUsd was dead // surface — one env var pointing free-auto at a paid model meant unbounded spend). spendLimits, - priceStep: (modelName: string, inputTokens: number, outputTokens: number) => priceRun(modelName, inputTokens, outputTokens), + priceStep: convexPriceRequest, onTextDelta: (delta, step) => onPublicTextDelta(delta, step), onStreamEvent: (event) => recordStreamEvent(event), onTrace: (event) => { diff --git a/convex/agentJobs.ts b/convex/agentJobs.ts index a9178743..6b0bdc0a 100644 --- a/convex/agentJobs.ts +++ b/convex/agentJobs.ts @@ -28,13 +28,14 @@ import { } from "../src/nodeagent/guardrails/egressPolicy"; import { parseBulkCompanyIngest } from "../src/nodeagent/skills/finance/bulkIngest"; import { finalizePublicAgentJobStreamAfterTerminal } from "./agentStreamTerminal"; +import { DEFAULT_AGENT_MODEL, DEFAULT_FILE_EGRESS_MODEL, DEFAULT_TOP_PAID_MODEL } from "./modelDefaults"; +import { resolveModelAlias, UnknownModelError } from "../src/nodeagent/models/modelCatalog"; // BOUND: cap a single bulk-diligence fan-out so one command can't enqueue unbounded jobs. const MAX_BULK_COMPANIES = 50; const MAX_INVESTIGATION_ELEMENTS = 8_192; const MAX_INVESTIGATION_WRITE_IDS = 8_192; const LEASE_EXPIRY_GRACE_MS = 30_000; -const DEFAULT_FILE_EGRESS_MODEL = "z-ai/glm-4.7-flash"; const INVESTIGATION_RESEARCH_GOAL = "Research every pending or stale company in the exact authorized dataset: claim only its editable research fields, retrieve public sources, write evidence-bearing summary/funding/headcount/recent_signal plus source/source2/last_researched/status, and release every managed lock."; const INVESTIGATION_WRITABLE_FIELD_IDS = [ "summary", @@ -2094,13 +2095,29 @@ function defaultEntrypointForRoute(routePolicy: RoutePolicy): DurableStartEntryp return routePolicy === "free_auto" ? "free" : "public_ask"; } -function defaultModelPolicyForRoute(args: { routePolicy: RoutePolicy; entrypoint: DurableStartEntrypoint; mode?: "variance" | "research"; modelPolicy?: string }) { - if (args.modelPolicy) return args.modelPolicy; +/** SPEC A.0.10 at the public API boundary: a model string the catalog does not know is refused before a job row persists it. */ +function knownModelPolicy(modelPolicy: string): string { + try { + resolveModelAlias(modelPolicy); + } catch (error) { + if (error instanceof UnknownModelError) throw new Error("unknown_model_policy"); + throw error; + } + return modelPolicy; +} + +/** `runRoomAgent` records a PlanPreview-blocked run with this marker: no model runs for a blocked job, so there is no id to check. */ +const NO_MODEL_MARKER = "not_started"; + +function defaultModelPolicyForRoute(args: { routePolicy: RoutePolicy; entrypoint: DurableStartEntrypoint; mode?: "variance" | "research"; modelPolicy?: string; initialStatus?: "running" | "blocked" }) { + if (args.modelPolicy === NO_MODEL_MARKER && args.initialStatus === "blocked") return args.modelPolicy; + if (args.modelPolicy) return knownModelPolicy(args.modelPolicy); if (args.routePolicy === "explicit") throw new Error("explicit_route_requires_modelPolicy"); if (args.routePolicy === "free_auto" || args.entrypoint === "free") return "openrouter/free-auto"; - if (args.routePolicy === "top_paid") return process.env.AGENT_TOP_PAID_MODEL ?? process.env.AGENT_MODEL ?? "anthropic/claude-sonnet-4"; - if (args.mode === "research") return process.env.AGENT_RESEARCH_MODEL ?? process.env.AGENT_WORKER_MODEL ?? "minimax/minimax-m3"; - return process.env.AGENT_ORCHESTRATOR_MODEL ?? process.env.AGENT_MODEL ?? "gemini-3.5-flash"; + // Env defaults are operator-set strings and reach the job row like a caller's argument, so they pass the same boundary check. + if (args.routePolicy === "top_paid") return knownModelPolicy(process.env.AGENT_TOP_PAID_MODEL ?? process.env.AGENT_MODEL ?? DEFAULT_TOP_PAID_MODEL); + if (args.mode === "research") return knownModelPolicy(process.env.AGENT_RESEARCH_MODEL ?? process.env.AGENT_WORKER_MODEL ?? "minimax/minimax-m3"); + return knownModelPolicy(process.env.AGENT_ORCHESTRATOR_MODEL ?? process.env.AGENT_MODEL ?? DEFAULT_AGENT_MODEL); } function configuredFileEgressModel() { @@ -2451,7 +2468,7 @@ async function startDurableAgentJob(ctx: any, a: DurableStartAgentJobArgs): Prom throw new Error("investigation_write_scope_empty"); } } - let modelPolicy = defaultModelPolicyForRoute({ routePolicy, entrypoint, mode: a.mode, modelPolicy: a.modelPolicy }); + let modelPolicy = defaultModelPolicyForRoute({ routePolicy, entrypoint, mode: a.mode, modelPolicy: a.modelPolicy, initialStatus: a.initialStatus }); const egressArtifacts = await providerEgressArtifactsForRoom(ctx, a.roomId, artifact); const freeFileEgressDecision = providerEgressDecision({ model: modelPolicy, @@ -2464,7 +2481,7 @@ async function startDurableAgentJob(ctx: any, a: DurableStartAgentJobArgs): Prom if (promotedForFileEgress) { entrypoint = "public_ask"; routePolicy = "explicit"; - modelPolicy = configuredFileEgressModel(); + modelPolicy = knownModelPolicy(configuredFileEgressModel()); } const defaultMaxAttempts = execution === "inline" ? 1 diff --git a/convex/http.ts b/convex/http.ts index 7403d7b4..26f00878 100644 --- a/convex/http.ts +++ b/convex/http.ts @@ -15,6 +15,7 @@ import type { StreamId } from "@convex-dev/persistent-text-streaming"; import { streamingComponent } from "./streaming"; import { streamPrivateReplyText } from "./streamingModel"; import { privateAgentSystemPrompt } from "./agentPrompts"; +import { DEFAULT_AGENT_MODEL } from "./modelDefaults"; import { auth } from "./auth"; const http = httpRouter(); @@ -32,7 +33,7 @@ const NVIDIA_STT_MODEL = "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning"; const NVIDIA_AUDIO_MAX_BYTES = 5 * 1024 * 1024; const CLOUDFLARE_AI_MODEL_TTS = "@cf/myshell-ai/melotts"; const GEMINI_TTS_BASE_URL = "https://generativelanguage.googleapis.com/v1beta/models"; -const GEMINI_TTS_MODEL = "gemini-3.1-flash-tts-preview"; +const GEMINI_TTS_MODEL = "gemini-3.8-flash-tts"; const GOOGLE_TTS_URL = "https://texttospeech.googleapis.com/v1/text:synthesize"; type VoiceSttProvider = "openai" | "nvidia"; type VoiceTtsProvider = "openai" | "cloudflare" | "gemini" | "google" | "disabled"; @@ -172,7 +173,7 @@ http.route({ const userMsg = `ROOM CONTEXT\n${meta.roomContext}\n\n${meta.requesterName} asks: ${meta.goal}`; let answer = ""; try { - answer = await streamPrivateReplyText(process.env.AGENT_MODEL ?? "gemini-3.5-flash", system, userMsg, append); + answer = await streamPrivateReplyText(process.env.AGENT_MODEL ?? DEFAULT_AGENT_MODEL, system, userMsg, append); } catch (error) { // HONEST_STATUS: the partial text persists, the error is visible text, never a silent 2xx void. const msg = `(private agent error: ${error instanceof Error ? error.message.slice(0, 160) : "model call failed"})`; diff --git a/convex/modelDefaults.ts b/convex/modelDefaults.ts new file mode 100644 index 00000000..b19bbab4 --- /dev/null +++ b/convex/modelDefaults.ts @@ -0,0 +1,28 @@ +/** + * Convex-safe default model ids (no imports, no Node APIs). One literal per default so a model + * refresh edits one line. Env chains (AGENT_MODEL ?? ...) stay at the call sites. + * + * Spec: model-defaults SPEC Part A §A.2 (NodeRoom). + */ + +/** + * Default agent model. Was "gemini-3.5-flash": Google deprecated it 2026-10-08 and auto-routes it + * to gemini-3.6-flash. Merge is gated on the L1-L4 ladder receipt in + * docs/eval/ladder/model-refresh-noderoom.json (see CHANGELOG/qa/model-refresh-noderoom.md). + */ +export const DEFAULT_AGENT_MODEL = "gemini-3.8-flash"; + +/** File-egress fallback (OpenRouter). Listed, no expiry; unchanged by the refresh. */ +export const DEFAULT_FILE_EGRESS_MODEL = "z-ai/glm-4.7-flash"; + +/** + * `top_paid` route default (convex/agentJobs.ts; env chain AGENT_TOP_PAID_MODEL ?? AGENT_MODEL ?? this stays at the call site). + * Was "anthropic/claude-sonnet-4" (retired 2026-06-15). SPEC-v9 D2 (Homen, 2026-10-16): no working OpenRouter key exists, so + * an OpenRouter-routed role moves to the newest `:free` model that the public https://openrouter.ai/api/v1/models list shows + * with the capabilities the role needs. This role runs the tool loop (forced tool_choice, tools, structured outputs) over long + * room context, so it needs `tools`, `tool_choice`, `structured_outputs` and a >= 256K context. List fetched + * 2026-10-16T11:32Z: nvidia/nemotron-3.5-super-vl:free (created 2026-10-15, 1,000,000 context, text output, no expiration_date). + * LIVE-UNVERIFIED (no key, Homen 2026-10-16): never called. Unlike the old paid default this is a free route, so a room with + * an uploaded file is blocked by the free-file-egress gate unless AGENT_TOP_PAID_MODEL / AGENT_MODEL is set (CHANGELOG/qa). + */ +export const DEFAULT_TOP_PAID_MODEL = "nvidia/nemotron-3.5-super-vl:free"; diff --git a/convex/streaming.ts b/convex/streaming.ts index b8d10b62..a3ebf6b8 100644 --- a/convex/streaming.ts +++ b/convex/streaming.ts @@ -25,6 +25,7 @@ import { actorProofV, actorV, requireActiveAgentJobLease, requireActorCanUseChan import { summarizeRoomForPrivate } from "./agentPrompts"; import { assertProviderEgressAllowed } from "../src/nodeagent/guardrails/egressPolicy"; import { streamingComponent } from "./streamingComponent"; +import { DEFAULT_AGENT_MODEL } from "./modelDefaults"; export { streamingComponent } from "./streamingComponent"; const roomsFullRef = makeFunctionReference<"query">("rooms:full"); @@ -65,7 +66,7 @@ export const createPrivateReplyStream = mutation({ const roomState = await ctx.runQuery(roomsFullRef, { roomId: a.roomId, requester: a.requester }); if (!roomState) throw new Error("room_not_found"); assertProviderEgressAllowed({ - model: process.env.AGENT_MODEL ?? "gemini-3.5-flash", + model: process.env.AGENT_MODEL ?? DEFAULT_AGENT_MODEL, entrypoint: "private_agent", artifacts: roomState.artifacts.map((art: { title: string; kind: string; meta?: unknown }) => ({ title: art.title, diff --git a/convex/streamingModel.ts b/convex/streamingModel.ts index b8df4456..27816911 100644 --- a/convex/streamingModel.ts +++ b/convex/streamingModel.ts @@ -8,6 +8,8 @@ */ import { redactPII } from "../src/nodeagent/guardrails/gateway"; import { assertProviderRouteAllowed } from "../src/nodeagent/guardrails/egressPolicy"; +import { isNodeAgentFreeAutoModel, resolveModelAlias } from "../src/nodeagent/models/modelCatalog"; +import { isOpenRouterFreeAutoModel } from "../src/nodeagent/models/openRouterFreeModels"; import { openAiCompatibleTokenLimitParam } from "../src/nodeagent/models/openAiTokenLimit"; export type StreamAppend = (text: string) => Promise; @@ -55,7 +57,13 @@ export async function streamPrivateReplyText( userMsg: string, append: StreamAppend, ): Promise { - const route = assertProviderRouteAllowed({ model: modelId, entrypoint: "private_agent", env: process.env }); + // SPEC A.0.10: an id the catalog does not know never dispatches (convexModel() throws the same error), and the RESOLVED id is + // what the egress policy routes: an unknown free-looking id becomes `openrouter/free-auto`. That is an internal sentinel (the + // free-model picker resolves it per call; so does `nodeagent/free-auto`), not an OpenRouter model id, and this single-shot stream has no picker, so it is + // refused here before any request instead of posting the sentinel or the raw string. + const resolved = resolveModelAlias(modelId); + if (isOpenRouterFreeAutoModel(resolved) || isNodeAgentFreeAutoModel(resolved)) throw new Error(`private_stream_free_auto_unsupported:${modelId}`); + const route = assertProviderRouteAllowed({ model: resolved, entrypoint: "private_agent", env: process.env }); const safeSystem = redactPII(system).text; const safeUser = redactPII(userMsg).text; if (route.provider === "gemini") return geminiStream(route.resolvedModel, safeSystem, safeUser, append); diff --git a/docs/eval/openrouter-free-models-2026-10-16.json b/docs/eval/openrouter-free-models-2026-10-16.json new file mode 100644 index 00000000..9ae7c8e3 --- /dev/null +++ b/docs/eval/openrouter-free-models-2026-10-16.json @@ -0,0 +1,189 @@ +{ + "source": "https://openrouter.ai/api/v1/models (public, no key)", + "fetchedAt": "2026-10-16T11:32:04Z", + "totalRows": 465, + "note": "SPEC-v9 D2: the :free rows of the public list. top_paid needs tools + tool_choice + structured_outputs, context >= 256000, text output, no expiration_date; the newest such row is selected.", + "selected": "nvidia/nemotron-3.5-super-vl:free", + "rows": [ + { + "id": "nvidia/nemotron-3.5-super-vl:free", + "created": 1792076662, + "contextLength": 1000000, + "outputModalities": [ + "text" + ], + "tools": true, + "toolChoice": true, + "structuredOutputs": true, + "expirationDate": null + }, + { + "id": "dots-studio/dots-3-note-preview:free", + "created": 1786680361, + "contextLength": 512000, + "outputModalities": [ + "text" + ], + "tools": true, + "toolChoice": true, + "structuredOutputs": true, + "expirationDate": "2026-12-31" + }, + { + "id": "liquid/lfm-2.5-2.6b:free", + "created": 1786470519, + "contextLength": 65536, + "outputModalities": [ + "text" + ], + "tools": true, + "toolChoice": true, + "structuredOutputs": true, + "expirationDate": null + }, + { + "id": "nvidia/nemotron-3.5-lightning:free", + "created": 1786452751, + "contextLength": 1000000, + "outputModalities": [ + "text" + ], + "tools": true, + "toolChoice": true, + "structuredOutputs": false, + "expirationDate": null + }, + { + "id": "thinkingmachines/inkling-small:free", + "created": 1785443117, + "contextLength": 1048576, + "outputModalities": [ + "text" + ], + "tools": true, + "toolChoice": false, + "structuredOutputs": false, + "expirationDate": null + }, + { + "id": "poolside/laguna-s-2.1:free", + "created": 1784652683, + "contextLength": 262144, + "outputModalities": [ + "text" + ], + "tools": true, + "toolChoice": true, + "structuredOutputs": false, + "expirationDate": "2026-10-31" + }, + { + "id": "thinkingmachines/inkling:free", + "created": 1784325956, + "contextLength": 1048576, + "outputModalities": [ + "text" + ], + "tools": true, + "toolChoice": false, + "structuredOutputs": false, + "expirationDate": null + }, + { + "id": "poolside/laguna-xs-2.1:free", + "created": 1783002429, + "contextLength": 262144, + "outputModalities": [ + "text" + ], + "tools": true, + "toolChoice": true, + "structuredOutputs": false, + "expirationDate": "2026-10-31" + }, + { + "id": "cohere/north-mini-code:free", + "created": 1781723748, + "contextLength": 256000, + "outputModalities": [ + "text" + ], + "tools": true, + "toolChoice": true, + "structuredOutputs": false, + "expirationDate": null + }, + { + "id": "nvidia/nemotron-3.5-content-safety:free", + "created": 1780581864, + "contextLength": 128000, + "outputModalities": [ + "text" + ], + "tools": false, + "toolChoice": false, + "structuredOutputs": false, + "expirationDate": null + }, + { + "id": "nvidia/nemotron-3-ultra-550b-a55b:free", + "created": 1780551208, + "contextLength": 1000000, + "outputModalities": [ + "text" + ], + "tools": true, + "toolChoice": true, + "structuredOutputs": false, + "expirationDate": null + }, + { + "id": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free", + "created": 1777393095, + "contextLength": 256000, + "outputModalities": [ + "text" + ], + "tools": true, + "toolChoice": true, + "structuredOutputs": false, + "expirationDate": null + }, + { + "id": "google/gemma-4-26b-a4b-it:free", + "created": 1775227989, + "contextLength": 262144, + "outputModalities": [ + "text" + ], + "tools": true, + "toolChoice": true, + "structuredOutputs": false, + "expirationDate": null + }, + { + "id": "google/gemma-4-31b-it:free", + "created": 1775148486, + "contextLength": 262144, + "outputModalities": [ + "text" + ], + "tools": true, + "toolChoice": true, + "structuredOutputs": false, + "expirationDate": null + }, + { + "id": "nvidia/nemotron-3-super-120b-a12b:free", + "created": 1773245239, + "contextLength": 262144, + "outputModalities": [ + "text" + ], + "tools": true, + "toolChoice": true, + "structuredOutputs": true, + "expirationDate": null + } + ] +} \ No newline at end of file diff --git a/evals/chatIntakeRuntime.ts b/evals/chatIntakeRuntime.ts index ed011286..1ff9aa1d 100644 --- a/evals/chatIntakeRuntime.ts +++ b/evals/chatIntakeRuntime.ts @@ -533,7 +533,7 @@ if (isCli) { const route = optionValue("--real"); const lockMode: ChatIntakeLockMode = process.argv.includes("--managed-locks") ? "runtime_managed_lock" : "explicit_agent_lock"; const agent = route - ? (await import("../src/nodeagent/models/adapter")).model(route) + ? (await import("../src/nodeagent/models/adapter")).model(route, { operatorSelected: true }) : scriptedModel( lockMode === "runtime_managed_lock" ? chatIntakeManagedCapturePlan() : chatIntakeCapturePlan(), lockMode === "runtime_managed_lock" ? "scripted-chat-intake-managed" : "scripted-chat-intake", diff --git a/evals/financeModelLive.ts b/evals/financeModelLive.ts index 2113b48b..6ccde665 100644 --- a/evals/financeModelLive.ts +++ b/evals/financeModelLive.ts @@ -1022,7 +1022,7 @@ async function main(): Promise { const traceRefs: string[] = []; for (let run = 1; run <= runs; run++) { - const agent = scripted ? scriptedModel(financeModelSolvePlan(gold), `scripted-finance-solver-${run}`) : realModel(route); + const agent = scripted ? scriptedModel(financeModelSolvePlan(gold), `scripted-finance-solver-${run}`) : realModel(route, { operatorSelected: true }); const roomVariant = variantCycle[(run - 1) % variantCycle.length]; const report = await runFinanceModelLiveSolve({ gold, pack, agent, diff --git a/evals/ladder.ts b/evals/ladder.ts index bb17f2b5..a8e86451 100644 --- a/evals/ladder.ts +++ b/evals/ladder.ts @@ -6,8 +6,10 @@ * npm run ladder:real -- gpt-5.4-nano,gemini-3.1-flash-lite */ import "../scripts/benchmark/loadEnv"; -import { mkdirSync, writeFileSync } from "node:fs"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { execSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { pathToFileURL } from "node:url"; import { appendEvalRuns, computeCaseSetHash, DEFAULT_STORE, runKey, type EvalRunRecord } from "./evalStore"; import { dirname, join, relative, resolve } from "node:path"; import { z } from "zod"; @@ -20,7 +22,20 @@ import { runAgent } from "../src/nodeagent/core/runtime"; import type { AgentMessage, AgentModel, AgentResult, AgentTool, AgentTraceEvent, RoomTools } from "../src/nodeagent/core/types"; import { scriptedModel, type Planner } from "../src/nodeagent/models/scripted"; import { recomputeVariancePlan } from "../src/nodeagent/core/plans"; -import { model as realModel, priceRun } from "../src/nodeagent/models/adapter"; +import { + MAX_DISPATCH_RECORDS, + createSpendMeter, + model as realModel, + priceRequest, + openRouterFreeAutoLimit, + priceRequestStrict, + requestInputTokenUpperBound, + type DispatchPolicy, + type DispatchRecord, + type SpendMeter, +} from "../src/nodeagent/models/adapter"; +import { getModelPricing, lookupModelAlias, priceTokens, resolveModelAlias } from "../src/nodeagent/models/modelCatalog"; +import { modelMaxOutputTokens } from "../src/nodeagent/models/outputTokens"; import { selectOpenRouterFreeModels } from "../src/nodeagent/models/openRouterFreeModels"; import { stableJournalHash } from "../src/nodeagent/core/journal"; @@ -97,6 +112,23 @@ interface Rung { interface RuntimeBudget { rungTimeoutMs?: number; reserveMs: number; + /** Dollar cap for a --real run (LADDER_MAX_COST_USD / --max-cost-usd). Mandatory when any provider is called. */ + maxCostUsd?: number; +} + +/** Every --real run: no adapter retries, no cross-model fallback, so one step = at most one reserved request. */ +export const RECEIPT_DISPATCH_POLICY: DispatchPolicy = { maxRetries: 0, crossModelFallback: false }; + +interface LadderBudgetReport { + maxUsd: number; + estimateUsd: number; + spentUsd: number; + requests: number; + /** Pre-flight request ceiling (routes x rungs x max steps, fan-out included); `requests` above it fails review (A.0.3). */ + requestCeiling: number; + stoppedBy: SpendMeter["stoppedBy"]; + /** Requests past the dispatch-record cap: counted in `requests`/`spentUsd` but absent from `rows`. Non-zero means the receipt's rows are incomplete. */ + dispatchesDropped: number; } interface RungResult { @@ -116,6 +148,8 @@ interface RungResult { checks: Record; /** P1-5: for each FAILING trace-derived check, a pointer at the offending trace event. */ checkDetails?: Record; + /** Every provider request this rung issued (metered runs only): echoed model, usage, cost, fallbackUsed. */ + dispatches?: DispatchRecord[]; trace: AgentTraceEvent[]; } @@ -562,7 +596,7 @@ function checkFailurePointers(rung: Rung, result: AgentResult, checks: Record AgentModel, modelName: string, budget: RuntimeBudget): Promise { +async function runRung(rung: Rung, maker: (rung: Rung) => AgentModel, modelName: string, budget: RuntimeBudget, meter?: SpendMeter): Promise { const env = (rung.makeEnv ?? demoEnv)(); rung.setup?.(env); const rt = new CountingRoomTools(env.engine, env.roomId, env.artifactId, env.actor, env.sessionId, env.stats); @@ -581,6 +615,10 @@ async function runRung(rung: Rung, maker: (rung: Rung) => AgentModel, modelName: }; const t0 = Date.now(); const deadlineAt = budget.rungTimeoutMs ? t0 + budget.rungTimeoutMs : undefined; + const dispatchesFrom = meter?.dispatches.length ?? 0; + // Rung cost is the meter's own spend delta, not the sum of recorded rows: rows stop at MAX_DISPATCH_RECORDS, spend does not. + const spentFrom = meter?.spentUsd ?? 0; + const rungDispatches = () => (meter ? { dispatches: meter.dispatches.slice(dispatchesFrom) } : {}); try { const runOnce = (goal: string, m: AgentModel, maxSteps: number) => runAgent({ rt, @@ -593,6 +631,9 @@ async function runRung(rung: Rung, maker: (rung: Rung) => AgentModel, modelName: deadlineAt, reserveMs: budget.reserveMs, onTrace: (event) => rung.onTrace?.(event, env), + // A metered run hands the runtime the dollars still left job-wide and the strict (throwing) price. + // The tier belongs to one request's prompt, so a step is priced per request (never from aggregated usage). + ...(meter ? { spendLimits: { maxCostUsd: Math.max(0, meter.maxUsd - meter.spentUsd) }, priceStep: priceRequestStrict } : { priceStep: priceRequest }), }); let result: AgentResult; let slice1: AgentResult | undefined; @@ -625,8 +666,11 @@ async function runRung(rung: Rung, maker: (rung: Rung) => AgentModel, modelName: const combinedTrace = slice1 ? [...slice1.trace, { step: -1, tool: "slice_boundary", args: { forced: true, slice1Exhausted: slice1.exhausted }, result: { handoff: slice1.handoff?.summary ?? null }, ms: 0 } as AgentTraceEvent, ...result.trace] : result.trace; - const cost = (result.usage ? priceRun(resolvedModel, result.usage.inputTokens, result.usage.outputTokens) : 0) - + (slice1?.usage ? priceRun(resolvedModel, slice1.usage.inputTokens, slice1.usage.outputTokens) : 0); + // Per-request costs only: the meter's settled dispatches on a metered run (a paid run never costs a guessed + // $1/$5 row), the runtime's per-step priced total otherwise. Never price summed usage: a tier is per request. + const cost = meter + ? meter.spentUsd - spentFrom + : (result.usage?.costUsd ?? 0) + (slice1?.usage?.costUsd ?? 0); return { pass, checkDetails, @@ -642,6 +686,7 @@ async function runRung(rung: Rung, maker: (rung: Rung) => AgentModel, modelName: handoff: result.handoff?.summary, reason: pass ? "" : rung.diagnose?.(result, env) ?? result.handoff?.summary ?? "failed", checks, + ...rungDispatches(), trace: combinedTrace, }; } catch (error) { @@ -655,9 +700,11 @@ async function runRung(rung: Rung, maker: (rung: Rung) => AgentModel, modelName: level: rung.level, ms: Date.now() - t0, tools: 0, - cost: 0, + // A rung that failed after issuing charged dispatches still spent that money: the row carries it, as in the try branch. + cost: meter ? meter.spentUsd - spentFrom : 0, error: error instanceof Error ? error.message : String(error), checks: { completed: false }, + ...rungDispatches(), trace: [], }; } @@ -691,6 +738,7 @@ function parseRuntimeBudget(): RuntimeBudget { return { rungTimeoutMs: optionNumber("--rung-timeout-ms", "LADDER_RUNG_TIMEOUT_MS"), reserveMs: optionNumber("--reserve-ms", "LADDER_RESERVE_MS") ?? 5_000, + maxCostUsd: optionNumber("--max-cost-usd", "LADDER_MAX_COST_USD"), }; } @@ -737,17 +785,45 @@ async function parseModelRoutes(): Promise { return routes; } -function writeJsonReport(results: RungResult[], selectedRungs: Rung[], requestedRoutes: string[]): void { +/** What makes a --real report a receipt (SPEC A.0.3): the tested commit, the constant it tested, the spend, the policy. */ +interface ReportReceipt { + startedAt: string; + budget: LadderBudgetReport | null; + dispatchPolicy: DispatchPolicy | null; +} + +/** Run environment a reviewer needs to reproduce the receipt. No keys, no hostnames, no paths. */ +function receiptEnvironment(): { node: string; platform: string; ci: boolean } { + return { node: process.version, platform: process.platform, ci: Boolean(process.env.CI) }; +} + +function sha256OfFile(path: string): string | null { + try { return createHash("sha256").update(readFileSync(path)).digest("hex"); } catch { return null; } +} + +function writeJsonReport(results: RungResult[], selectedRungs: Rung[], requestedRoutes: string[], receipt: ReportReceipt): void { const outPath = optionValue("--json-out"); if (!outPath) return; const absolute = resolve(process.cwd(), outPath); mkdirSync(dirname(absolute), { recursive: true }); + const real = receipt.budget !== null; + const identity = real ? readGitIdentity() : undefined; + // One flat row per provider request (A.0.3 `rows`), tagged with the rung that issued it. + const rows = results.flatMap((r) => (r.dispatches ?? []).map((d) => ({ rung: r.rung, ...d }))); + const echoedModels = [...new Set(rows.map((r) => r.echoedModel).filter((m): m is string => m !== null))]; writeFileSync( absolute, JSON.stringify({ generatedAt: new Date().toISOString(), + startedAt: receipt.startedAt, + ...(real ? { command: ["tsx", "evals/ladder.ts", ...process.argv.slice(2)].join(" ") } : {}), requestedRoutes, + ...(real ? { candidateIds: requestedRoutes, echoedModels, rows } : {}), rungs: selectedRungs.map((r) => ({ id: r.id, level: r.level, label: r.label })), + ...(identity ? { testedCommit: identity.commitSha, gitDirty: identity.gitDirty, constantSha256: sha256OfFile("convex/modelDefaults.ts") } : {}), + budget: receipt.budget, + dispatchPolicy: receipt.dispatchPolicy, + ...(real ? { environment: receiptEnvironment() } : {}), results: results.map(publicResult), }, null, 2), ); @@ -817,7 +893,8 @@ function readGitIdentity(): GitIdentity { let commitSha = "nocommit"; let status = ""; let diff = ""; - try { commitSha = execSync("git rev-parse --short HEAD", { stdio: ["ignore", "pipe", "ignore"] }).toString().trim(); } catch { /* not a git repo */ } + // Full SHA: the receipt's testedCommit must be an ancestor-checkable 40-hex id (SPEC A.0.3), not a short prefix. + try { commitSha = execSync("git rev-parse HEAD", { stdio: ["ignore", "pipe", "ignore"] }).toString().trim(); } catch { /* not a git repo */ } try { status = execSync("git status --porcelain=v1", { stdio: ["ignore", "pipe", "ignore"] }).toString(); } catch { /* not a git repo */ } try { diff = execSync("git diff --no-ext-diff --binary HEAD --", { stdio: ["ignore", "pipe", "ignore"], maxBuffer: 16 * 1024 * 1024 }).toString(); } catch { /* large or unavailable diff */ } const gitDirty = status.trim().length > 0; @@ -857,24 +934,134 @@ function normalizePath(value: string): string { return value.replace(/\\/g, "/"); } -async function main() { +/** Opening-context size and per-step growth of one rung, read from a scripted dry run (zero provider requests). */ +interface RungShape { + rung: Rung; + ctxTokens0: number; + growthTokens: number; + /** Provider-request upper bound for one model on this rung (resume rungs run two slices). */ + maxSteps: number; +} + +async function scriptedRungShape(rung: Rung, budget: RuntimeBudget): Promise { + const sizes: number[] = []; + const probe = (r: Rung): AgentModel => { + const base = r.scripted(); + return { + get name() { return base.name; }, + async next(input) { + sizes.push(requestInputTokenUpperBound(input.system, input.messages, input.tools)); + return base.next(input); + }, + }; + }; + await runRung(rung, probe, "scripted", { ...budget, maxCostUsd: undefined }); + const ctxTokens0 = sizes[0] ?? 0; + const growthTokens = sizes.length > 1 ? Math.max(0, ((sizes.at(-1) ?? 0) - ctxTokens0) / (sizes.length - 1)) : 0; + const maxSteps = (rung.maxSteps ?? 18) + (rung.resume?.slice1MaxSteps ?? 0); + return { rung, ctxTokens0, growthTokens, maxSteps }; +} + +/** + * Provider-request ceiling for the grid: routes x rungs x max steps. A free-auto route fans out over the candidate + * limit the ADAPTER enforces (`openRouterFreeAutoLimit`), not over `--free-auto-top`, which only decides which extra + * concrete routes join the grid. + */ +export function ladderRequestCeiling(routes: string[], shapes: RungShape[], freeAutoCandidates = openRouterFreeAutoLimit()): number { + const steps = shapes.reduce((sum, shape) => sum + shape.maxSteps, 0); + return routes.reduce((sum, route) => sum + routeFanOut(route, freeAutoCandidates) * steps, 0); +} + +/** Dispatch resolves aliases first ("free", "openrouter-auto" -> openrouter/free-auto), so the fan-out test must too. */ +function routeFanOut(route: string, freeAutoCandidates: number): number { + return /free-auto$/i.test(resolveModelAlias(route)) ? Math.max(1, freeAutoCandidates) : 1; +} + +/** + * Worst-case USD before the first paid request: + * Σ_models Σ_rungs Σ_{step=1..maxSteps} ((ctx0 + (step-1)·growth) · inPerM + maxOut · outPerM) / 1e6. + * ctx0 and growth come from the scripted dry run, so this is a pre-flight sanity bound ("can the cap cover a full + * grid at all?"), not the spend guard: the guard is the per-dispatch reservation, which prices each request's real + * input bytes. A free-auto route fans out over its candidates, so it multiplies by the candidate limit. The + * request count holds only under RECEIPT_DISPATCH_POLICY: with retries and fallback each step could issue several. + */ +export function estimateLadderUsd(routes: string[], shapes: RungShape[], freeAutoCandidates = openRouterFreeAutoLimit()): number { + let total = 0; + for (const route of routes) { + const pricing = getModelPricing(resolveModelAlias(route)); + if (!pricing) throw new Error(`no pricing row for "${route}"`); + const maxOut = modelMaxOutputTokens(resolveModelAlias(route)); + const fanOut = routeFanOut(route, freeAutoCandidates); + for (const shape of shapes) { + for (let step = 1; step <= shape.maxSteps; step++) { + total += fanOut * priceTokens(pricing, shape.ctxTokens0 + (step - 1) * shape.growthTokens, maxOut); + } + } + } + return total; +} + +export async function main() { const realModels = await parseModelRoutes(); const budget = parseRuntimeBudget(); - const models: { name: string; maker: (rung: Rung) => AgentModel }[] = realModels.length - ? realModels.map((id) => ({ name: id, maker: () => realModel(id) })) - : [{ name: "scripted", maker: (rung) => rung.scripted() }]; - + const startedAt = new Date().toISOString(); const rungFilter = parseRungFilter(); const selectedRungs = rungFilter ? RUNGS.filter((r) => rungFilter.has(r.id)) : RUNGS; + const real = realModels.length > 0; + + let meter: SpendMeter | undefined; + let estimateUsd = 0; + let requestCeiling = 0; + const budgetReport = (): LadderBudgetReport | null => (meter + ? { maxUsd: meter.maxUsd, estimateUsd, spentUsd: meter.spentUsd, requests: meter.requests, requestCeiling, stoppedBy: meter.stoppedBy, dispatchesDropped: meter.droppedDispatches } + : null); + const refuse = (message: string) => { + console.error(`ladder: ${message}`); + writeJsonReport([], selectedRungs, realModels, { startedAt, budget: budgetReport(), dispatchPolicy: RECEIPT_DISPATCH_POLICY }); + process.exitCode = 1; + }; + + if (real) { + // A paid run needs a dollar cap, and every id it will call must have a verified price: no cap, no guessed row. + if (budget.maxCostUsd === undefined) { + console.error("ladder: --real needs a dollar cap (LADDER_MAX_COST_USD or --max-cost-usd). Refusing to start; no request was made."); + process.exitCode = 1; + return; + } + meter = createSpendMeter(budget.maxCostUsd); + const unpriced = realModels.find((id) => !getModelPricing(lookupModelAlias(id))); + if (unpriced) { + meter.stoppedBy = "unpriced_model"; + refuse(`no pricing row for "${unpriced}"; add a verified row (modelCatalog.modelPricing) before a paid run. No request was made.`); + return; + } + const shapes: RungShape[] = []; + for (const rung of selectedRungs) shapes.push(await scriptedRungShape(rung, budget)); + estimateUsd = estimateLadderUsd(realModels, shapes); + requestCeiling = ladderRequestCeiling(realModels, shapes); + console.log(`pre-flight: worst case $${estimateUsd.toFixed(4)} for ${realModels.length} route(s) x ${selectedRungs.length} rung(s); cap $${meter.maxUsd.toFixed(4)}`); + if (estimateUsd > meter.maxUsd) { + meter.stoppedBy = "spend_budget"; + refuse(`worst-case estimate $${estimateUsd.toFixed(4)} exceeds the $${meter.maxUsd.toFixed(4)} cap. Raise the cap or narrow --levels; no request was made.`); + return; + } + } + + const models: { name: string; maker: (rung: Rung) => AgentModel }[] = real + ? realModels.map((id) => ({ name: id, maker: () => realModel(id, { dispatch: RECEIPT_DISPATCH_POLICY, meter, operatorSelected: true }) })) + : [{ name: "scripted", maker: (rung) => rung.scripted() }]; const grid: Record> = {}; const results: RungResult[] = []; for (const model of models) { grid[model.name] = {}; for (const rung of selectedRungs) { + if (meter?.stoppedBy) break; // the cap is reached: no further rung may spend process.stdout.write(` ${model.name} · ${rung.id} ... `); - const res = await runRung(rung, model.maker, model.name, budget); + const res = await runRung(rung, model.maker, model.name, budget, meter); results.push(res); + // The runtime's own pre-call gate can end a rung on the cap before the adapter is asked to dispatch. + if (meter && res.stopReason === "spend_budget") meter.stoppedBy ??= "spend_budget"; grid[model.name][rung.id] = res.pass; const stop = "stopReason" in res && res.stopReason !== "done" ? `, ${res.stopReason}` : ""; const resolved = res.resolvedModel !== model.name ? `, resolved=${res.resolvedModel}` : ""; @@ -891,13 +1078,18 @@ async function main() { } console.log("-".repeat(72)); console.log("legend: " + selectedRungs.map((r) => `L${r.level}=${r.label}`).join(" | ")); - writeJsonReport(results, selectedRungs, models.map((m) => m.name)); + if (meter) console.log(`spend: $${meter.spentUsd.toFixed(4)} of $${meter.maxUsd.toFixed(4)} over ${meter.requests} provider request(s)${meter.droppedDispatches ? `; ${meter.droppedDispatches} request(s) have no receipt row (cap ${MAX_DISPATCH_RECORDS})` : ""}${meter.stoppedBy ? `; STOPPED by ${meter.stoppedBy} (not a receipt)` : ""}`); + writeJsonReport(results, selectedRungs, models.map((m) => m.name), { startedAt, budget: budgetReport(), dispatchPolicy: real ? RECEIPT_DISPATCH_POLICY : null }); recordRunsToStore(results); + if (meter?.stoppedBy) process.exitCode = 1; // a budget stop is an incomplete run, never a pass + if (meter?.droppedDispatches) process.exitCode = 1; // requests past the record cap have no receipt row: incomplete, never a pass if (Object.values(grid).some((row) => Object.values(row).some((ok) => !ok))) process.exitCode = 1; } -main().catch((error) => { - console.error(error); - process.exit(1); -}); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error); + process.exit(1); + }); +} diff --git a/evals/ladderBudget.test.ts b/evals/ladderBudget.test.ts new file mode 100644 index 00000000..9135541d --- /dev/null +++ b/evals/ladderBudget.test.ts @@ -0,0 +1,528 @@ +/** + * Ladder spend budget (SPEC Part A §A.2.9 "Ladder budget", Codex R1, v4-new-2, v5-new-1). + * + * Tests the ACTUAL adapter path, not a fake AgentModel: `model(id, { dispatch, meter })` runs the real AI-SDK + * Google provider with `fetch` replaced by a fake transport that counts physical requests. The claims: + * - the meter moves at DISPATCHES (one reservation per physical request, retries and fallbacks included); + * - a request whose worst case would cross the cap is never issued; + * - the AI SDK's own retry loop adds no request of its own (pinned to 0); + * - an id the catalog cannot price is refused before any request, never reserved at a guessed price. + */ +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + createSpendMeter, + model, + priceRequestStrict, + requestInputTokenUpperBound, + reserveDispatch, + SpendBudgetError, + UnpricedModelError, + type DispatchPolicy, +} from "../src/nodeagent/models/adapter"; +import { UnknownModelError } from "../src/nodeagent/models/modelCatalog"; +import { modelMaxOutputTokens } from "../src/nodeagent/models/outputTokens"; + +const PRICED = "gemini-3.8-flash"; +// Round 3 (SPEC A.0.10): an id the catalog has never heard of no longer reaches the adapter at all. `UNPRICED` was +// "routes to the Google provider, has no pricing row" and used to be dispatched; it now throws UnknownModelError at +// model() (see the A.0.10 test below). The (g) invariants - zero requests, meter untouched, terminal stop - are about an +// id the adapter CAN route but the catalog cannot price, which is now only an id the alias table declares without a +// price row: a persisted-room string such as "claude-sonnet-4.5" (resolves to claude-sonnet-4-5, priced nowhere). +const UNPRICED = "gemini-9.9-unpriced-test"; +const UNPRICED_ALIAS = "claude-sonnet-4.5"; +const RECEIPT: DispatchPolicy = { maxRetries: 0, crossModelFallback: false }; +const TURN = { system: "s", messages: [{ role: "user" as const, content: "hi" }], tools: [] }; +const BOUND = requestInputTokenUpperBound(TURN.system, TURN.messages, TURN.tools); +const RESERVE = reserveDispatch(PRICED, BOUND, Number.POSITIVE_INFINITY); + +type Call = { url: string; body: Record }; +let calls: Call[] = []; +let billedUsd = 0; + +function ok(inputTokens = 1_000, outputTokens = 200) { + return new Response(JSON.stringify({ + candidates: [{ content: { role: "model", parts: [{ text: "ok" }] }, finishReason: "STOP", index: 0 }], + usageMetadata: { promptTokenCount: inputTokens, candidatesTokenCount: outputTokens, totalTokenCount: inputTokens + outputTokens }, + }), { status: 200, headers: { "content-type": "application/json" } }); +} +const unavailable = () => new Response(JSON.stringify({ error: { code: 503, message: "Service unavailable", status: "UNAVAILABLE" } }), { status: 503, headers: { "content-type": "application/json" } }); + +function fakeTransport(answer: (n: number, call: Call) => Response | Promise) { + calls = []; + vi.stubGlobal("fetch", async (input: string | URL | Request, init?: RequestInit) => { + const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; + const call: Call = { url, body: JSON.parse(typeof init?.body === "string" ? init.body : "{}") }; + calls.push(call); + return answer(calls.length, call); + }); +} + +const SAVED = { key: process.env.GOOGLE_GENERATIVE_AI_API_KEY, fb: process.env.AGENT_FALLBACK_MODEL, cap: process.env.AGENT_MODEL_MAX_OUTPUT_TOKENS, ladderCap: process.env.LADDER_MAX_COST_USD }; +beforeEach(() => { + process.env.GOOGLE_GENERATIVE_AI_API_KEY = "test-google-key"; + delete process.env.AGENT_FALLBACK_MODEL; + delete process.env.AGENT_MODEL_MAX_OUTPUT_TOKENS; + delete process.env.LADDER_MAX_COST_USD; + billedUsd = 0; +}); +afterEach(() => { + vi.unstubAllGlobals(); + vi.useRealTimers(); + for (const [k, v] of [["GOOGLE_GENERATIVE_AI_API_KEY", SAVED.key], ["AGENT_FALLBACK_MODEL", SAVED.fb], ["AGENT_MODEL_MAX_OUTPUT_TOKENS", SAVED.cap], ["LADDER_MAX_COST_USD", SAVED.ladderCap]] as const) { + if (v === undefined) delete process.env[k]; else process.env[k] = v; + } +}); + +describe("meter moves at dispatches (adapter path, fake transport)", () => { + it("(a) a charged response then ECONNRESET under the receipt policy: exactly two dispatches, error propagates, failed reservation is kept", async () => { + fakeTransport((n) => { + if (n === 1) return ok(1_000, 200); + throw Object.assign(new Error("fetch failed: ECONNRESET"), { code: "ECONNRESET" }); + }); + const meter = createSpendMeter(10); + const m = model(PRICED, { dispatch: RECEIPT, meter }); + + await m.next(TURN); + expect(meter.requests).toBe(1); + expect(meter.spentUsd).toBeCloseTo(priceRequestStrict(PRICED, 1_000, 200), 12); // reservation replaced by the priced usage + + await expect(m.next(TURN)).rejects.toThrow(/ECONNRESET/); + expect(calls).toHaveLength(2); // maxRetries 0: no retry, no SDK retry, no fallback + expect(meter.requests).toBe(2); + expect(meter.spentUsd).toBeCloseTo(priceRequestStrict(PRICED, 1_000, 200) + RESERVE, 12); // the failed dispatch's reservation is retained + }); + + it("(b) the same sequence (charged response, then ECONNRESET) under the default policy reserves once per attempt and never crosses the cap", async () => { + vi.useFakeTimers(); + fakeTransport((n) => { + if (n === 1) return ok(1_000, 200); + throw Object.assign(new Error("fetch failed: ECONNRESET"), { code: "ECONNRESET" }); // transient: the default policy retries it + }); + const first = priceRequestStrict(PRICED, 1_000, 200); + const meter = createSpendMeter(first + RESERVE * 2.5); // after the charged answer: room for two more reservations, not three + const m = model(PRICED, { meter }); // default policy: 3 retries, fallback allowed (no AGENT_FALLBACK_MODEL set) + + await m.next(TURN); + expect(calls).toHaveLength(1); + expect(meter.spentUsd).toBeCloseTo(first, 12); + + const outcome = m.next(TURN).then(() => null, (e: unknown) => e); + await vi.advanceTimersByTimeAsync(120_000); + const error = await outcome; + + expect(error).toBeInstanceOf(SpendBudgetError); + expect((error as SpendBudgetError).receipt.stopReason).toBe("spend_budget"); + expect(calls).toHaveLength(3); // the charged answer + two reset attempts; the attempt whose reservation would cross the cap is not issued + expect(meter.requests).toBe(3); + expect(meter.spentUsd).toBeCloseTo(first + RESERVE * 2, 12); // each issued attempt kept its own reservation + expect(meter.spentUsd).toBeLessThanOrEqual(meter.maxUsd); + expect(meter.stoppedBy).toBe("spend_budget"); + }); + + it("every issued request leaves one receipt row: requested/resolved id, the provider's echoed model, usage, cost, fallbackUsed", async () => { + fakeTransport(() => new Response(JSON.stringify({ + candidates: [{ content: { role: "model", parts: [{ text: "ok" }] }, finishReason: "STOP", index: 0 }], + usageMetadata: { promptTokenCount: 1_000, candidatesTokenCount: 200, totalTokenCount: 1_200 }, + modelVersion: "gemini-3.6-flash", // Google serving the request under another model than the one asked for + }), { status: 200, headers: { "content-type": "application/json" } })); + const meter = createSpendMeter(10); + await model(PRICED, { dispatch: RECEIPT, meter }).next(TURN); + expect(meter.dispatches).toEqual([{ + requestedModel: PRICED, + resolvedModel: PRICED, + echoedModel: "gemini-3.6-flash", + usage: { inputTokens: 1_000, outputTokens: 200 }, + costUsd: priceRequestStrict(PRICED, 1_000, 200), + fallbackUsed: false, + }]); + }); + + it("a cross-model fallback dispatch is marked fallbackUsed: true (the receipt rejects such a row)", async () => { + vi.useFakeTimers(); + process.env.AGENT_FALLBACK_MODEL = "gemini-3.5-flash-lite"; + fakeTransport((_n, call) => call.url.includes("/models/gemini-3.5-flash-lite:") ? ok(100, 10) : unavailable()); + const meter = createSpendMeter(10); + const outcome = model(PRICED, { meter }).next(TURN); // default policy: 4 attempts on the primary, then the fallback model + await vi.advanceTimersByTimeAsync(120_000); + await outcome; + expect(meter.dispatches.map((d) => [d.resolvedModel, d.requestedModel, d.fallbackUsed])).toEqual([ + [PRICED, PRICED, false], [PRICED, PRICED, false], [PRICED, PRICED, false], [PRICED, PRICED, false], + ["gemini-3.5-flash-lite", PRICED, true], + ]); + }); + + it("a request that throws keeps its full reservation as its row cost, with no echoed model or usage", async () => { + fakeTransport(() => { throw Object.assign(new Error("fetch failed: ECONNRESET"), { code: "ECONNRESET" }); }); + const meter = createSpendMeter(10); + await expect(model(PRICED, { dispatch: RECEIPT, meter }).next(TURN)).rejects.toThrow(/ECONNRESET/); + expect(meter.dispatches).toEqual([{ requestedModel: PRICED, resolvedModel: PRICED, echoedModel: null, usage: null, costUsd: RESERVE, fallbackUsed: false }]); + }); + + it("(c) a reservation above the remaining budget before the first dispatch throws spend_budget with zero fetch calls", async () => { + fakeTransport(() => ok()); + const meter = createSpendMeter(RESERVE / 2); + await expect(model(PRICED, { dispatch: RECEIPT, meter }).next(TURN)).rejects.toMatchObject({ receipt: { stopReason: "spend_budget" } }); + expect(calls).toHaveLength(0); + expect(meter.requests).toBe(0); + expect(meter.spentUsd).toBe(0); + }); + + it("(d) every issued request body carries the output cap the reservation priced", async () => { + fakeTransport(() => ok()); + const meter = createSpendMeter(10); + const m = model(PRICED, { dispatch: RECEIPT, meter }); + await m.next(TURN); + await m.next(TURN); + expect(calls.map((c) => c.body.generationConfig?.maxOutputTokens)).toEqual([modelMaxOutputTokens(), modelMaxOutputTokens()]); + process.env.AGENT_MODEL_MAX_OUTPUT_TOKENS = "2048"; + await m.next(TURN); + expect(calls.at(-1)!.body.generationConfig.maxOutputTokens).toBe(2048); + }); + + it("(f) retryable 503s under { maxRetries: 3 }: exactly four requests, four reservations; the SDK adds none", async () => { + vi.useFakeTimers(); + fakeTransport(() => unavailable()); + const meter = createSpendMeter(10); + const m = model(PRICED, { dispatch: { maxRetries: 3, crossModelFallback: false }, meter }); + const outcome = m.next(TURN).then(() => null, (e: unknown) => e); + await vi.advanceTimersByTimeAsync(120_000); + expect(await outcome).toBeInstanceOf(Error); + expect(calls).toHaveLength(4); + expect(meter.requests).toBe(4); + expect(meter.spentUsd).toBeCloseTo(RESERVE * 4, 12); + }); +}); + +describe("(g) zero requests for an unpriced id", () => { + it("model() with a $10 meter and at the $0.01 equality case: UnpricedModelError, zero fetch calls, meter untouched", async () => { + fakeTransport(() => ok()); // the transport would answer 200 if anything were sent + for (const maxUsd of [10, 0.01]) { + const meter = createSpendMeter(maxUsd); + const error = await model(UNPRICED_ALIAS, { dispatch: RECEIPT, meter }).next(TURN).then(() => null, (e: unknown) => e); + expect(error).toBeInstanceOf(UnpricedModelError); + expect((error as UnpricedModelError).receipt.stopReason).toBe("spend_budget"); + expect(meter.requests).toBe(0); + expect(meter.spentUsd).toBe(0); + expect(meter.stoppedBy).toBe("unpriced_model"); + } + expect(calls).toHaveLength(0); + }); + + it("(A.0.10) an id the catalog has never heard of never reaches the adapter: model() throws before any request, metered or not", async () => { + fakeTransport(() => ok()); + const meter = createSpendMeter(10); + expect(() => model(UNPRICED, { dispatch: RECEIPT, meter })).toThrow(UnknownModelError); + expect(() => model(UNPRICED)).toThrow(UnknownModelError); // the unmetered legacy caller is refused too: no paid default + expect(calls).toHaveLength(0); + expect(meter.requests).toBe(0); + expect(meter.spentUsd).toBe(0); + }); + + it("a priced primary answering four 503s with an unpriced AGENT_FALLBACK_MODEL makes the primary's four requests and none for the fallback", async () => { + vi.useFakeTimers(); + process.env.AGENT_FALLBACK_MODEL = UNPRICED_ALIAS; + fakeTransport(() => unavailable()); + const meter = createSpendMeter(10); + const outcome = model(PRICED, { meter }).next(TURN).then(() => null, (e: unknown) => e); // default policy + await vi.advanceTimersByTimeAsync(120_000); + expect(await outcome).toBeInstanceOf(UnpricedModelError); + expect(calls).toHaveLength(4); + expect(calls.every((c) => c.url.includes(`/models/${PRICED}:`))).toBe(true); + expect(meter.requests).toBe(4); + }); + + it("a terminal stop ends the dispatch: an unpriced primary never reaches a priced AGENT_FALLBACK_MODEL", async () => { + process.env.AGENT_FALLBACK_MODEL = PRICED; // would be affordable and priced: must still not be dispatched + fakeTransport(() => ok()); + const meter = createSpendMeter(10); + const error = await model(UNPRICED_ALIAS, { meter }).next(TURN).then(() => null, (e: unknown) => e); // default policy: fallback allowed + expect(error).toBeInstanceOf(UnpricedModelError); + expect(meter.stoppedBy).toBe("unpriced_model"); + expect(calls).toHaveLength(0); + expect(meter.requests).toBe(0); + }); + + it("a terminal stop ends the dispatch: a cap-refused primary never reaches a cheaper AGENT_FALLBACK_MODEL", async () => { + process.env.AGENT_FALLBACK_MODEL = "gemini-3.5-flash-lite"; // its reservation would fit under the cap + fakeTransport(() => ok()); + const cheap = reserveDispatch("gemini-3.5-flash-lite", BOUND, Number.POSITIVE_INFINITY); + expect(cheap).toBeLessThan(RESERVE); + const meter = createSpendMeter((RESERVE + cheap) / 2); // primary does not fit, the fallback would + const error = await model(PRICED, { meter }).next(TURN).then(() => null, (e: unknown) => e); + expect(error).toBeInstanceOf(SpendBudgetError); + expect(meter.stoppedBy).toBe("spend_budget"); + expect(calls).toHaveLength(0); + expect(meter.requests).toBe(0); + expect(meter.spentUsd).toBe(0); + }); + + it("after a stop the meter refuses every further dispatch, even a different model that would fit", async () => { + fakeTransport(() => ok()); + const meter = createSpendMeter(10); + meter.stoppedBy = "spend_budget"; + await expect(model("gemini-3.5-flash-lite", { dispatch: RECEIPT, meter }).next(TURN)).rejects.toBeInstanceOf(SpendBudgetError); + expect(calls).toHaveLength(0); + }); + + it("an unmetered adapter is unchanged: no reservation, no refusal", async () => { + const savedKey = process.env.ANTHROPIC_API_KEY; + process.env.ANTHROPIC_API_KEY = "test-anthropic-key"; + try { + fakeTransport(() => anthropicAnswer(10, 5)); + const step = await model(UNPRICED_ALIAS).next(TURN); // no meter: legacy callers are not budget-gated + expect(step.text).toBe("done"); + expect(calls).toHaveLength(1); + } finally { + if (savedKey === undefined) delete process.env.ANTHROPIC_API_KEY; else process.env.ANTHROPIC_API_KEY = savedKey; + } + }); +}); + +describe("main(): pre-flight and the grid", () => { + let dir: string; + const argvBefore = process.argv; + const exitBefore = process.exitCode; + beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "ladder-budget-")); vi.spyOn(console, "log").mockImplementation(() => {}); vi.spyOn(console, "error").mockImplementation(() => {}); }); + afterEach(() => { process.argv = argvBefore; process.exitCode = exitBefore; rmSync(dir, { recursive: true, force: true }); vi.restoreAllMocks(); }); + + async function runMain(args: string[]) { + const out = join(dir, "report.json"); + process.argv = ["node", "ladder.ts", ...args, "--json-out", out]; + process.exitCode = undefined; + const { main } = await import("./ladder"); + await main(); + const exitCode = process.exitCode; + process.exitCode = exitBefore; + let report: Record | undefined; + try { report = JSON.parse(readFileSync(out, "utf8")); } catch { /* refusal before any report */ } + return { exitCode, report }; + } + + it("refuses --real without a dollar cap: exit 1, no request", async () => { + fakeTransport(() => ok()); + const { exitCode } = await runMain(["--real", PRICED, "--levels=1-4"]); + expect(exitCode).toBe(1); + expect(calls).toHaveLength(0); + }); + + it("(g) --real with an unpriced id exits 1 at pre-flight: stoppedBy unpriced_model, no result rows, zero fetch calls", async () => { + fakeTransport(() => ok()); + const { exitCode, report } = await runMain(["--real", UNPRICED, "--levels=1-4", "--max-cost-usd=1"]); + expect(exitCode).toBe(1); + expect(report?.budget).toMatchObject({ stoppedBy: "unpriced_model", requests: 0, spentUsd: 0 }); + expect(report?.results).toEqual([]); + expect(report?.dispatchPolicy).toEqual(RECEIPT); + expect(calls).toHaveLength(0); + }); + + it("refuses to start when the worst-case estimate exceeds the cap (the human raises the cap, never the check)", async () => { + fakeTransport(() => ok()); + const { exitCode, report } = await runMain(["--real", PRICED, "--levels=1-4", "--max-cost-usd=0.0001"]); + expect(exitCode).toBe(1); + expect(report?.budget.estimateUsd).toBeGreaterThan(0.0001); + expect(report?.budget).toMatchObject({ stoppedBy: "spend_budget", requests: 0 }); + expect(calls).toHaveLength(0); + }); + + it("(e) usage within each reservation, a context that outgrows the scripted pre-flight: the grid stops BEFORE the cap, spentUsd <= maxUsd, budget.requests equals the fetch count", async () => { + // Pass the pre-flight with the smallest admissible cap (the estimate itself), then answer every step with a tool + // call plus 20K characters of text so the real prompt outgrows the scripted dry run the estimate was built on. + // Each answer bills half its request body in tokens (realistic, and below the 1-byte-per-token reservation), so + // no settlement ever exceeds its reservation: the only thing that can end the grid is a refused reservation. + fakeTransport(() => ok()); + const probe = await runMain(["--real", PRICED, "--levels=1-4", "--max-cost-usd=0.0001"]); + const cap = Math.ceil(probe.report!.budget.estimateUsd * 1e4) / 1e4; // >= the estimate, so the pre-flight admits it + expect(calls).toHaveLength(0); + + fakeTransport((_n, call) => { + const tokens = Math.floor(JSON.stringify(call.body).length / 2); + billedUsd += priceRequestStrict(PRICED, tokens, 50); + return new Response(JSON.stringify({ + candidates: [{ content: { role: "model", parts: [{ text: "x".repeat(20_000) }, { functionCall: { name: "list_artifacts", args: {} } }] }, finishReason: "STOP", index: 0 }], + usageMetadata: { promptTokenCount: tokens, candidatesTokenCount: 50, totalTokenCount: tokens + 50 }, + modelVersion: PRICED, + }), { status: 200, headers: { "content-type": "application/json" } }); + }); + + const { exitCode, report } = await runMain(["--real", PRICED, "--levels=1-4", `--max-cost-usd=${cap}`]); + const budget = report!.budget; + expect(budget.stoppedBy).toBe("spend_budget"); // a reservation was refused... + expect(budget.spentUsd).toBeLessThanOrEqual(budget.maxUsd); // ...so the cap was never crossed + expect(budget.spentUsd).toBeLessThan(budget.maxUsd); // the grid stopped short of it, not on it + expect(billedUsd).toBeLessThanOrEqual(budget.maxUsd); + expect(calls.length).toBeGreaterThan(2); + expect(budget.requests).toBe(calls.length); // nothing was issued after the stop + expect(budget.requests).toBeLessThan(budget.requestCeiling); + expect(exitCode).toBe(1); // a stop is not a receipt + }, 30_000); + + it("the --json-out report carries what an A.0.3 review needs: command, candidateIds, echoedModels, rows, environment, requestCeiling", async () => { + fakeTransport(() => new Response(JSON.stringify({ + candidates: [{ content: { role: "model", parts: [{ text: "done" }] }, finishReason: "STOP", index: 0 }], + usageMetadata: { promptTokenCount: 500, candidatesTokenCount: 20, totalTokenCount: 520 }, + modelVersion: "gemini-3.6-flash", // Google answering as another model: the receipt must show it + }), { status: 200, headers: { "content-type": "application/json" } })); + const probe = await runMain(["--real", PRICED, "--levels=1-4", "--max-cost-usd=0.0001"]); + const cap = Math.ceil(probe.report!.budget.estimateUsd * 1e4) / 1e4; + + const { report } = await runMain(["--real", PRICED, "--levels=1-4", `--max-cost-usd=${cap}`]); + expect(report?.command).toContain(`--real ${PRICED}`); + expect(report?.candidateIds).toEqual([PRICED]); + expect(report?.echoedModels).toEqual(["gemini-3.6-flash"]); + expect(report?.environment).toMatchObject({ node: process.version, platform: process.platform }); + expect(report?.rows.length).toBe(calls.length); + expect(report?.rows[0]).toMatchObject({ + requestedModel: PRICED, + resolvedModel: PRICED, + echoedModel: "gemini-3.6-flash", + usage: { inputTokens: 500, outputTokens: 20 }, + fallbackUsed: false, + }); + expect(report?.budget.requests).toBeLessThanOrEqual(report?.budget.requestCeiling); + expect(report?.results.every((r: { dispatches: unknown[] }) => Array.isArray(r.dispatches))).toBe(true); + }); + + it("a rung that issued charged dispatches and then failed on a provider error reports what it spent, not 0 (round 3)", async () => { + // Persona: the operator runs the L1 rung on a paid route; the network resets on the second request. The rung row + // used to say cost 0 while its own dispatch rows and the job-wide meter carried the charge (Codex: 0.05301075 vs 0). + fakeTransport(() => ok()); + const probe = await runMain(["--real", PRICED, "--levels=1-1", "--max-cost-usd=0.0001"]); + const cap = Math.ceil(probe.report!.budget.estimateUsd * 1e4) / 1e4; + + fakeTransport((n) => { + if (n === 1) { + return new Response(JSON.stringify({ + candidates: [{ content: { role: "model", parts: [{ functionCall: { name: "list_artifacts", args: {} } }] }, finishReason: "STOP", index: 0 }], + usageMetadata: { promptTokenCount: 2_000, candidatesTokenCount: 40, totalTokenCount: 2_040 }, + modelVersion: PRICED, + }), { status: 200, headers: { "content-type": "application/json" } }); + } + throw Object.assign(new Error("fetch failed: ECONNRESET"), { code: "ECONNRESET" }); + }); + const { report } = await runMain(["--real", PRICED, "--levels=1-1", `--max-cost-usd=${cap}`]); + expect(calls).toHaveLength(2); // the charged answer, then the reset (receipt policy: no retry, no fallback) + const row = report!.results[0]; + expect(row.error).toMatch(/ECONNRESET/); + expect(row.pass).toBe(false); + const dispatchCost = row.dispatches.reduce((sum: number, d: { costUsd: number }) => sum + d.costUsd, 0); + expect(dispatchCost).toBeGreaterThan(0); + expect(row.cost).toBeCloseTo(dispatchCost, 12); + expect(row.cost).toBeCloseTo(report!.budget.spentUsd, 12); + expect(row.dispatches[0].costUsd).toBeCloseTo(priceRequestStrict(PRICED, 2_000, 40), 12); // the settled one + }); +}); + +// ───────────────────────────── review round 2 ───────────────────────────── + +/** An Anthropic Messages answer reporting `inputTokens` / `outputTokens`; `toolUse` makes the step ask for a tool. */ +function anthropicAnswer(inputTokens: number, outputTokens: number, toolUse = false) { + return new Response(JSON.stringify({ + id: "msg_test", type: "message", role: "assistant", model: "claude-haiku-5-5", + content: toolUse ? [{ type: "tool_use", id: "tu_1", name: "list_artifacts", input: {} }] : [{ type: "text", text: "done" }], + stop_reason: toolUse ? "tool_use" : "end_turn", stop_sequence: null, + usage: { input_tokens: inputTokens, output_tokens: outputTokens }, + }), { status: 200, headers: { "content-type": "application/json" } }); +} + +describe("the long-context tier belongs to one request, never to a sum of requests", () => { + const HAIKU = "claude-haiku-5-5"; + const SAVED_ANTHROPIC = process.env.ANTHROPIC_API_KEY; + beforeEach(() => { process.env.ANTHROPIC_API_KEY = "test-anthropic-key"; }); + afterEach(() => { if (SAVED_ANTHROPIC === undefined) delete process.env.ANTHROPIC_API_KEY; else process.env.ANTHROPIC_API_KEY = SAVED_ANTHROPIC; }); + + const base = (inTok: number, outTok: number) => (inTok * 0.10 + outTok * 0.50) / 1_000_000; + + it("meter settlement: two 60K/1K dispatches (120K prompt tokens in total) settle at the base tier, one 120K dispatch at the tier", async () => { + fakeTransport(() => anthropicAnswer(60_000, 1_000)); + const meter = createSpendMeter(10); + const m = model(HAIKU, { dispatch: RECEIPT, meter }); + await m.next(TURN); + await m.next(TURN); + expect(meter.dispatches.map((d) => d.costUsd)).toEqual([base(60_000, 1_000), base(60_000, 1_000)]); + expect(meter.spentUsd).toBeCloseTo(0.013, 9); // was $0.065 when the sum was tiered + + fakeTransport(() => anthropicAnswer(120_000, 2_000)); + const long = createSpendMeter(10); + await model(HAIKU, { dispatch: RECEIPT, meter: long }).next(TURN); + expect(long.spentUsd).toBeCloseTo((120_000 * 0.50 + 2_000 * 2.50) / 1_000_000, 9); // 0.065: a real 120K prompt IS tiered + }); + + it("ladder rung cost is the sum of its settled dispatches (two sub-100K requests are not billed as one 120K request)", async () => { + const dir = mkdtempSync(join(tmpdir(), "ladder-tier-")); + const argvBefore = process.argv; + const exitBefore = process.exitCode; + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + try { + const run = async (cap: string) => { + const out = join(dir, `report-${cap}.json`); + process.argv = ["node", "ladder.ts", "--real", HAIKU, "--levels=1-1", `--max-cost-usd=${cap}`, "--json-out", out]; + process.exitCode = undefined; + const { main } = await import("./ladder"); + await main(); + process.exitCode = exitBefore; + try { return JSON.parse(readFileSync(out, "utf8")) as Record; } catch { return undefined; } + }; + fakeTransport(() => anthropicAnswer(1, 1)); + const probe = await run("0.0001"); + const cap = String(Math.ceil(probe!.budget.estimateUsd * 1e4) / 1e4); + // Step 1 asks for a tool, step 2 answers: exactly two dispatches of 60K/1K. + fakeTransport((n) => anthropicAnswer(60_000, 1_000, n === 1)); + const report = await run(cap); + expect(calls).toHaveLength(2); + const row = report!.results[0]; + expect(row.dispatches.map((d: { costUsd: number }) => d.costUsd)).toEqual([base(60_000, 1_000), base(60_000, 1_000)]); + expect(row.cost).toBeCloseTo(0.013, 9); // was $0.065 + expect(row.cost).toBeCloseTo(report!.budget.spentUsd, 9); + } finally { + process.argv = argvBefore; + process.exitCode = exitBefore; + rmSync(dir, { recursive: true, force: true }); + vi.restoreAllMocks(); + } + }); +}); + +describe("pre-flight request ceiling follows the candidate limit the adapter enforces", () => { + const shapes = [{ maxSteps: 3, ctxTokens0: 1_000, growthTokens: 0 }, { maxSteps: 2, ctxTokens0: 1_000, growthTokens: 0 }] as never; + const SAVED_LIMIT = process.env.OPENROUTER_FREE_AUTO_LIMIT; + afterEach(() => { if (SAVED_LIMIT === undefined) delete process.env.OPENROUTER_FREE_AUTO_LIMIT; else process.env.OPENROUTER_FREE_AUTO_LIMIT = SAVED_LIMIT; }); + + it("a free-auto route multiplies by OPENROUTER_FREE_AUTO_LIMIT (default 8, max 20), a concrete route by 1", async () => { + const { ladderRequestCeiling, estimateLadderUsd } = await import("./ladder"); + const steps = 3 + 2; + delete process.env.OPENROUTER_FREE_AUTO_LIMIT; + expect(ladderRequestCeiling([PRICED], shapes)).toBe(steps); + expect(ladderRequestCeiling(["openrouter/free-auto"], shapes)).toBe(8 * steps); // Codex observed 8 requests for one step whose ceiling was 1 + expect(ladderRequestCeiling([PRICED, "openrouter/free-auto"], shapes)).toBe(steps + 8 * steps); + process.env.OPENROUTER_FREE_AUTO_LIMIT = "5"; + expect(ladderRequestCeiling(["openrouter/free-auto"], shapes)).toBe(5 * steps); + process.env.OPENROUTER_FREE_AUTO_LIMIT = "99"; + expect(ladderRequestCeiling(["openrouter/free-auto"], shapes)).toBe(20 * steps); + expect(estimateLadderUsd(["openrouter/free-auto"], shapes)).toBe(0); // free route: any fan-out costs nothing + }); + + it("an alias of a free-auto route fans out like the route it resolves to (dispatch resolves aliases first)", async () => { + const { ladderRequestCeiling, estimateLadderUsd } = await import("./ladder"); + const steps = 3 + 2; + delete process.env.OPENROUTER_FREE_AUTO_LIMIT; + // 'free' and 'openrouter-auto' used to get a ceiling of 1 step each while one model("free").next() issues up to 8 requests. + for (const alias of ["free", "free-auto", "openrouter-auto", "openrouter-free-auto", "FREE", "nodeagent-auto", "nodeagent/free-auto"]) { + expect(ladderRequestCeiling([alias], shapes), alias).toBe(8 * steps); + if (!alias.startsWith("nodeagent")) expect(() => estimateLadderUsd([alias], shapes), alias).not.toThrow(); // nodeagent/free-auto has no price row: main() refuses it at pre-flight + } + expect(ladderRequestCeiling(["openrouter/free"], shapes), "the single-model meta route does not fan out").toBe(steps); + expect(ladderRequestCeiling(["claude-sonnet"], shapes), "a concrete alias does not fan out").toBe(steps); + process.env.OPENROUTER_FREE_AUTO_LIMIT = "3"; + expect(ladderRequestCeiling(["free", PRICED], shapes)).toBe(3 * steps + steps); + }); + + it("an unknown route is refused by the ceiling and the estimate instead of being counted as one request (A.0.10)", async () => { + const { ladderRequestCeiling, estimateLadderUsd } = await import("./ladder"); + expect(() => ladderRequestCeiling(["gemini-9-flash-unknown"], shapes)).toThrow(/unknown model id/); + expect(() => estimateLadderUsd(["gemini-9-flash-unknown"], shapes)).toThrow(/unknown model id/); + }); +}); diff --git a/evals/managedLockPerf.ts b/evals/managedLockPerf.ts index 6fdae549..a199efd0 100644 --- a/evals/managedLockPerf.ts +++ b/evals/managedLockPerf.ts @@ -140,10 +140,10 @@ async function main() { const jsonOut = optionValue("--json-out") ?? DEFAULT_JSON_OUT; const real = Boolean(realRoute); const explicitAgent = real - ? realModel(realRoute!) + ? realModel(realRoute!, { operatorSelected: true }) : scriptedModel(recomputeVariancePlan(TARGETS, { lock: true }), "scripted-explicit-lock"); const managedAgent = real - ? realModel(realRoute!) + ? realModel(realRoute!, { operatorSelected: true }) : scriptedModel(managedBatchVariancePlan(TARGETS), "scripted-managed-lock"); const explicit = await runCase({ lane: "explicit_agent_lock", agent: explicitAgent, real }); diff --git a/evals/professionalLiveCatalog.ts b/evals/professionalLiveCatalog.ts index b35d949e..5ed3b5dc 100644 --- a/evals/professionalLiveCatalog.ts +++ b/evals/professionalLiveCatalog.ts @@ -182,7 +182,7 @@ async function runProfessionalLiveCatalogCase( const started = Date.now(); const prompt = buildProfessionalLiveCatalogPrompt(evalCase); try { - const raw = await judge(modelName, prompt); + const raw = await judge(modelName, prompt, { operatorSelected: true }); const answer = parseProfessionalLiveCatalogAnswer(raw); const checks = gradeProfessionalLiveCatalogAnswer(evalCase, answer); const score = scoreChecks(checks); diff --git a/evals/professionalRuntimeLive.ts b/evals/professionalRuntimeLive.ts index e86dc31f..c8ae050e 100644 --- a/evals/professionalRuntimeLive.ts +++ b/evals/professionalRuntimeLive.ts @@ -141,7 +141,7 @@ async function runProfessionalRuntimeLiveCase( const built = buildRuntimeRoom(scenario); const agent = options.modelName === "scripted" ? scriptedModel(runtimeSmokePlan(scenario, built.noteId), `scripted:${evalCase.id}`) - : realModel(options.modelName); + : realModel(options.modelName, { operatorSelected: true }); const started = Date.now(); try { const result = await runAgent({ diff --git a/package-lock.json b/package-lock.json index 140d6c65..8f5db488 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "0.1.1", "license": "MIT", "dependencies": { - "@ai-sdk/anthropic": "^3.0.84", + "@ai-sdk/anthropic": "^3.0.120", "@ai-sdk/google": "^3.0.82", "@ai-sdk/openai": "^3.0.71", "@assistant-ui/react-o11y": "^0.0.25", @@ -99,13 +99,13 @@ } }, "node_modules/@ai-sdk/anthropic": { - "version": "3.0.117", - "resolved": "https://registry.npmjs.org/@ai-sdk/anthropic/-/anthropic-3.0.117.tgz", - "integrity": "sha512-ZdcpT3tWRifu93DPL5jSGvx9QmRi4zi0H7JiCKp76dH6dytQKYdg12Jpl1HHamCT3KgxSrNMOPKCv8q+gXqAVw==", + "version": "3.0.120", + "resolved": "https://registry.npmjs.org/@ai-sdk/anthropic/-/anthropic-3.0.120.tgz", + "integrity": "sha512-nDdCokLr0nexRUwUpWzzXzK73+KbejgmZMJdo8XthIeeeCXQvtKuhvpNs091N97NEaM3XHmOI55Z8/TWuNSWcQ==", "license": "Apache-2.0", "dependencies": { "@ai-sdk/provider": "3.0.16", - "@ai-sdk/provider-utils": "4.0.51" + "@ai-sdk/provider-utils": "4.0.52" }, "engines": { "node": ">=18" @@ -114,6 +114,33 @@ "zod": "^3.25.76 || ^4.1.8" } }, + "node_modules/@ai-sdk/anthropic/node_modules/@ai-sdk/provider-utils": { + "version": "4.0.52", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-4.0.52.tgz", + "integrity": "sha512-cUimyIz1jjwYoF9n6HO7sU+UBW7Nu3T4lW7562yRvEhM9/8JqlW4cecxSpkiafjRzAphZG2NfJ3/Ch5cHoCIhg==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.16", + "@standard-schema/spec": "^1.1.0", + "eventsource-parser": "^3.0.8", + "undici": "^6.28.0" + }, + "engines": { + "node": ">=18.17" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/anthropic/node_modules/undici": { + "version": "6.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.29.0.tgz", + "integrity": "sha512-R+RODBqp6i2pPflGdq+xIOUkl+RNfGgHwoinecKu/JCuf2uO06cOKoDbI2P7Dn6KcswdKwrczbU6IYJ6K8X+wg==", + "license": "MIT", + "engines": { + "node": ">=18.17" + } + }, "node_modules/@ai-sdk/gateway": { "version": "3.0.191", "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-3.0.191.tgz", diff --git a/package.json b/package.json index a0b382fd..4b866fac 100644 --- a/package.json +++ b/package.json @@ -250,7 +250,7 @@ "eval:credit": "tsx evals/creditEval.ts", "ladder": "tsx evals/ladder.ts", "ladder:real": "tsx evals/ladder.ts --real", - "ladder:free": "tsx evals/ladder.ts --real openrouter/free-auto --free-auto-top=5 --levels=1-4 --rung-timeout-ms=540000 --reserve-ms=30000 --json-out docs/eval/free-auto-router-ladder.json", + "ladder:free": "tsx evals/ladder.ts --real openrouter/free-auto --free-auto-top=5 --max-cost-usd=1 --levels=1-4 --rung-timeout-ms=540000 --reserve-ms=30000 --json-out docs/eval/free-auto-router-ladder.json", "eval:diff": "tsx evals/evalDiff.ts", "proofs:staleness": "tsx scripts/proof-staleness-check.ts", "proofs:publish": "tsx scripts/proof-release-publish.ts --write-readme", @@ -275,7 +275,7 @@ "voice:prod-deploy:proof": "tsx scripts/voice-prod-deploy-proof.ts" }, "dependencies": { - "@ai-sdk/anthropic": "^3.0.84", + "@ai-sdk/anthropic": "^3.0.120", "@ai-sdk/google": "^3.0.82", "@ai-sdk/openai": "^3.0.71", "@assistant-ui/react-o11y": "^0.0.25", diff --git a/packages/walkthrough-review-cli/README.md b/packages/walkthrough-review-cli/README.md index b5b03896..a5654cf6 100644 --- a/packages/walkthrough-review-cli/README.md +++ b/packages/walkthrough-review-cli/README.md @@ -22,7 +22,7 @@ Generic shape: walkthrough-review run \ --config walkthrough-review.config.json \ --base http://127.0.0.1:5178 \ - --model gemini-3.5-flash \ + --model gemini-3.6-flash \ --ui-review \ startup-diligence-war-room ``` diff --git a/packages/walkthrough-review-cli/src/core.ts b/packages/walkthrough-review-cli/src/core.ts index 4f70b55f..16e40ad0 100644 --- a/packages/walkthrough-review-cli/src/core.ts +++ b/packages/walkthrough-review-cli/src/core.ts @@ -75,7 +75,7 @@ type CommandVariables = { }; const DEFAULT_CONFIG = "walkthrough-review.config.json"; -const DEFAULT_MODEL = "gemini-3.5-flash"; +const DEFAULT_MODEL = "gemini-3.6-flash"; const DEFAULT_OUTPUT_ROOT = join("docs", "eval", "walkthrough-review"); const DEFAULT_DRY_OUTPUT_ROOT = join(".tmp-qa", "walkthrough-review"); const DEFAULT_MEDIA_ROOT = join("docs", "walkthroughs"); diff --git a/scripts/benchmark/pricingAtRun.ts b/scripts/benchmark/pricingAtRun.ts new file mode 100644 index 00000000..b7e6fe74 --- /dev/null +++ b/scripts/benchmark/pricingAtRun.ts @@ -0,0 +1,69 @@ +/** + * Pricing the benchmark records on every row (`pricingAtRun`). The OpenRouter /models snapshot of the run wins + * (dated by `fetchedAt` + `routeSnapshotId`); the catalog row is the fallback; a flagged $1/$5 `fallback_estimate` + * is the last resort. Split out of run.ts, which runs the benchmark on import, so the pricing agreement test + * (SPEC A.0.6) can load these paths. + */ +import { getModelPricing, lookupModelAlias, type ModelPricing } from "../../src/nodeagent/models/modelCatalog"; +import type { OpenRouterModelInfo } from "../../src/nodeagent/models/openRouterFreeModels"; + +export type PricingAtRun = { + source: "openrouter_snapshot" | "catalog" | "fallback_estimate"; + inputPer1M: number; + outputPer1M: number; + cachedInputPer1M?: number; + contextWindow?: number; + routeSnapshotId?: string; +}; + +export type RouteSnapshot = { + routeSnapshotId: string; + fetchedAt: string; + source: "openrouter" | "unavailable"; + modelsById: Map; + error?: string; +}; + +export function priceFromOpenRouterInfo(info: OpenRouterModelInfo, routeSnapshotId: string): PricingAtRun | undefined { + const prompt = Number(info.pricing?.prompt); + const completion = Number(info.pricing?.completion); + if (!Number.isFinite(prompt) || !Number.isFinite(completion)) return undefined; + const cacheRead = Number(info.pricing?.input_cache_read); + return { + source: "openrouter_snapshot", + inputPer1M: prompt * 1_000_000, + outputPer1M: completion * 1_000_000, + ...(Number.isFinite(cacheRead) ? { cachedInputPer1M: cacheRead * 1_000_000 } : {}), + contextWindow: info.context_length ?? info.top_provider?.context_length, + routeSnapshotId, + }; +} + +export function catalogPricingAtRun(modelId: string): PricingAtRun { + const pricing: ModelPricing | null = getModelPricing(lookupModelAlias(modelId)); + if (!pricing) { + return { + source: "fallback_estimate", + inputPer1M: 1, + outputPer1M: 5, + }; + } + return { + source: "catalog", + inputPer1M: pricing.inputPer1M, + outputPer1M: pricing.outputPer1M, + ...(pricing.cachedInputPer1M != null ? { cachedInputPer1M: pricing.cachedInputPer1M } : {}), + contextWindow: pricing.contextWindow, + }; +} + +export function pricingAtRunFor(modelId: string, snapshot?: RouteSnapshot): PricingAtRun { + const resolved = lookupModelAlias(modelId); + const fromSnapshot = snapshot?.modelsById.get(resolved); + if (snapshot && fromSnapshot) return priceFromOpenRouterInfo(fromSnapshot, snapshot.routeSnapshotId) ?? catalogPricingAtRun(resolved); + return catalogPricingAtRun(resolved); +} + +export function costFromPricing(pricing: PricingAtRun, inputTokens: number, outputTokens: number): number { + return (inputTokens * pricing.inputPer1M + outputTokens * pricing.outputPer1M) / 1_000_000; +} diff --git a/scripts/benchmark/probe.ts b/scripts/benchmark/probe.ts index 8bd2755a..0ea97084 100644 --- a/scripts/benchmark/probe.ts +++ b/scripts/benchmark/probe.ts @@ -6,33 +6,39 @@ const { generateText } = await import("ai"); const { anthropic } = await import("@ai-sdk/anthropic"); const { openai, createOpenAI } = await import("@ai-sdk/openai"); const { google } = await import("@ai-sdk/google"); +const { claudeEffortProviderOptions } = await import("../../src/nodeagent/models/modelCatalog"); const or = createOpenAI({ apiKey: process.env.OPENROUTER_API_KEY, baseURL: "https://openrouter.ai/api/v1" }); const cases: [string, any][] = [ - // Anthropic latest + // Anthropic latest (5.5 = the migration-guide successors; claude-opus-4-8 is still Active) + ["claude-haiku-5-5", anthropic("claude-haiku-5-5")], + ["claude-sonnet-5-5", anthropic("claude-sonnet-5-5")], + ["claude-opus-5-5", anthropic("claude-opus-5-5")], ["claude-opus-4-8", anthropic("claude-opus-4-8")], ["claude-sonnet-4-6", anthropic("claude-sonnet-4-6")], ["claude-haiku-4-5", anthropic("claude-haiku-4-5")], - // Google latest (gemini 3.x) + // Google latest (gemini 3.x). gemini-3.5-flash stays on purpose: Google auto-routes it to 3.6 since 2026-10-08. + ["gemini-3.8-flash", google("gemini-3.8-flash")], + ["gemini-3.5-flash-lite", google("gemini-3.5-flash-lite")], ["gemini-3.5-flash", google("gemini-3.5-flash")], ["gemini-3-flash-preview", google("gemini-3-flash-preview")], ["gemini-3.1-flash-lite", google("gemini-3.1-flash-lite")], ["gemini-2.5-flash", google("gemini-2.5-flash")], ["gemini-2.5-flash-lite", google("gemini-2.5-flash-lite")], - // OpenAI latest (gpt-5.x) + // OpenAI latest (gpt-5.x / gpt-6.x) + ["gpt-6.1-sol", openai("gpt-6.1-sol")], ["gpt-5.5", openai("gpt-5.5")], ["gpt-5.4", openai("gpt-5.4")], ["gpt-5.4-mini", openai("gpt-5.4-mini")], ["gpt-5.4-nano", openai("gpt-5.4-nano")], // OpenRouter (cheap/free) — simple-call reachability - ["OR nex-agi/nex-n2-pro:free", or.chat("nex-agi/nex-n2-pro:free")], ["OR ibm-granite/granite-4.1-8b", or.chat("ibm-granite/granite-4.1-8b")], ["OR deepseek/deepseek-v4-flash", or.chat("deepseek/deepseek-v4-flash")], ["OR z-ai/glm-4.7-flash", or.chat("z-ai/glm-4.7-flash")], ]; for (const [name, model] of cases) { try { - const r = await generateText({ model, prompt: "Reply with the word OK." }); + const r = await generateText({ model, prompt: "Reply with the word OK.", ...claudeEffortProviderOptions(name) }); // explicit SPEC §1.3 effort on claude-*-5-5 console.log(`✅ ${name.padEnd(26)} -> "${(r.text || "").slice(0, 16).trim()}"`); } catch (e: any) { const status = e?.statusCode ?? e?.status ?? ""; diff --git a/scripts/benchmark/run.ts b/scripts/benchmark/run.ts index cdb58093..b25807f2 100644 --- a/scripts/benchmark/run.ts +++ b/scripts/benchmark/run.ts @@ -20,7 +20,8 @@ import { buildDemoRoom, RESEARCH_COMPANIES } from "../../src/engine/demoRoom"; import { InMemoryRoomTools } from "../../src/nodeagent/skills/integration/noderoomAdapter"; import { runAgent } from "../../src/nodeagent/core/runtime"; import { model, priceRun, judge } from "../../src/nodeagent/models/adapter"; -import { getModelPricing, resolveModelAlias, type ModelPricing } from "../../src/nodeagent/models/modelCatalog"; +import { lookupModelAlias } from "../../src/nodeagent/models/modelCatalog"; +import { costFromPricing, pricingAtRunFor, type PricingAtRun, type RouteSnapshot } from "./pricingAtRun"; import { fetchSourceReal } from "../../src/nodeagent/skills/search/fetchSource"; import type { AgentMessage, AgentModel, AgentTool, RoomTools, SourceResult } from "../../src/nodeagent/core/types"; import { isOpenRouterFreeAutoModel, selectOpenRouterFreeModels, type OpenRouterModelInfo } from "../../src/nodeagent/models/openRouterFreeModels"; @@ -62,7 +63,7 @@ const CHECKS = [ // Fixed cheap judge (isolated yes/no per the grounded-eval rule — low variance, not a holistic grade). const JUDGE = process.env.JUDGE_MODEL || "gemini-3.1-flash-lite"; async function scoreJudgeCompany(company: string, summary: string, evidence: string): Promise { - return judgeCompanyWith(judge, JUDGE, company, summary, evidence); + return judgeCompanyWith((id, prompt) => judge(id, prompt, { operatorSelected: true }), JUDGE, company, summary, evidence); } // Default research smoke set. The full supported route matrix lives in @@ -95,23 +96,6 @@ interface RunModelOptions { routeSnapshot?: RouteSnapshot; } -type PricingAtRun = { - source: "openrouter_snapshot" | "catalog" | "fallback_estimate"; - inputPer1M: number; - outputPer1M: number; - cachedInputPer1M?: number; - contextWindow?: number; - routeSnapshotId?: string; -}; - -type RouteSnapshot = { - routeSnapshotId: string; - fetchedAt: string; - source: "openrouter" | "unavailable"; - modelsById: Map; - error?: string; -}; - function unique(items: string[]): string[] { return [...new Set(items.filter(Boolean))]; } @@ -192,52 +176,8 @@ async function loadRouteSnapshot(): Promise { } } -function priceFromOpenRouterInfo(info: OpenRouterModelInfo, routeSnapshotId: string): PricingAtRun | undefined { - const prompt = Number(info.pricing?.prompt); - const completion = Number(info.pricing?.completion); - if (!Number.isFinite(prompt) || !Number.isFinite(completion)) return undefined; - const cacheRead = Number(info.pricing?.input_cache_read); - return { - source: "openrouter_snapshot", - inputPer1M: prompt * 1_000_000, - outputPer1M: completion * 1_000_000, - ...(Number.isFinite(cacheRead) ? { cachedInputPer1M: cacheRead * 1_000_000 } : {}), - contextWindow: info.context_length ?? info.top_provider?.context_length, - routeSnapshotId, - }; -} - -function catalogPricingAtRun(modelId: string): PricingAtRun { - const pricing: ModelPricing | null = getModelPricing(resolveModelAlias(modelId)); - if (!pricing) { - return { - source: "fallback_estimate", - inputPer1M: 1, - outputPer1M: 5, - }; - } - return { - source: "catalog", - inputPer1M: pricing.inputPer1M, - outputPer1M: pricing.outputPer1M, - ...(pricing.cachedInputPer1M != null ? { cachedInputPer1M: pricing.cachedInputPer1M } : {}), - contextWindow: pricing.contextWindow, - }; -} - -function pricingAtRunFor(modelId: string, snapshot?: RouteSnapshot): PricingAtRun { - const resolved = resolveModelAlias(modelId); - const fromSnapshot = snapshot?.modelsById.get(resolved); - if (fromSnapshot) return priceFromOpenRouterInfo(fromSnapshot, snapshot.routeSnapshotId) ?? catalogPricingAtRun(resolved); - return catalogPricingAtRun(resolved); -} - -function costFromPricing(pricing: PricingAtRun, inputTokens: number, outputTokens: number): number { - return (inputTokens * pricing.inputPer1M + outputTokens * pricing.outputPer1M) / 1_000_000; -} - function isOpenRouterRoute(modelId: string): boolean { - const resolved = resolveModelAlias(modelId); + const resolved = lookupModelAlias(modelId); return isOpenRouterFreeAutoModel(resolved) || resolved.includes("/"); } @@ -273,7 +213,7 @@ async function routeSmoke(modelId: string): Promise<{ ok: true; resolvedModels: const timer = setTimeout(() => ctl.abort(), timeoutMs); const resolvedModels: string[] = []; try { - const schemaRoute = model(modelId); + const schemaRoute = model(modelId, { operatorSelected: true }); const schemaStep = await schemaRoute.next({ system: "You are a route-contract smoke test. Return only JSON, no prose.", messages: [{ @@ -296,7 +236,7 @@ async function routeSmoke(modelId: string): Promise<{ ok: true; resolvedModels: }; } - const toolRoute = model(modelId); + const toolRoute = model(modelId, { operatorSelected: true }); const smokeTool: AgentTool = { name: "report_answer", description: "Report a short route-smoke answer.", @@ -421,7 +361,7 @@ function runModelInChild(modelId: string, options: RunModelOptions, snapshot?: R } function recordingModel(modelId: string): { agentModel: AgentModel; resolvedModels: string[] } { - const baseModel = model(modelId); + const baseModel = model(modelId, { operatorSelected: true }); const resolvedModels: string[] = []; return { resolvedModels, diff --git a/scripts/design-qa/rubric-panel.ts b/scripts/design-qa/rubric-panel.ts index d4921683..ad359785 100644 --- a/scripts/design-qa/rubric-panel.ts +++ b/scripts/design-qa/rubric-panel.ts @@ -19,7 +19,7 @@ import "../benchmark/loadEnv"; import { mkdirSync, readFileSync, writeFileSync, existsSync } from "node:fs"; import { dirname, join } from "node:path"; -import { generateObject, type ModelMessage } from "ai"; +import { generateObject, type UserContent } from "ai"; import { google } from "@ai-sdk/google"; import { z } from "zod"; @@ -28,7 +28,7 @@ const opt = (n: string) => args.find((a) => a.startsWith(`--${n}=`))?.slice(n.le const mediaPath = opt("media"); const refPath = opt("reference"); const floorPath = opt("floor"); -const model = process.env.GEMINI_UI_REVIEW_MODEL ?? "gemini-3.5-flash"; +const model = process.env.GEMINI_UI_REVIEW_MODEL ?? "gemini-3.6-flash"; const outPath = opt("out") ?? join(process.cwd(), ".tmp-qa", "design-rubric.json"); if (!mediaPath) throw new Error("--media= required"); if (!process.env.GOOGLE_GENERATIVE_AI_API_KEY) throw new Error("GOOGLE_GENERATIVE_AI_API_KEY required"); @@ -59,7 +59,7 @@ const lensSchema = (dims: string[]) => z.object({ }); async function judgeLens(lens: typeof LENSES[number]) { - const content: ModelMessage["content"] = [ + const content: UserContent = [ { type: "text", text: [ `You are the ${lens.key} critic in a design-QA jury for NodeRoom (a calm multiplayer finance-diligence room).`, `Judge ONLY your lens: ${lens.brief}`, @@ -72,7 +72,7 @@ async function judgeLens(lens: typeof LENSES[number]) { { type: "file", data: candidate, filename: "candidate.png", mediaType: "image/png" }, ]; if (reference) content.push({ type: "file", data: reference, filename: "reference.png", mediaType: "image/png" }); - const { object } = await generateObject({ model: google(model), schema: lensSchema(lens.dims), messages: [{ role: "user", content }], temperature: 0.2 }); + const { object } = await generateObject({ model: google(model), schema: lensSchema(lens.dims), messages: [{ role: "user", content }] }); return { lens: lens.key, ...object }; } diff --git a/scripts/eval/painJudge.ts b/scripts/eval/painJudge.ts index db3283f9..e7f2c04f 100644 --- a/scripts/eval/painJudge.ts +++ b/scripts/eval/painJudge.ts @@ -5,7 +5,7 @@ * point is addressed. This is the independent "does the live workflow truly relieve the * pain" check, before the Remotion demo. * - * npx tsx scripts/eval/painJudge.ts # gemini-3.5-flash + * npx tsx scripts/eval/painJudge.ts # gemini-3.6-flash * JUDGE_MODEL=gemini-3.1-flash-lite npx tsx scripts/eval/painJudge.ts */ import "../benchmark/loadEnv"; @@ -15,7 +15,7 @@ import { generateText } from "ai"; import { createGoogleGenerativeAI } from "@ai-sdk/google"; const ROOT = process.cwd(); -const MODEL = process.env.JUDGE_MODEL || "gemini-3.5-flash"; +const MODEL = process.env.JUDGE_MODEL || "gemini-3.6-flash"; const google = createGoogleGenerativeAI({ apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY }); const FRAMES: [string, string][] = [ diff --git a/scripts/eval/providerParseSmoke.ts b/scripts/eval/providerParseSmoke.ts index faac869d..bde76df0 100644 --- a/scripts/eval/providerParseSmoke.ts +++ b/scripts/eval/providerParseSmoke.ts @@ -5,7 +5,7 @@ * Verifies the architecture invariant: canonical Convex storageId != disposable provider file id. * * npx tsx scripts/eval/providerParseSmoke.ts # gemini default - * npx tsx scripts/eval/providerParseSmoke.ts gemini-3.5-flash,gpt-5.4-mini,claude-haiku-4-5 + * npx tsx scripts/eval/providerParseSmoke.ts gemini-3.8-flash,gpt-5.4-mini,claude-haiku-5-5 */ import "../benchmark/loadEnv"; import { z } from "zod"; @@ -13,12 +13,12 @@ import { generateObject } from "ai"; import { createGoogleGenerativeAI } from "@ai-sdk/google"; import { createOpenAI } from "@ai-sdk/openai"; import { createAnthropic } from "@ai-sdk/anthropic"; -import { getProviderForModel } from "../../src/nodeagent/models/modelCatalog"; -import { priceRun } from "../../src/nodeagent/models/adapter"; +import { claudeEffortProviderOptions, getProviderForModel } from "../../src/nodeagent/models/modelCatalog"; +import { priceRequest } from "../../src/nodeagent/models/adapter"; import { artifactsFromProviderExtraction, providerFileCacheMeta, type CanonicalFileRef, type ProviderExtraction, type ProviderUploadResult } from "../../src/app/providerParserAdapter"; import type { CellPayload, ProviderParser } from "../../src/engine/types"; -const MODELS = (process.argv[2] || "gemini-3.5-flash,gpt-5.4-mini,claude-haiku-4-5").split(",").map((s) => s.trim()); +const MODELS = (process.argv[2] || "gemini-3.8-flash,gpt-5.4-mini,claude-haiku-5-5").split(",").map((s) => s.trim()); // Synthetic messy diligence text — the ParselyFi/PitchBook archetype (unstructured → structured), no private data. const SAMPLE = `Diligence scratch notes (messy): @@ -41,7 +41,7 @@ const SCHEMA = z.object({ function providerModel(modelId: string) { const p = getProviderForModel(modelId); - if (p === "google") return createGoogleGenerativeAI({ apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY })(modelId); + if (p === "gemini") return createGoogleGenerativeAI({ apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY })(modelId); if (p === "anthropic") return createAnthropic({ apiKey: process.env.ANTHROPIC_API_KEY })(modelId); if (p === "openrouter") return createOpenAI({ apiKey: process.env.OPENROUTER_API_KEY, baseURL: "https://openrouter.ai/api/v1" }).chat(modelId); return createOpenAI({ apiKey: process.env.OPENAI_API_KEY }).chat(modelId); // openai @@ -50,7 +50,7 @@ function providerModel(modelId: string) { async function smoke(modelId: string) { const provider = getProviderForModel(modelId) as ProviderParser; const t0 = Date.now(); - const res = await generateObject({ model: providerModel(modelId), schema: SCHEMA, prompt: PROMPT }); + const res = await generateObject({ model: providerModel(modelId), schema: SCHEMA, prompt: PROMPT, ...claudeEffortProviderOptions(modelId) }); const ms = Date.now() - t0; const extraction = res.object as ProviderExtraction; @@ -60,7 +60,7 @@ async function smoke(modelId: string) { const providerFile = providerFileCacheMeta(file, upload); const artifacts = artifactsFromProviderExtraction({ file, providerFile, provider, model: modelId, extraction }); - const cost = priceRun(modelId, res.usage?.inputTokens ?? 0, res.usage?.outputTokens ?? 0); + const cost = priceRequest(modelId, res.usage?.inputTokens ?? 0, res.usage?.outputTokens ?? 0); const art = artifacts[0]; const tbl = extraction.tables?.[0]; console.log(`\n=== ${modelId} (${provider}) · ${(ms / 1000).toFixed(1)}s · $${cost.toFixed(4)} ===`); diff --git a/scripts/gemini-demo-media-judge.ts b/scripts/gemini-demo-media-judge.ts index e1c209fc..74052e76 100644 --- a/scripts/gemini-demo-media-judge.ts +++ b/scripts/gemini-demo-media-judge.ts @@ -18,7 +18,7 @@ import { z } from "zod"; const ROOT = process.cwd(); const args = process.argv.slice(2); const invokedCommand = ["npm", "run", "media:gemini-judge", "--", ...args].map(quoteCommandArg).join(" "); -const model = optionValue("--model") ?? process.env.GEMINI_MEDIA_JUDGE_MODEL ?? "gemini-3.5-flash"; +const model = optionValue("--model") ?? process.env.GEMINI_MEDIA_JUDGE_MODEL ?? "gemini-3.6-flash"; const runId = optionValue("--run-id") ?? timestampId(new Date()); const only = optionValue("--only")?.toLowerCase(); const inputPaths = optionValues("--input"); @@ -238,7 +238,6 @@ async function judgeAsset(asset: Asset): Promise { model: google(model), schema: judgeSchema, messages, - temperature: 0.2, }); const judge = normalizeJudge(result.object); return { diff --git a/scripts/gemini-mp4-workflow-review.ts b/scripts/gemini-mp4-workflow-review.ts index ed6c8a44..56928617 100644 --- a/scripts/gemini-mp4-workflow-review.ts +++ b/scripts/gemini-mp4-workflow-review.ts @@ -12,7 +12,7 @@ import { z } from "zod"; const args = process.argv.slice(2); const mediaPath = optionValue("--media"); const outPath = optionValue("--out") ?? join(process.cwd(), "docs", "eval", "agent-improvement-loop", "mp4-workflow-review.json"); -const model = optionValue("--model") ?? process.env.GEMINI_UI_REVIEW_MODEL ?? "gemini-3.5-flash"; +const model = optionValue("--model") ?? process.env.GEMINI_UI_REVIEW_MODEL ?? "gemini-3.6-flash"; if (!mediaPath) throw new Error("--media=