diff --git a/CHANGELOG.md b/CHANGELOG.md index bc97026..4f42b0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,68 @@ # NodeVoice changelog +## 2026-10-14 — Remove the retired gpt-4.1-nano router row + +OpenAI retires `gpt-4.1-nano` on 2026-10-23. The room model picker no longer +offers it, rooms that already stored it are rewritten to `gpt-5.4-mini` by +`rooms:migrateRetiredRoomModels` (audit rows in `roomModelMigrations`), and every +request the coordinator builds from `room.model` (the turn, the human-steer +interpreter, the planning-worker fallback) resolves it through `validModel`. +New planning workers also store `validModel(room.model)` at spawn, so a host +steer in a room that has not been migrated yet does not create a worker that +sends the retired id. `watchRoom` serializes `validModel(room.model)`, so until +the migration runs the picker and the cost panels show the model the coordinator +actually sends. `gpt-5-nano` / `gpt-5-mini` rows are marked "retires +2026-12-11" (in the dropdown options, not only the hover title) and are removed +in a follow-up PR through the same migration. The `gpt-5.4-mini` literals in +`src/live/pipeline.ts` and `src/providers/openai.ts` now import `DEFAULT_MODEL`. `scripts/model-eval.mjs` carries +`PRICING_AS_OF` (the date its per-M prices were typed, not re-verified). Spec: +model-defaults SPEC Part A section A.2 (NodeVoice). + +**Planning workers.** Worker rows created BEFORE this change store `gpt-4.1-nano` +too, and `retryV3Worker` copies the stored model forward. `runV3Worker` resolves an +`execution_plan` row through `validModel(worker.model ?? room.model)`, so a retry +before the migration still requests `gpt-5.4-mini`; the stored rows would +nonetheless keep the retired id, so `rooms:migrateRetiredRoomModels` also +pages `workers` (`{"table":"workers"}`): `execution_plan` rows whose model is not +in `ROUTER_MODELS` become `gpt-5.4-mini` and get an audit row (`workerId` set). +`web_research` rows are never touched, so the `OPENAI_WEB_MODEL` caller argument +is preserved. Pages are 6 rows (`MIGRATION_PAGE_SIZE`): a room's `goal` has no length +cap, so a page of 1 MiB documents read twice (page, then `patch`) must stay under +Convex's 16 MiB per-mutation read limit and 16 MiB write limit (the test also holds writes under a stricter 8 MiB margin). After the deploy run +`node scripts/migrate-retired-models.mjs --prod`, which walks every page of both tables +(a bare `npx convex run` targets the DEV deployment, so the script refuses to spawn unless +the arguments name a deployment: `--prod`, `--deployment-name`, `--preview-name`, or `--url` +with `--admin-key`; it also refuses while `CONVEX_DEPLOY_KEY` / `CONVEX_DEPLOYMENT_TOKEN` is set in the environment, +`.env.local` or `.env` (`KEY=v` and dotenv's `KEY: v`) and refuses `--env-file` (it can carry a key), because the Convex CLI then ignores `--prod` and migrates the key's deployment; whatever that preflight misses, the CLI is spawned with both variables set to `""`, which dotenv never overrides and the CLI reads as unset, so a file key cannot take effect; it also fails on the CLI's "Ignoring `--prod`" warning); run it again and expect `migrated: 0` for both tables. BEFORE the removing +deploy, record the running production rooms with `node scripts/migrate-retired-models.mjs --live-rooms --prod` +(SPEC A.2.5: live rooms are checked before removal). It runs `convex data rooms --format jsonl --limit 100000` through the same +guards as the migration (`convex data` shares the CLI's deployment selection, so a visible deploy key would otherwise make it list another +deployment's rooms) and prints the count and codes of the `"running":true` rooms. `convex data` lists only the newest `--limit` +rooms, rooms are never deleted, and `jsonl` prints no truncation warning, so the script refuses a result that fills the limit. + +**Pricing sources (SPEC A.0.6).** The voice/Realtime unit prices at +`README.md:1206-1209` (`gpt-realtime`, `gpt-4o-mini-tts`, `whisper-1`, +`gpt-4o-mini-transcribe`) are as of 2026-07-04: typed in commit `40cd6b6`, the same +day as the router prices, and not re-read from the vendor since. They are not router +ids and the repo has no second source, so no agreement test covers them. + +**Pricing agreement (SPEC A.0.6).** `ROUTER_PRICING_AS_OF` (2026-07-04) dates the +`expectedCostUsd` figures; a test recomputes every recorded call's cost from the +eval script's per-M table and checks it against `docs/model-eval-results.json` and +`ROUTER_MODELS`. A second test reads the README `$/turn` table and the +`scripts/generate-chart.mjs` DATA rows (both frozen, read-only) and checks each +against `byModel.costPerTurn` at the 2 significant figures they print. + +**Validation:** `tests/roomModelMigration.test.ts` (24 tests) fails on the +previous code (request carried `gpt-4.1-nano`) and passes now, with one +request-shape test per changed coordinator role (turn, steer interpreter, +planning worker spawned through the real steer path). `completeV3Worker` no +longer writes a model outside `ROUTER_MODELS` onto an `execution_plan` row, so an +in-flight planning call cannot undo the workers migration by echoing the retired +id back. `npm run check`, +`check:client`, `check:convex`, `check:citations` and `npm test` pass. No +provider calls made. + ## 2026-10-07 — Make setup and proof limits reachable Developers and coding agents can reach the existing `HANDOFF.md` from the README diff --git a/convex/coordinator.ts b/convex/coordinator.ts index 48d1bc0..b6bf392 100644 --- a/convex/coordinator.ts +++ b/convex/coordinator.ts @@ -2,7 +2,7 @@ import { v } from "convex/values"; import { internalAction, action } from "./_generated/server"; import { internal, api } from "./_generated/api"; -import { agentForSlot, nextSlot, estimateSpeechMs, deriveHumanSteeringIntentFallback, validProfile, coerceCountTurn, type CountTask, type Slot } from "./shared"; +import { agentForSlot, nextSlot, estimateSpeechMs, deriveHumanSteeringIntentFallback, validProfile, validModel, coerceCountTurn, type CountTask, type Slot } from "./shared"; import { generateAgentTurn, interpretHumanSteer as interpretHumanSteerWithModel, @@ -18,7 +18,7 @@ import type { Id } from "./_generated/dataModel"; * The commit re-validates floor/token server-side (the LLM+TTS work here is * seconds long and non-transactional) and reports whether it landed. */ -async function produceTurn( +export async function produceTurn( ctx: any, roomId: Id<"rooms">, room: any, @@ -38,7 +38,7 @@ async function produceTurn( ? coerceCountTurn({ text: String(countTask.next), speechAct: "task_action", done: countTask.next >= countTask.target }, countTask) : await generateAgentTurn({ goal: room.goal, - model: room.model, + model: validModel(room.model), // stale stored ids (retired models) fall back to DEFAULT_MODEL profile: validProfile(room.profile), persona: agent.persona, selfName: agent.name, @@ -124,7 +124,7 @@ export const interpretHumanSteer = internalAction({ intent = await interpretHumanSteerWithModel({ text: args.text, currentGoal: room.goal, - model: room.model, + model: validModel(room.model), profile: validProfile(room.profile), transcript, }); @@ -191,7 +191,7 @@ export const runV3Worker = internalAction({ const output = worker.kind === "web_research" ? await runWebResearchWorker({ goal: goal.title, task: taskTitle, roomGoal: room.goal, model: worker.model }) - : await runPlanningWorker({ goal: goal.title, task: taskTitle, roomGoal: room.goal, model: worker.model ?? room.model }); + : await runPlanningWorker({ goal: goal.title, task: taskTitle, roomGoal: room.goal, model: validModel(worker.model ?? room.model) }); // execution_plan models only ever come from validModel(room.model) (rooms.ts spawn/retry), so a stale stored id falls back to DEFAULT_MODEL here too; web_research above is untouched await ctx.runMutation(internal.rooms.completeV3Worker, { workerId: args.workerId, diff --git a/convex/rooms.ts b/convex/rooms.ts index 4217802..c74c8f8 100644 --- a/convex/rooms.ts +++ b/convex/rooms.ts @@ -7,6 +7,7 @@ import { activeSlots, CAPABILITY_PROFILES, ROUTER_MODELS, + DEFAULT_MODEL, DEFAULT_GOAL, agentIndexFromSlot, isAgentSlot, @@ -176,7 +177,7 @@ async function spawnAgentOsGoal( ? [{ kind: "deterministic_count", title: "Prepare exact count sequence", model: undefined as string | undefined }] : [ { kind: "web_research", title: "Research current external context", model: "gpt-4.1-mini" }, - { kind: "execution_plan", title: "Draft execution plan", model: room.model }, + { kind: "execution_plan", title: "Draft execution plan", model: validModel(room.model) }, ]; for (const spec of workerSpecs) { const budgetBlocked = workersUsed >= policy.budgetMaxWorkers; @@ -307,7 +308,7 @@ async function serializeRoom(ctx: QueryCtx, roomId: Id<"rooms">) { }), state: { goal: room.goal, - model: room.model, + model: validModel(room.model), // an unmigrated room still stores a retired id; show what the coordinator actually sends agentCount: roomAgentCount(room), profile: roomProfile(room), floorOwner: room.floorOwner, @@ -417,6 +418,64 @@ export const getV3WorkerRaw = internalQuery({ }, }); +/** + * Page size. Convex caps a mutation at 16 MiB read and 16 MiB written, and a document at 1 MiB + * (docs.convex.dev/production/state/limits, read 2026-10-16); + * `goal` has no length cap, so a room can be ~1 MiB. Each stale document is read twice (the page, + * then `patch`) and written once: 6 x 1 MiB x 2 = 12 MiB read, 6 MiB written, with room left for + * the audit inserts. Do not raise this without re-deriving that bound. (maximumBytesRead is not + * used: a byte-limited page can come back incomplete with pageStatus SplitRequired, and this + * migration must never skip rows.) + */ +export const MIGRATION_PAGE_SIZE = 6; + +/** + * Rewrite stored model ids that are no longer in ROUTER_MODELS (a retired id such as + * gpt-4.1-nano, OpenAI 2026-10-23) to DEFAULT_MODEL and record {roomId, [workerId], from, to, at}. + * `validModel` only runs on writes, so without this a stale value reaches OpenAI as-is. + * + * Two tables hold such a value: `rooms.model`, and `workers.model` on `execution_plan` rows + * (spawnAgentOsGoal stores the room's model there and retryV3Worker copies it forward, so a + * retry of a pre-change planning worker would otherwise resend the retired id). `web_research` + * workers are left alone: their model is the OPENAI_WEB_MODEL caller argument, not a router row. + * + * Paged, and run against the PRODUCTION deployment once AFTER the deploy that removes the row: + * `node scripts/migrate-retired-models.mjs --prod` walks every page of both tables (a bare + * `npx convex run` targets the dev deployment). Run it a second time: it must report + * `migrated: 0` for both tables. Until the rooms pass finishes, the coordinator read-side + * guard (validModel) covers unmigrated rooms. + */ +export async function retireRoomModels(ctx: MutationCtx, cursor: string | null) { + const page = await ctx.db.query("rooms").paginate({ numItems: MIGRATION_PAGE_SIZE, cursor }); + let migrated = 0; + for (const room of page.page) { + if (ROUTER_MODELS.some((m) => m.id === room.model)) continue; + const from = room.model; + await ctx.db.patch(room._id, { model: DEFAULT_MODEL }); + await ctx.db.insert("roomModelMigrations", { roomId: room._id, from, to: DEFAULT_MODEL, at: Date.now() }); + migrated += 1; + } + return { migrated, isDone: page.isDone, continueCursor: page.continueCursor }; +} + +export async function retireWorkerModels(ctx: MutationCtx, cursor: string | null) { + const page = await ctx.db.query("workers").paginate({ numItems: MIGRATION_PAGE_SIZE, cursor }); + let migrated = 0; + for (const worker of page.page) { + const from = worker.model; + if (worker.kind !== "execution_plan" || !from || ROUTER_MODELS.some((m) => m.id === from)) continue; + await ctx.db.patch(worker._id, { model: DEFAULT_MODEL }); + await ctx.db.insert("roomModelMigrations", { roomId: worker.roomId, workerId: worker._id, from, to: DEFAULT_MODEL, at: Date.now() }); + migrated += 1; + } + return { migrated, isDone: page.isDone, continueCursor: page.continueCursor }; +} + +export const migrateRetiredRoomModels = internalMutation({ + args: { table: v.optional(v.union(v.literal("rooms"), v.literal("workers"))), cursor: v.optional(v.union(v.string(), v.null())) }, + handler: (ctx, args) => (args.table === "workers" ? retireWorkerModels : retireRoomModels)(ctx, args.cursor ?? null), +}); + export const insertTraceInternal = internalMutation({ args: { roomId: v.id("rooms"), kind: v.string(), summary: v.string(), payload: v.any() }, handler: async (ctx, args) => { @@ -479,7 +538,9 @@ export const completeV3Worker = internalMutation({ await ctx.db.patch(args.workerId, { status: "completed", summary: args.summary.slice(0, 500), - ...(args.model ? { model: args.model } : {}), + // An in-flight planning call echoes the id it was started with. If the worker migration ran meanwhile, + // writing a retired id back would undo it and retryV3Worker would copy it. web_research rows keep the echo. + ...(args.model && (worker.kind !== "execution_plan" || ROUTER_MODELS.some((m) => m.id === args.model)) ? { model: args.model } : {}), completedAt: now, updatedAt: now, }); diff --git a/convex/schema.ts b/convex/schema.ts index 4017cac..8f7284e 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -142,4 +142,15 @@ export default defineSchema({ }) .index("by_room", ["roomId", "createdAt"]) .index("by_goal", ["goalId", "createdAt"]), + + // Audit rows written by rooms:migrateRetiredRoomModels when a stored room.model, or an + // execution_plan worker.model (workerId set), is no longer in ROUTER_MODELS + // (e.g. gpt-4.1-nano, retired by OpenAI 2026-10-23). + roomModelMigrations: defineTable({ + roomId: v.id("rooms"), + workerId: v.optional(v.id("workers")), + from: v.string(), + to: v.string(), + at: v.number(), + }).index("by_room", ["roomId", "at"]), }); diff --git a/scripts/migrate-retired-models.mjs b/scripts/migrate-retired-models.mjs new file mode 100644 index 0000000..7045924 --- /dev/null +++ b/scripts/migrate-retired-models.mjs @@ -0,0 +1,147 @@ +// Runs rooms:migrateRetiredRoomModels over EVERY page of `rooms` and `workers` and prints the +// totals. The deployment is never defaulted: `convex run` targets the DEV deployment unless told +// otherwise, so pass an explicit selector (--prod, --deployment-name, --preview-name, or --url with +// --admin-key); any other argument list is refused before spawning, e.g. node scripts/migrate-retired-models.mjs --prod +// It also refuses --env-file, and refuses to run while CONVEX_DEPLOY_KEY / CONVEX_DEPLOYMENT_TOKEN is visible (env, .env.local, .env), because the +// CLI then silently ignores --prod and targets the key's deployment. Run it after the deploy, then run it again: the second run must report migrated: 0 for both +// tables (that is the exhaustive stale-row check; `convex data` only lists the newest 100 rows). +// `--live-rooms ` is runbook step 1: it lists the running rooms through the same guards, because `convex data` also obeys a visible deploy key. +import { existsSync, readFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { createRequire } from "node:module"; +import { dirname, join } from "node:path"; +import { pathToFileURL } from "node:url"; + +// The Convex CLI is run as `node ...`, never through a shell: on Windows cmd.exe strips the quotes +// from the JSON argument and `convex run` (JSON5) then rejects `{table:rooms,cursor:null}`. +// The package's `exports` hides bin/main.js, so locate it from package.json and its `bin` field. +export function convexBin() { + const pkgPath = createRequire(import.meta.url).resolve("convex/package.json"); + const bin = JSON.parse(readFileSync(pkgPath, "utf8")).bin.convex; + return join(dirname(pkgPath), bin); +} + +/** Pages through one table until isDone, summing `migrated`. `run(args)` returns the parsed result. */ +export async function migrateTable(run, table) { + let cursor = null; + let migrated = 0; + let pages = 0; + for (;;) { + const page = await run({ table, cursor }); + migrated += page.migrated; + pages += 1; + if (page.isDone) return { table, migrated, pages }; + cursor = page.continueCursor; + } +} + +/** `convex run` prints strict JSON when stdout is not a TTY; function logs come first, so take the last top-level object. */ +export function parseRunOutput(stdout) { + const start = stdout.lastIndexOf("\n{") + 1; + return JSON.parse(stdout.slice(start)); +} + +/** `convex run` defaults to the DEV deployment, so refuse any argument list that does not name one explicitly. */ +export function assertDeploymentSelector(args) { + const has = (flag) => args.some((a) => a === flag || a.startsWith(`${flag}=`)); + const named = has("--prod") || has("--deployment-name") || has("--preview-name") || (has("--url") && has("--admin-key")); + if (!named) { + throw new Error("Pass an explicit deployment selector to convex run: --prod, --deployment-name , --preview-name , or --url with --admin-key . Without one it would migrate your DEV deployment."); + } +} + +// The Convex CLI's bundled dotenv@16.6.1 parse (cli.bundle.cjs, require_main), copied so the preflight sees the values the CLI +// would see: an empty value (K=, K="", K= # note) is unset, and a later line for the same key wins. +const DOTENV_LINE = /(?:^|^)\s*(?:export\s+)?([\w.-]+)(?:\s*=\s*?|:\s+?)(\s*'(?:\\'|[^'])*'|\s*"(?:\\"|[^"])*"|\s*`(?:\\`|[^`])*`|[^#\r\n]+)?\s*(?:#.*)?(?:$|$)/gm; +function dotenvParse(src) { + const obj = {}; + for (const m of src.replace(/\r\n?/gm, "\n").matchAll(DOTENV_LINE)) { + let value = (m[2] || "").trim(); + const quote = value[0]; + value = value.replace(/^(['"`])([\s\S]*)\1$/gm, "$2"); + if (quote === '"') value = value.replace(/\\n/g, "\n").replace(/\\r/g, "\r"); + obj[m[1]] = value; + } + return obj; +} + +/** + * The Convex CLI reads CONVEX_DEPLOY_KEY / CONVEX_DEPLOYMENT_TOKEN from the environment, `.env.local` and `.env` (the + * same sources, in the cwd) and, when it finds one, IGNORES --prod / --deployment-name / --preview-name and runs against + * the key's deployment (stderr warning only, exit 0). So a flag selector cannot be trusted while a key is visible: + * refuse instead of migrating, and proving clean, the wrong deployment. `--url` + `--admin-key` skips the key lookup. + * The file scan below runs the CLI's own dotenv line grammar (`KEY=v`, `KEY: v`, optional `export`) and flags only non-empty + * values, to fail fast with a readable message; convexRun's blanked child env is what actually keeps a key the scan misses from taking effect. + */ +export function assertNoDeployKey(selector, { cwd = process.cwd(), env = process.env } = {}) { + // `--env-file ` is a fourth key source the CLI reads (and obeys) before any selector; it is not inspected here, so refuse it. + if (selector.some((a) => a === "--env-file" || a.startsWith("--env-file="))) { + throw new Error("Refusing to run: --env-file can carry a CONVEX_DEPLOY_KEY that makes the CLI ignore the deployment flag and migrate the key's deployment. Drop it and select the deployment with --prod / --deployment-name / --preview-name, or --url --admin-key ."); + } + if (selector.some((a) => a === "--url" || a.startsWith("--url=")) && selector.some((a) => a === "--admin-key" || a.startsWith("--admin-key="))) return; + const names = ["CONVEX_DEPLOY_KEY", "CONVEX_DEPLOYMENT_TOKEN"]; + const where = names.filter((n) => env[n]).map((n) => `${n} in the environment`); + for (const file of [".env.local", ".env"]) { + const path = join(cwd, file); + if (!existsSync(path)) continue; + const text = readFileSync(path, "utf8"); + const parsed = dotenvParse(text); + for (const n of names) if (parsed[n]) where.push(`${n} in ${file}`); + } + if (where.length) { + throw new Error(`Refusing to run: ${where.join(", ")}. The Convex CLI would ignore the deployment flag and migrate the key's deployment instead. Unset/remove it, or select the deployment with --url --admin-key .`); + } +} + +/** Guards every spawn the same way: a named deployment, no visible deploy key, blanked key env, and a fail on the CLI's "Ignoring" warning. Returns exec(argv) -> stdout. */ +function convexExec(selector, cli, { cwd, env }) { + assertDeploymentSelector(selector); + assertNoDeployKey(selector, { cwd, env }); + return (argv) => { + // The CLI loads .env.local/.env with dotenv, which never overrides a variable that already exists, and treats "" as unset: + // blanking both here shadows any file key whatever its syntax, so the preflight above is a fail-fast message, not the guarantee. + const childEnv = { ...env, CONVEX_DEPLOY_KEY: "", CONVEX_DEPLOYMENT_TOKEN: "" }; + const r = spawnSync(process.execPath, [cli, ...argv], { encoding: "utf8", cwd, env: childEnv, maxBuffer: 1 << 30 }); + if (r.error) throw r.error; + if (r.status !== 0) throw new Error(`convex ${argv[0]} failed (exit ${r.status}): ${r.stderr}`); + // Backstop for any other way the CLI swaps the target: it only warns on stderr and exits 0. + if (/Ignoring `--prod`/.test(r.stderr)) throw new Error(`convex ${argv[0]} ignored the deployment selector, so nothing here can be trusted: ${r.stderr}`); + if (r.stderr) process.stderr.write(r.stderr); + return r.stdout; + }; +} + +export function convexRun(selector, cli = convexBin(), { cwd = process.cwd(), env = process.env } = {}) { + const exec = convexExec(selector, cli, { cwd, env }); + return (args) => parseRunOutput(exec(["run", ...selector, "rooms:migrateRetiredRoomModels", JSON.stringify(args)])); +} + +/** + * Runbook step 1 (SPEC A.2.5, live rooms are checked before removal): the rooms with `running: true`. It goes through the same + * guards as convexRun because `convex data` shares the CLI's deployment selection, so a visible deploy key would otherwise make it + * list ANOTHER deployment's rooms. `convex data` keeps only the newest `limit` rooms and `jsonl` prints no truncation warning, + * so a result that fills the limit is refused: raise the limit. + */ +export function liveRooms(selector, cli = convexBin(), { cwd = process.cwd(), env = process.env, limit = 100000 } = {}) { + const exec = convexExec(selector, cli, { cwd, env }); + const lines = exec(["data", "rooms", ...selector, "--format", "jsonl", "--limit", String(limit)]).split("\n").filter((l) => l.trim()); + if (lines.length >= limit) throw new Error(`convex data returned ${lines.length} rooms, the --limit: the list may be cut off. Raise the limit.`); + return lines.map((l) => JSON.parse(l)).filter((room) => room.running === true); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + const live = process.argv[2] === "--live-rooms"; + const selector = process.argv.slice(live ? 3 : 2); + try { + if (live) { + const running = liveRooms(selector); + console.log(JSON.stringify({ running: running.length, codes: running.map((r) => r.code ?? r._id) })); + } else { + const run = convexRun(selector); + for (const table of ["rooms", "workers"]) console.log(JSON.stringify(await migrateTable(run, table))); + } + } catch (e) { + console.error(e.message); + process.exit(2); + } +} diff --git a/scripts/model-eval.mjs b/scripts/model-eval.mjs index 3ccc2f3..7293765 100644 --- a/scripts/model-eval.mjs +++ b/scripts/model-eval.mjs @@ -20,10 +20,15 @@ const env = fs.readFileSync(resolve(root, ".env.local"), "utf8"); const KEY = env.match(/OPENAI_API_KEY=(.+)/)?.[1]?.trim(); if (!KEY) throw new Error("OPENAI_API_KEY missing from .env.local"); -// candidate coordinator models — [inputPrice, outputPrice] per 1M tokens +// gpt-4.1-nano dropped: retires 2026-10-23. gpt-5-nano/gpt-5-mini retire 2026-12-11 (drop in the follow-up PR). +// No new row without a measured run (paid, manual). +// candidate coordinator models — [inputPrice, outputPrice] per 1M tokens. +// PRICING_AS_OF = the day these prices were typed for the measured run behind docs/model-eval-results.json +// (commit 40cd6b6, 2026-07-04). They have NOT been re-read from the vendor since: SPEC A.2 requires a re-read +// on the day of the next paid run, then this date moves. It is not a "verified today" stamp. +const PRICING_AS_OF = "2026-07-04"; const MODELS = [ { id: "gpt-4o-mini", nextgen: false, price: [0.15, 0.6] }, - { id: "gpt-4.1-nano", nextgen: false, price: [0.1, 0.4] }, { id: "gpt-4.1-mini", nextgen: false, price: [0.4, 1.6] }, { id: "gpt-5-nano", nextgen: true, price: [0.05, 0.4] }, { id: "gpt-5-mini", nextgen: true, price: [0.25, 2.0] }, @@ -161,7 +166,7 @@ async function main() { }; }).sort((a, b) => b.quality - a.quality); - const out = { ranAt: new Date().toISOString?.() ?? Date.now(), judge: JUDGE, byModel, detail: judged }; + const out = { ranAt: new Date().toISOString?.() ?? Date.now(), pricingAsOf: PRICING_AS_OF, judge: JUDGE, byModel, detail: judged }; fs.writeFileSync(resolve(root, "docs/model-eval-results.json"), JSON.stringify(out, null, 2)); console.log("model quality p50 latency $/turn reasoning fails"); diff --git a/src/client/live/LiveRoom.tsx b/src/client/live/LiveRoom.tsx index 510358d..e3ac81b 100644 --- a/src/client/live/LiveRoom.tsx +++ b/src/client/live/LiveRoom.tsx @@ -1054,7 +1054,7 @@ function V3WorkPanel({ ); } -function ModelSelect({ rm }: { rm: ReturnType }) { +export function ModelSelect({ rm }: { rm: ReturnType }) { const room = rm.room!; const cur = room.models.find((m) => m.id === room.state.model); return ( @@ -1070,7 +1070,7 @@ function ModelSelect({ rm }: { rm: ReturnType }) { > {room.models.map((m) => ( ))} diff --git a/src/core/routerModels.ts b/src/core/routerModels.ts index 15b1d82..9c9a849 100644 --- a/src/core/routerModels.ts +++ b/src/core/routerModels.ts @@ -7,15 +7,22 @@ * One table, read by the local Node server and the hosted Convex backend alike. */ +// gpt-4.1-nano was removed here: OpenAI retires it 2026-10-23 (platform.openai.com +// deprecations page). Rooms that stored it are rewritten by rooms:migrateRetiredRoomModels +// and `validModel` falls back to DEFAULT_MODEL for any stale id. gpt-5-nano / gpt-5-mini +// (gpt-5-*-2025-08-07) retire 2026-12-11: marked in `note`, removed in a follow-up PR +// through the same migration. export const ROUTER_MODELS = [ { id: "gpt-5.4-mini", label: "GPT-5.4 mini", tier: "smart + fast", note: "default · smartest mini that stays ~1.3s", expectedLatencyMs: 1300, expectedCostUsd: 0.0007164375, qualityScore: 4.75 }, - { id: "gpt-4.1-nano", label: "GPT-4.1 nano", tier: "cheapest", note: "fastest (~0.7s) + cheapest", expectedLatencyMs: 700, expectedCostUsd: 0.000033425, qualityScore: 4.15 }, { id: "gpt-4.1-mini", label: "GPT-4.1 mini", tier: "balanced", note: "fast, mid capability", expectedLatencyMs: 700, expectedCostUsd: 0.0001409, qualityScore: 4.1 }, { id: "gpt-4o-mini", label: "GPT-4o mini", tier: "legacy", note: "older baseline", expectedLatencyMs: 1000, expectedCostUsd: 0.0000505875, qualityScore: 4.5 }, - { id: "gpt-5-nano", label: "GPT-5 nano", tier: "cheap + smart", note: "smart but ~3s (reasoning)", expectedLatencyMs: 3200, expectedCostUsd: 0.0001320625, qualityScore: 4.6 }, - { id: "gpt-5-mini", label: "GPT-5 mini", tier: "max quality", note: "top quality, ~3s+ (reasoning)", expectedLatencyMs: 3000, expectedCostUsd: 0.0007873125, qualityScore: 5 }, + { id: "gpt-5-nano", label: "GPT-5 nano", tier: "cheap + smart", note: "smart but ~3s (reasoning) · retires 2026-12-11", expectedLatencyMs: 3200, expectedCostUsd: 0.0001320625, qualityScore: 4.6 }, + { id: "gpt-5-mini", label: "GPT-5 mini", tier: "max quality", note: "top quality, ~3s+ (reasoning) · retires 2026-12-11", expectedLatencyMs: 3000, expectedCostUsd: 0.0007873125, qualityScore: 5 }, ] as const; +/** Day `expectedCostUsd` was measured: the run behind docs/model-eval-results.json (scripts/model-eval.mjs). */ +export const ROUTER_PRICING_AS_OF = "2026-07-04"; + export const DEFAULT_MODEL = "gpt-5.4-mini"; /** Never let a caller-supplied model id through unchecked. */ diff --git a/src/live/pipeline.ts b/src/live/pipeline.ts index ec188ed..bf6863a 100644 --- a/src/live/pipeline.ts +++ b/src/live/pipeline.ts @@ -6,16 +6,16 @@ */ import { coerceCountTurn, normalizeHumanSteeringIntent, type CapabilityProfile, type HumanSteeringIntent, type CountTask } from "../core/steering.js"; +import { DEFAULT_MODEL } from "../core/routerModels.js"; const OPENAI = "https://api.openai.com/v1"; const ELEVEN = "https://api.elevenlabs.io/v1"; const STT_MODEL = process.env.STT_MODEL ?? "whisper-1"; -// gpt-5.4-mini: latest, smartest mini, and — measured — as fast as gpt-4.1-nano -// (~1.4s/turn, 0 reasoning tokens on short conversational turns). Output price is -// higher but at ~70 tokens/turn it is dwarfed by STT/TTS cost. For the cheapest -// option instead, set OPENAI_MODEL=gpt-4.1-nano. -const LLM_MODEL = process.env.OPENAI_MODEL ?? "gpt-5.4-mini"; +// DEFAULT_MODEL (gpt-5.4-mini, src/core/routerModels.ts): measured ~1.4s/turn, 0 reasoning tokens on +// short turns (docs/model-eval-results.json); output price is dwarfed by STT/TTS cost at ~70 tokens/turn. +// gpt-4.1-nano retires 2026-10-23 (OpenAI deprecations page) — do not point OPENAI_MODEL at it. +const LLM_MODEL = process.env.OPENAI_MODEL ?? DEFAULT_MODEL; // gpt-5.x / o-series reasoning models use a different param shape and take a // reasoning_effort. Keep it low so the voice loop stays snappy. const REASONING_EFFORT = process.env.REASONING_EFFORT ?? "low"; diff --git a/src/providers/openai.ts b/src/providers/openai.ts index 1ec0689..6c355dd 100644 --- a/src/providers/openai.ts +++ b/src/providers/openai.ts @@ -1,4 +1,5 @@ import { isNextGenModel } from "../live/pipeline.js"; +import { DEFAULT_MODEL } from "../core/routerModels.js"; export type OpenAIChatMessage = { role: "system" | "user" | "assistant"; @@ -10,14 +11,11 @@ export type OpenAIChatOptions = { timeoutMs?: number; }; -/** Repo-measured coordinator default (see src/live/pipeline.ts model notes). */ -const FALLBACK_OPENAI_MODEL = "gpt-5.4-mini"; - export function getOpenAIModelName(override?: string): string { const trimmedOverride = override?.trim(); if (trimmedOverride) return trimmedOverride; const envModel = process.env.OPENAI_MODEL?.trim(); - return envModel || FALLBACK_OPENAI_MODEL; + return envModel || DEFAULT_MODEL; } /** Server-side only: the key is read from process.env and never leaves the server. */ diff --git a/tests/liveSteering.test.ts b/tests/liveSteering.test.ts index eefb0c3..3d84578 100644 --- a/tests/liveSteering.test.ts +++ b/tests/liveSteering.test.ts @@ -207,7 +207,7 @@ describe("live room steering", () => { it("exposes router cost and latency estimates for the live UI", () => { for (const models of [convexSteering.ROUTER_MODELS, ROUTER_MODELS]) { - expect(models).toHaveLength(6); + expect(models).toHaveLength(5); // was 6: gpt-4.1-nano removed, OpenAI retires it 2026-10-23 (platform.openai.com deprecations page) for (const model of models) { expect(model.expectedLatencyMs, model.id).toBeGreaterThan(0); expect(model.expectedCostUsd, model.id).toBeGreaterThan(0); diff --git a/tests/modelWritePath.test.ts b/tests/modelWritePath.test.ts new file mode 100644 index 0000000..37402d6 --- /dev/null +++ b/tests/modelWritePath.test.ts @@ -0,0 +1,71 @@ +/** + * SPEC-v3 D1: NodeVoice's write-path normalization of unknown model ids stays. A caller who sends a + * model id that is not a router row (a typo, a ":free" id, a retired id) gets the room's default model, + * not an error. Offline: no keys, no provider calls. + */ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { handleLive } from "../src/live/roomServer.js"; +import { DEFAULT_LLM_MODEL } from "../src/live/pipeline.js"; +import { DEFAULT_MODEL } from "../src/core/routerModels.js"; + +// convex/*.ts is type-checked by `npm run check:convex`; load by non-literal specifier as the other tests do. +const load = (path: string) => import(/* @vite-ignore */ path); +const rooms = await load("../convex/rooms.ts"); + +const UNKNOWN = ["not-a-model", "some-model:free", "gpt-4.1-nano", "GPT-5.4-MINI"]; + +function fakeCtx() { + const inserted: { table: string; doc: Record }[] = []; + const patches: Record[] = []; + const chain: any = { withIndex: () => chain, order: () => chain, first: async () => null, collect: async () => [], take: async () => [] }; + const ctx = { + db: { + query: () => chain, + get: async () => ({ _id: "room1", turn: 0 }), + insert: async (table: string, doc: Record) => (inserted.push({ table, doc }), `${table}${inserted.length}`), + patch: async (_id: string, fields: Record) => void patches.push(fields), + }, + }; + return { ctx: ctx as any, inserted, patches }; +} + +describe("D1: Convex createRoom / setModel normalize an unknown id to DEFAULT_MODEL", () => { + it.each(UNKNOWN)("%s", async (model) => { + const create = fakeCtx(); + await rooms.createRoom._handler(create.ctx, { model }); + expect(create.inserted.find((i) => i.table === "rooms")!.doc.model).toBe(DEFAULT_MODEL); + const set = fakeCtx(); + await rooms.setModel._handler(set.ctx, { roomId: "room1", model }); + expect(set.patches[0]!.model).toBe(DEFAULT_MODEL); + }); +}); + +describe("D1: the local server normalizes an unknown id to DEFAULT_LLM_MODEL", () => { + let server: Server; + let base = ""; + beforeAll(async () => { + server = createServer(async (req, res) => { + if (await handleLive(req, res, new URL(req.url ?? "/", "http://localhost").pathname)) return; + res.writeHead(404).end(); + }); + await new Promise((resolve) => server.listen(0, resolve)); + base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + }); + afterAll(() => new Promise((resolve) => server.close(() => resolve()))); + + const post = async (path: string, body: unknown) => { + const r = await fetch(base + path, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }); + return { status: r.status, body: (await r.json()) as any }; + }; + + it.each(UNKNOWN)("%s", async (model) => { + const created = await post("/live/rooms", { goal: "count to 3", model }); + expect(created.status).toBe(200); + expect(created.body.room.state.model).toBe(DEFAULT_LLM_MODEL); + const set = await post(`/live/rooms/${created.body.roomId}/model`, { model }); + expect(set.status).toBe(200); + expect(set.body.room.state.model).toBe(DEFAULT_LLM_MODEL); + }); +}); diff --git a/tests/roomModelMigration.test.ts b/tests/roomModelMigration.test.ts new file mode 100644 index 0000000..380dc62 --- /dev/null +++ b/tests/roomModelMigration.test.ts @@ -0,0 +1,615 @@ +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { DEFAULT_MODEL, ROUTER_MODELS, ROUTER_PRICING_AS_OF, validModel } from "../src/core/routerModels.js"; + +// convex/*.ts are type-checked by `npm run check:convex` (Convex module settings); the root +// `tsc` cannot compile them, so they are loaded by a non-literal specifier, as the app +// bundler does. Everything the tests call on them is asserted at runtime. +const load = (path: string) => import(/* @vite-ignore */ path); +const rooms = await load("../convex/rooms.ts"); +const coordinator = await load("../convex/coordinator.ts"); +const liveRoom = await load("../src/client/live/LiveRoom.tsx"); + +// OpenAI retires gpt-4.1-nano on 2026-10-23 (platform.openai.com deprecations page). +// A host who created a room before the row was removed still has it stored on `rooms.model`; +// `validModel` only runs on writes, so the coordinator used to send the dead id to OpenAI. +// Fixture of the PRE-CHANGE validator: the id list as it was before the row was removed. +const PRE_CHANGE_IDS = ["gpt-5.4-mini", "gpt-4.1-nano", "gpt-4.1-mini", "gpt-4o-mini", "gpt-5-nano", "gpt-5-mini"]; +const preChangeValidModel = (m?: string) => (m && PRE_CHANGE_IDS.includes(m) ? m : "gpt-5.4-mini"); + +afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); +}); + +type Row = Record; +const MIB = 1024 * 1024; +/** In-memory stand-in for a Convex MutationCtx: just the db calls the migration makes. */ +function fakeMutationCtx(roomRows: Row[], workerRows: Row[] = []) { + const patches: { id: string; fields: Row }[] = []; + const inserted: { table: string; doc: Row }[] = []; + // Convex's per-mutation limits (docs.convex.dev/production/state/limits, read 2026-10-16): 16 MiB read, 16 MiB written. + // The write cap below is a deliberate stricter margin (8 MiB, half the vendor limit), not the vendor figure: the page size must pass under either. + // A row's `bytes` is its document size; paginate reads it, patch re-reads and rewrites it. + const txn = { read: 0, written: 0 }; + const spend = (kind: "read" | "written", n: number, max: number) => { + txn[kind] += n; + if (txn[kind] > max) throw new Error(`Transaction exceeded ${kind} limit: ${txn[kind]} > ${max}`); + }; + const ctx = { + txn, + db: { + query: (table: string) => ({ + paginate: async ({ numItems, cursor }: { numItems: number; cursor: string | null }) => { + const rows = table === "rooms" ? roomRows : table === "workers" ? workerRows : null; + expect(rows, `unexpected table ${table}`).not.toBeNull(); + const start = cursor ? Number(cursor) : 0; + const page = rows!.slice(start, start + numItems); + for (const r of page) spend("read", r.bytes ?? 0, 16 * MIB); + const end = start + page.length; + return { page, isDone: end >= rows!.length, continueCursor: String(end) }; + }, + }), + patch: async (id: string, fields: Row) => { + const row = [...roomRows, ...workerRows].find((r) => r._id === id)!; + spend("read", row.bytes ?? 0, 16 * MIB); + spend("written", row.bytes ?? 0, 8 * MIB); + patches.push({ id, fields }); + Object.assign(row, fields); + }, + insert: async (table: string, doc: Row) => void inserted.push({ table, doc }), + }, + }; + return { ctx: ctx as any, patches, inserted, txn }; +} + +describe("retired router model: gpt-4.1-nano", () => { + it("was a selectable id before the change and is not now", () => { + expect(preChangeValidModel("gpt-4.1-nano")).toBe("gpt-4.1-nano"); // the stale value is reachable + expect(ROUTER_MODELS.map((m) => m.id)).not.toContain("gpt-4.1-nano"); + expect(validModel("gpt-4.1-nano")).toBe(DEFAULT_MODEL); + expect(ROUTER_MODELS).toHaveLength(5); + }); + + it("the picker options a host chooses from carry the retirement date for the gpt-5 rows only", () => { + // render the real ; its