From a1703804573c1b4008f84fd5b9af3610b8a120c1 Mon Sep 17 00:00:00 2001 From: Homen Shum Date: Wed, 14 Oct 2026 01:50:45 -0700 Subject: [PATCH 01/19] chore(models): refresh stale model defaults (NodeVoice) Remove the gpt-4.1-nano router row (OpenAI retirement 2026-10-23), migrate stored rooms via rooms:migrateRetiredRoomModels, guard the coordinator with validModel(room.model), mark gpt-5-nano/gpt-5-mini as retiring 2026-12-11, prune scripts/model-eval.mjs, and import DEFAULT_MODEL in the two literal copies. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 15 +++++ convex/coordinator.ts | 12 ++-- convex/rooms.ts | 26 +++++++ convex/schema.ts | 9 +++ scripts/model-eval.mjs | 3 +- src/core/routerModels.ts | 10 ++- src/live/pipeline.ts | 10 +-- src/providers/openai.ts | 6 +- tests/liveSteering.test.ts | 2 +- tests/roomModelMigration.test.ts | 112 +++++++++++++++++++++++++++++++ 10 files changed, 185 insertions(+), 20 deletions(-) create mode 100644 tests/roomModelMigration.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index bc97026..fefbe5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # NodeVoice changelog +## 2026-10-14 — Remove the retired gpt-4.1-nano router row + +OpenAI retires `gpt-4.1-nano` on 2026-10-23. The room model picker no longer +offers it, rooms that already stored it are rewritten to `gpt-5.4-mini` by +`rooms:migrateRetiredRoomModels` (audit rows in `roomModelMigrations`), and the +coordinator resolves `validModel(room.model)` so a stale stored id can no longer +reach OpenAI. `gpt-5-nano` / `gpt-5-mini` rows are marked "retires 2026-12-11" +and are removed in a follow-up PR through the same migration. The three literal +`gpt-5.4-mini` copies now import `DEFAULT_MODEL`. Spec: model-defaults SPEC +Part A section A.2 (NodeVoice). + +**Validation:** `tests/roomModelMigration.test.ts` fails on the previous code +(request carried `gpt-4.1-nano`) and passes now; `npm run check`, `check:client`, +`check:convex`, `check:citations` and `npm test` pass. No provider calls made. + ## 2026-10-07 — Make setup and proof limits reachable Developers and coding agents can reach the existing `HANDOFF.md` from the README diff --git a/convex/coordinator.ts b/convex/coordinator.ts index 48d1bc0..62ae525 100644 --- a/convex/coordinator.ts +++ b/convex/coordinator.ts @@ -2,7 +2,7 @@ import { v } from "convex/values"; import { internalAction, action } from "./_generated/server"; import { internal, api } from "./_generated/api"; -import { agentForSlot, nextSlot, estimateSpeechMs, deriveHumanSteeringIntentFallback, validProfile, coerceCountTurn, type CountTask, type Slot } from "./shared"; +import { agentForSlot, nextSlot, estimateSpeechMs, deriveHumanSteeringIntentFallback, validProfile, validModel, coerceCountTurn, type CountTask, type Slot } from "./shared"; import { generateAgentTurn, interpretHumanSteer as interpretHumanSteerWithModel, @@ -18,7 +18,7 @@ import type { Id } from "./_generated/dataModel"; * The commit re-validates floor/token server-side (the LLM+TTS work here is * seconds long and non-transactional) and reports whether it landed. */ -async function produceTurn( +export async function produceTurn( ctx: any, roomId: Id<"rooms">, room: any, @@ -38,7 +38,7 @@ async function produceTurn( ? coerceCountTurn({ text: String(countTask.next), speechAct: "task_action", done: countTask.next >= countTask.target }, countTask) : await generateAgentTurn({ goal: room.goal, - model: room.model, + model: validModel(room.model), // stale stored ids (retired models) fall back to DEFAULT_MODEL profile: validProfile(room.profile), persona: agent.persona, selfName: agent.name, @@ -124,7 +124,7 @@ export const interpretHumanSteer = internalAction({ intent = await interpretHumanSteerWithModel({ text: args.text, currentGoal: room.goal, - model: room.model, + model: validModel(room.model), profile: validProfile(room.profile), transcript, }); @@ -190,8 +190,8 @@ export const runV3Worker = internalAction({ const taskTitle = raw.task?.title ?? worker.title; const output = worker.kind === "web_research" - ? await runWebResearchWorker({ goal: goal.title, task: taskTitle, roomGoal: room.goal, model: worker.model }) - : await runPlanningWorker({ goal: goal.title, task: taskTitle, roomGoal: room.goal, model: worker.model ?? room.model }); + ? await runWebResearchWorker({ goal: goal.title, task: taskTitle, roomGoal: room.goal, model: worker.model ? validModel(worker.model) : undefined }) + : await runPlanningWorker({ goal: goal.title, task: taskTitle, roomGoal: room.goal, model: validModel(worker.model ?? room.model) }); await ctx.runMutation(internal.rooms.completeV3Worker, { workerId: args.workerId, diff --git a/convex/rooms.ts b/convex/rooms.ts index 4217802..55a3edc 100644 --- a/convex/rooms.ts +++ b/convex/rooms.ts @@ -7,6 +7,7 @@ import { activeSlots, CAPABILITY_PROFILES, ROUTER_MODELS, + DEFAULT_MODEL, DEFAULT_GOAL, agentIndexFromSlot, isAgentSlot, @@ -417,6 +418,31 @@ export const getV3WorkerRaw = internalQuery({ }, }); +/** + * Rewrite rooms whose stored `model` is no longer in ROUTER_MODELS (a retired id such as + * gpt-4.1-nano, OpenAI 2026-10-23) to DEFAULT_MODEL and record {roomId, from, to, at}. + * `validModel` only runs on writes, so without this a stale value reaches OpenAI as-is. + * Paged: run `npx convex run rooms:migrateRetiredRoomModels` before AND after the deploy + * that removes the row; re-run with {"cursor": } while isDone is false. + */ +export async function retireRoomModels(ctx: MutationCtx, cursor: string | null) { + const page = await ctx.db.query("rooms").paginate({ numItems: 200, cursor }); + let migrated = 0; + for (const room of page.page) { + if (ROUTER_MODELS.some((m) => m.id === room.model)) continue; + const from = room.model; + await ctx.db.patch(room._id, { model: DEFAULT_MODEL }); + await ctx.db.insert("roomModelMigrations", { roomId: room._id, from, to: DEFAULT_MODEL, at: Date.now() }); + migrated += 1; + } + return { migrated, isDone: page.isDone, continueCursor: page.continueCursor }; +} + +export const migrateRetiredRoomModels = internalMutation({ + args: { cursor: v.optional(v.union(v.string(), v.null())) }, + handler: (ctx, args) => retireRoomModels(ctx, args.cursor ?? null), +}); + export const insertTraceInternal = internalMutation({ args: { roomId: v.id("rooms"), kind: v.string(), summary: v.string(), payload: v.any() }, handler: async (ctx, args) => { diff --git a/convex/schema.ts b/convex/schema.ts index 4017cac..7c92c7b 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -142,4 +142,13 @@ export default defineSchema({ }) .index("by_room", ["roomId", "createdAt"]) .index("by_goal", ["goalId", "createdAt"]), + + // Audit rows written by rooms:migrateRetiredRoomModels when a stored room.model + // is no longer in ROUTER_MODELS (e.g. gpt-4.1-nano, retired by OpenAI 2026-10-23). + roomModelMigrations: defineTable({ + roomId: v.id("rooms"), + from: v.string(), + to: v.string(), + at: v.number(), + }).index("by_room", ["roomId", "at"]), }); diff --git a/scripts/model-eval.mjs b/scripts/model-eval.mjs index 3ccc2f3..8f4939b 100644 --- a/scripts/model-eval.mjs +++ b/scripts/model-eval.mjs @@ -20,10 +20,11 @@ const env = fs.readFileSync(resolve(root, ".env.local"), "utf8"); const KEY = env.match(/OPENAI_API_KEY=(.+)/)?.[1]?.trim(); if (!KEY) throw new Error("OPENAI_API_KEY missing from .env.local"); +// gpt-4.1-nano dropped: retires 2026-10-23. gpt-5-nano/gpt-5-mini retire 2026-12-11 (drop in the follow-up PR). +// No new row without a measured run (paid, manual). // candidate coordinator models — [inputPrice, outputPrice] per 1M tokens const MODELS = [ { id: "gpt-4o-mini", nextgen: false, price: [0.15, 0.6] }, - { id: "gpt-4.1-nano", nextgen: false, price: [0.1, 0.4] }, { id: "gpt-4.1-mini", nextgen: false, price: [0.4, 1.6] }, { id: "gpt-5-nano", nextgen: true, price: [0.05, 0.4] }, { id: "gpt-5-mini", nextgen: true, price: [0.25, 2.0] }, diff --git a/src/core/routerModels.ts b/src/core/routerModels.ts index 15b1d82..c2c388a 100644 --- a/src/core/routerModels.ts +++ b/src/core/routerModels.ts @@ -7,13 +7,17 @@ * One table, read by the local Node server and the hosted Convex backend alike. */ +// gpt-4.1-nano was removed here: OpenAI retires it 2026-10-23 (platform.openai.com +// deprecations page). Rooms that stored it are rewritten by rooms:migrateRetiredRoomModels +// and `validModel` falls back to DEFAULT_MODEL for any stale id. gpt-5-nano / gpt-5-mini +// (gpt-5-*-2025-08-07) retire 2026-12-11: marked in `note`, removed in a follow-up PR +// through the same migration. export const ROUTER_MODELS = [ { id: "gpt-5.4-mini", label: "GPT-5.4 mini", tier: "smart + fast", note: "default · smartest mini that stays ~1.3s", expectedLatencyMs: 1300, expectedCostUsd: 0.0007164375, qualityScore: 4.75 }, - { id: "gpt-4.1-nano", label: "GPT-4.1 nano", tier: "cheapest", note: "fastest (~0.7s) + cheapest", expectedLatencyMs: 700, expectedCostUsd: 0.000033425, qualityScore: 4.15 }, { id: "gpt-4.1-mini", label: "GPT-4.1 mini", tier: "balanced", note: "fast, mid capability", expectedLatencyMs: 700, expectedCostUsd: 0.0001409, qualityScore: 4.1 }, { id: "gpt-4o-mini", label: "GPT-4o mini", tier: "legacy", note: "older baseline", expectedLatencyMs: 1000, expectedCostUsd: 0.0000505875, qualityScore: 4.5 }, - { id: "gpt-5-nano", label: "GPT-5 nano", tier: "cheap + smart", note: "smart but ~3s (reasoning)", expectedLatencyMs: 3200, expectedCostUsd: 0.0001320625, qualityScore: 4.6 }, - { id: "gpt-5-mini", label: "GPT-5 mini", tier: "max quality", note: "top quality, ~3s+ (reasoning)", expectedLatencyMs: 3000, expectedCostUsd: 0.0007873125, qualityScore: 5 }, + { id: "gpt-5-nano", label: "GPT-5 nano", tier: "cheap + smart", note: "smart but ~3s (reasoning) · retires 2026-12-11", expectedLatencyMs: 3200, expectedCostUsd: 0.0001320625, qualityScore: 4.6 }, + { id: "gpt-5-mini", label: "GPT-5 mini", tier: "max quality", note: "top quality, ~3s+ (reasoning) · retires 2026-12-11", expectedLatencyMs: 3000, expectedCostUsd: 0.0007873125, qualityScore: 5 }, ] as const; export const DEFAULT_MODEL = "gpt-5.4-mini"; diff --git a/src/live/pipeline.ts b/src/live/pipeline.ts index ec188ed..bf6863a 100644 --- a/src/live/pipeline.ts +++ b/src/live/pipeline.ts @@ -6,16 +6,16 @@ */ import { coerceCountTurn, normalizeHumanSteeringIntent, type CapabilityProfile, type HumanSteeringIntent, type CountTask } from "../core/steering.js"; +import { DEFAULT_MODEL } from "../core/routerModels.js"; const OPENAI = "https://api.openai.com/v1"; const ELEVEN = "https://api.elevenlabs.io/v1"; const STT_MODEL = process.env.STT_MODEL ?? "whisper-1"; -// gpt-5.4-mini: latest, smartest mini, and — measured — as fast as gpt-4.1-nano -// (~1.4s/turn, 0 reasoning tokens on short conversational turns). Output price is -// higher but at ~70 tokens/turn it is dwarfed by STT/TTS cost. For the cheapest -// option instead, set OPENAI_MODEL=gpt-4.1-nano. -const LLM_MODEL = process.env.OPENAI_MODEL ?? "gpt-5.4-mini"; +// DEFAULT_MODEL (gpt-5.4-mini, src/core/routerModels.ts): measured ~1.4s/turn, 0 reasoning tokens on +// short turns (docs/model-eval-results.json); output price is dwarfed by STT/TTS cost at ~70 tokens/turn. +// gpt-4.1-nano retires 2026-10-23 (OpenAI deprecations page) — do not point OPENAI_MODEL at it. +const LLM_MODEL = process.env.OPENAI_MODEL ?? DEFAULT_MODEL; // gpt-5.x / o-series reasoning models use a different param shape and take a // reasoning_effort. Keep it low so the voice loop stays snappy. const REASONING_EFFORT = process.env.REASONING_EFFORT ?? "low"; diff --git a/src/providers/openai.ts b/src/providers/openai.ts index 1ec0689..6c355dd 100644 --- a/src/providers/openai.ts +++ b/src/providers/openai.ts @@ -1,4 +1,5 @@ import { isNextGenModel } from "../live/pipeline.js"; +import { DEFAULT_MODEL } from "../core/routerModels.js"; export type OpenAIChatMessage = { role: "system" | "user" | "assistant"; @@ -10,14 +11,11 @@ export type OpenAIChatOptions = { timeoutMs?: number; }; -/** Repo-measured coordinator default (see src/live/pipeline.ts model notes). */ -const FALLBACK_OPENAI_MODEL = "gpt-5.4-mini"; - export function getOpenAIModelName(override?: string): string { const trimmedOverride = override?.trim(); if (trimmedOverride) return trimmedOverride; const envModel = process.env.OPENAI_MODEL?.trim(); - return envModel || FALLBACK_OPENAI_MODEL; + return envModel || DEFAULT_MODEL; } /** Server-side only: the key is read from process.env and never leaves the server. */ diff --git a/tests/liveSteering.test.ts b/tests/liveSteering.test.ts index eefb0c3..3d84578 100644 --- a/tests/liveSteering.test.ts +++ b/tests/liveSteering.test.ts @@ -207,7 +207,7 @@ describe("live room steering", () => { it("exposes router cost and latency estimates for the live UI", () => { for (const models of [convexSteering.ROUTER_MODELS, ROUTER_MODELS]) { - expect(models).toHaveLength(6); + expect(models).toHaveLength(5); // was 6: gpt-4.1-nano removed, OpenAI retires it 2026-10-23 (platform.openai.com deprecations page) for (const model of models) { expect(model.expectedLatencyMs, model.id).toBeGreaterThan(0); expect(model.expectedCostUsd, model.id).toBeGreaterThan(0); diff --git a/tests/roomModelMigration.test.ts b/tests/roomModelMigration.test.ts new file mode 100644 index 0000000..6c954fb --- /dev/null +++ b/tests/roomModelMigration.test.ts @@ -0,0 +1,112 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { DEFAULT_MODEL, ROUTER_MODELS, validModel } from "../src/core/routerModels.js"; + +// convex/*.ts are type-checked by `npm run check:convex` (Convex module settings); the root +// `tsc` cannot compile them, so they are loaded by a non-literal specifier, as the app +// bundler does. Everything the tests call on them is asserted at runtime. +const load = (path: string) => import(/* @vite-ignore */ path); +const rooms = await load("../convex/rooms.ts"); +const coordinator = await load("../convex/coordinator.ts"); + +// OpenAI retires gpt-4.1-nano on 2026-10-23 (platform.openai.com deprecations page). +// A host who created a room before the row was removed still has it stored on `rooms.model`; +// `validModel` only runs on writes, so the coordinator used to send the dead id to OpenAI. +// Fixture of the PRE-CHANGE validator: the id list as it was before the row was removed. +const PRE_CHANGE_IDS = ["gpt-5.4-mini", "gpt-4.1-nano", "gpt-4.1-mini", "gpt-4o-mini", "gpt-5-nano", "gpt-5-mini"]; +const preChangeValidModel = (m?: string) => (m && PRE_CHANGE_IDS.includes(m) ? m : "gpt-5.4-mini"); + +afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); +}); + +type Row = Record; +/** In-memory stand-in for a Convex MutationCtx: just the db calls the migration makes. */ +function fakeMutationCtx(roomRows: Row[]) { + const patches: { id: string; fields: Row }[] = []; + const inserted: { table: string; doc: Row }[] = []; + const ctx = { + db: { + query: (table: string) => ({ + paginate: async ({ numItems, cursor }: { numItems: number; cursor: string | null }) => { + expect(table).toBe("rooms"); + const start = cursor ? Number(cursor) : 0; + const page = roomRows.slice(start, start + numItems); + const end = start + page.length; + return { page, isDone: end >= roomRows.length, continueCursor: String(end) }; + }, + }), + patch: async (id: string, fields: Row) => { + patches.push({ id, fields }); + Object.assign(roomRows.find((r) => r._id === id)!, fields); + }, + insert: async (table: string, doc: Row) => void inserted.push({ table, doc }), + }, + }; + return { ctx: ctx as any, patches, inserted }; +} + +describe("retired router model: gpt-4.1-nano", () => { + it("was a selectable id before the change and is not now", () => { + expect(preChangeValidModel("gpt-4.1-nano")).toBe("gpt-4.1-nano"); // the stale value is reachable + expect(ROUTER_MODELS.map((m) => m.id)).not.toContain("gpt-4.1-nano"); + expect(validModel("gpt-4.1-nano")).toBe(DEFAULT_MODEL); + expect(ROUTER_MODELS).toHaveLength(5); + }); + + it("marks the gpt-5 rows as retiring 2026-12-11 in the picker", () => { + for (const id of ["gpt-5-nano", "gpt-5-mini"]) { + expect(ROUTER_MODELS.find((m) => m.id === id)?.note, id).toContain("retires 2026-12-11"); + } + }); + + it("migrateRetiredRoomModels rewrites stale rooms to DEFAULT_MODEL, records them, and leaves valid rooms alone", async () => { + const roomRows: Row[] = [ + { _id: "r1", model: "gpt-4.1-nano" }, + { _id: "r2", model: "gpt-5.4-mini" }, + { _id: "r3", model: "gpt-4.1-mini" }, + { _id: "r4", model: "gpt-4.1-nano" }, + ]; + const { ctx, patches, inserted } = fakeMutationCtx(roomRows); + const out = await rooms.retireRoomModels(ctx, null); + expect(out).toMatchObject({ migrated: 2, isDone: true }); + expect(patches).toEqual([ + { id: "r1", fields: { model: DEFAULT_MODEL } }, + { id: "r4", fields: { model: DEFAULT_MODEL } }, + ]); + expect(inserted.map((i) => i.table)).toEqual(["roomModelMigrations", "roomModelMigrations"]); + expect(inserted[0]!.doc).toMatchObject({ roomId: "r1", from: "gpt-4.1-nano", to: DEFAULT_MODEL }); + expect(typeof inserted[0]!.doc.at).toBe("number"); + // idempotent: the second run (before + after the deploy) finds nothing to do + const again = await rooms.retireRoomModels(ctx, null); + expect(again.migrated).toBe(0); + }); + + it("the coordinator's next turn for a stale room requests gpt-5.4-mini in the gpt-5 request shape", async () => { + vi.stubEnv("OPENAI_API_KEY", "test-key"); + const bodies: { url: string; body: Record }[] = []; + vi.stubGlobal("fetch", async (url: string | URL, init?: RequestInit) => { + const u = String(url); + bodies.push({ url: u, body: JSON.parse(String(init?.body)) }); + if (u.endsWith("/chat/completions")) { + return new Response(JSON.stringify({ choices: [{ message: { content: JSON.stringify({ text: "Let's begin.", speechAct: "task_action", done: false }) } }] }), { status: 200 }); + } + return new Response("no tts in test", { status: 500 }); // speech is best-effort + }); + const runMutation = vi.fn(async () => ({ committed: true })); + const ctx = { runQuery: vi.fn(async () => ({ utterances: [] })), runMutation } as any; + const staleRoom = { goal: "Plan a launch", model: "gpt-4.1-nano", floorOwner: "agent-001", agentCount: 2, goalVersion: 0, recentActs: [] }; + + const out = await coordinator.produceTurn(ctx, "room1" as any, staleRoom, 1); + + expect(out.committed).toBe(true); + const chat = bodies.find((b) => b.url.endsWith("/chat/completions"))!; + expect(chat.body.model).toBe("gpt-5.4-mini"); + // gpt-5.x shape: the 4.x-only keys must be absent + expect(chat.body).not.toHaveProperty("temperature"); + expect(chat.body).not.toHaveProperty("max_tokens"); + expect(chat.body.max_completion_tokens).toBe(1500); + expect(chat.body.reasoning_effort).toBe("low"); + expect(bodies.some((b) => b.body.model === "gpt-4.1-nano")).toBe(false); + }); +}); From 32c14deb349d278ee80c8e090733c2ee814c1874 Mon Sep 17 00:00:00 2001 From: homen Date: Wed, 14 Oct 2026 02:18:39 -0700 Subject: [PATCH 02/19] fix(models): keep worker.model as passed; connected migration test; post-deploy-only runbook Codex review round 1: the web-research branch wrapped a stored worker.model in validModel, which collapsed a valid out-of-picker id (e.g. an OPENAI_WEB_MODEL override stored by completeV3Worker and copied by retryV3Worker) to gpt-5.4-mini. Restore the caller argument; guard only the room.model fallback. Add the create -> migrate -> audit -> next-turn test for one room and a worker argument-preservation test. Runbook: run the migration once after the deploy. Co-Authored-By: Claude Sonnet 5.5 --- convex/coordinator.ts | 4 +-- convex/rooms.ts | 6 ++-- tests/roomModelMigration.test.ts | 47 ++++++++++++++++++++++++++------ 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/convex/coordinator.ts b/convex/coordinator.ts index 62ae525..029d7f8 100644 --- a/convex/coordinator.ts +++ b/convex/coordinator.ts @@ -190,8 +190,8 @@ export const runV3Worker = internalAction({ const taskTitle = raw.task?.title ?? worker.title; const output = worker.kind === "web_research" - ? await runWebResearchWorker({ goal: goal.title, task: taskTitle, roomGoal: room.goal, model: worker.model ? validModel(worker.model) : undefined }) - : await runPlanningWorker({ goal: goal.title, task: taskTitle, roomGoal: room.goal, model: validModel(worker.model ?? room.model) }); + ? await runWebResearchWorker({ goal: goal.title, task: taskTitle, roomGoal: room.goal, model: worker.model }) + : await runPlanningWorker({ goal: goal.title, task: taskTitle, roomGoal: room.goal, model: worker.model ?? validModel(room.model) }); await ctx.runMutation(internal.rooms.completeV3Worker, { workerId: args.workerId, diff --git a/convex/rooms.ts b/convex/rooms.ts index 55a3edc..628571a 100644 --- a/convex/rooms.ts +++ b/convex/rooms.ts @@ -422,8 +422,10 @@ export const getV3WorkerRaw = internalQuery({ * Rewrite rooms whose stored `model` is no longer in ROUTER_MODELS (a retired id such as * gpt-4.1-nano, OpenAI 2026-10-23) to DEFAULT_MODEL and record {roomId, from, to, at}. * `validModel` only runs on writes, so without this a stale value reaches OpenAI as-is. - * Paged: run `npx convex run rooms:migrateRetiredRoomModels` before AND after the deploy - * that removes the row; re-run with {"cursor": } while isDone is false. + * Paged: run `npx convex run rooms:migrateRetiredRoomModels` once AFTER the deploy that + * removes the row (before it the function does not exist, or gpt-4.1-nano is still in the + * table so it is a no-op); re-run with {"cursor": } while isDone is false. + * Until it finishes, the coordinator read-side guard (validModel) covers unmigrated rooms. */ export async function retireRoomModels(ctx: MutationCtx, cursor: string | null) { const page = await ctx.db.query("rooms").paginate({ numItems: 200, cursor }); diff --git a/tests/roomModelMigration.test.ts b/tests/roomModelMigration.test.ts index 6c954fb..c124f0b 100644 --- a/tests/roomModelMigration.test.ts +++ b/tests/roomModelMigration.test.ts @@ -77,12 +77,13 @@ describe("retired router model: gpt-4.1-nano", () => { expect(inserted.map((i) => i.table)).toEqual(["roomModelMigrations", "roomModelMigrations"]); expect(inserted[0]!.doc).toMatchObject({ roomId: "r1", from: "gpt-4.1-nano", to: DEFAULT_MODEL }); expect(typeof inserted[0]!.doc.at).toBe("number"); - // idempotent: the second run (before + after the deploy) finds nothing to do + // idempotent: a repeated run finds nothing to do const again = await rooms.retireRoomModels(ctx, null); expect(again.migrated).toBe(0); }); - it("the coordinator's next turn for a stale room requests gpt-5.4-mini in the gpt-5 request shape", async () => { + /** Run the coordinator's real turn for `room` against a fake OpenAI transport; return the bodies it sent. */ + async function turnRequestBodies(room: Row) { vi.stubEnv("OPENAI_API_KEY", "test-key"); const bodies: { url: string; body: Record }[] = []; vi.stubGlobal("fetch", async (url: string | URL, init?: RequestInit) => { @@ -93,13 +94,15 @@ describe("retired router model: gpt-4.1-nano", () => { } return new Response("no tts in test", { status: 500 }); // speech is best-effort }); - const runMutation = vi.fn(async () => ({ committed: true })); - const ctx = { runQuery: vi.fn(async () => ({ utterances: [] })), runMutation } as any; - const staleRoom = { goal: "Plan a launch", model: "gpt-4.1-nano", floorOwner: "agent-001", agentCount: 2, goalVersion: 0, recentActs: [] }; - - const out = await coordinator.produceTurn(ctx, "room1" as any, staleRoom, 1); - + const ctx = { runQuery: vi.fn(async () => ({ utterances: [] })), runMutation: vi.fn(async () => ({ committed: true })) } as any; + const out = await coordinator.produceTurn(ctx, "room1" as any, room, 1); expect(out.committed).toBe(true); + return bodies; + } + const baseRoom = { goal: "Plan a launch", floorOwner: "agent-001", agentCount: 2, goalVersion: 0, recentActs: [] }; + + it("the coordinator's next turn for a stale room requests gpt-5.4-mini in the gpt-5 request shape", async () => { + const bodies = await turnRequestBodies({ ...baseRoom, model: "gpt-4.1-nano" }); const chat = bodies.find((b) => b.url.endsWith("/chat/completions"))!; expect(chat.body.model).toBe("gpt-5.4-mini"); // gpt-5.x shape: the 4.x-only keys must be absent @@ -109,4 +112,32 @@ describe("retired router model: gpt-4.1-nano", () => { expect(chat.body.reasoning_effort).toBe("low"); expect(bodies.some((b) => b.body.model === "gpt-4.1-nano")).toBe(false); }); + + it("one room, end to end: created with the pre-change validator, migrated, audited, then its next turn requests gpt-5.4-mini", async () => { + // created through the PRE-CHANGE validator: nano was a valid pick for the host + const room: Row = { _id: "r1", ...baseRoom, model: preChangeValidModel("gpt-4.1-nano") }; + expect(room.model).toBe("gpt-4.1-nano"); + const { ctx, inserted } = fakeMutationCtx([room]); + await rooms.retireRoomModels(ctx, null); + expect(room.model).toBe(DEFAULT_MODEL); + expect(inserted).toHaveLength(1); + expect(inserted[0]!.doc).toMatchObject({ roomId: "r1", from: "gpt-4.1-nano", to: DEFAULT_MODEL }); + const bodies = await turnRequestBodies(room); + expect(bodies.find((b) => b.url.endsWith("/chat/completions"))!.body.model).toBe("gpt-5.4-mini"); + }); + + it("a web-research worker keeps its stored model, even one outside the picker (caller argument preserved)", async () => { + // completeV3Worker stores the resolved model (e.g. an OPENAI_WEB_MODEL override); retryV3Worker copies it. + vi.stubEnv("OPENAI_API_KEY", "test-key"); + const bodies: Record[] = []; + vi.stubGlobal("fetch", async (_u: string | URL, init?: RequestInit) => { + bodies.push(JSON.parse(String(init?.body))); + return new Response("stop", { status: 500 }); // only the request model matters here + }); + const worker = { kind: "web_research", title: "Research", model: "gpt-5.4" }; + const runQuery = vi.fn(async () => ({ worker, goal: { title: "Goal" }, room: { goal: "Plan", model: "gpt-4.1-nano" } })); + const runMutation = vi.fn(async (_ref: unknown, a: Record) => ("workerId" in a && !("error" in a) && !("content" in a) ? { started: true } : undefined)); + await coordinator.runV3Worker._handler({ runQuery, runMutation } as any, { workerId: "w1" }); + expect(bodies[0]!.model).toBe("gpt-5.4"); + }); }); From 59099aaa1339902d5833c2ec1225ebb60b48e50c Mon Sep 17 00:00:00 2001 From: homen Date: Wed, 14 Oct 2026 07:03:17 -0700 Subject: [PATCH 03/19] fix(models): guard new planning workers at spawn; per-role request-shape tests; picker retirement note; pricing as-of Review round 1 on PR 16. - convex/rooms.ts: spawnAgentOsGoal stored room.model on the execution_plan worker, so a steer in an unmigrated room created a worker that sent gpt-4.1-nano. Store validModel(room.model). - tests: request-shape tests for the human-steer interpreter and for a planning worker spawned through applyHumanIntent then run by runV3Worker (and the runV3Worker fallback); reverting each validModel guard now fails a test. Picker test renders the real options. - LiveRoom: dropdown options carry "retires 2026-12-11" for rows that have it. - scripts/model-eval.mjs: PRICING_AS_OF stamp (date typed, not re-verified) + id-coverage test. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 33 +++++++--- convex/rooms.ts | 2 +- scripts/model-eval.mjs | 8 ++- src/client/live/LiveRoom.tsx | 4 +- tests/roomModelMigration.test.ts | 108 +++++++++++++++++++++++++++++-- 5 files changed, 136 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fefbe5a..fe2a5e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,16 +4,29 @@ OpenAI retires `gpt-4.1-nano` on 2026-10-23. The room model picker no longer offers it, rooms that already stored it are rewritten to `gpt-5.4-mini` by -`rooms:migrateRetiredRoomModels` (audit rows in `roomModelMigrations`), and the -coordinator resolves `validModel(room.model)` so a stale stored id can no longer -reach OpenAI. `gpt-5-nano` / `gpt-5-mini` rows are marked "retires 2026-12-11" -and are removed in a follow-up PR through the same migration. The three literal -`gpt-5.4-mini` copies now import `DEFAULT_MODEL`. Spec: model-defaults SPEC -Part A section A.2 (NodeVoice). - -**Validation:** `tests/roomModelMigration.test.ts` fails on the previous code -(request carried `gpt-4.1-nano`) and passes now; `npm run check`, `check:client`, -`check:convex`, `check:citations` and `npm test` pass. No provider calls made. +`rooms:migrateRetiredRoomModels` (audit rows in `roomModelMigrations`), and every +request the coordinator builds from `room.model` (the turn, the human-steer +interpreter, the planning-worker fallback) resolves it through `validModel`. +New planning workers also store `validModel(room.model)` at spawn, so a host +steer in a room that has not been migrated yet does not create a worker that +sends the retired id. `gpt-5-nano` / `gpt-5-mini` rows are marked "retires +2026-12-11" (in the dropdown options, not only the hover title) and are removed +in a follow-up PR through the same migration. The three literal `gpt-5.4-mini` +copies now import `DEFAULT_MODEL`. `scripts/model-eval.mjs` carries +`PRICING_AS_OF` (the date its per-M prices were typed, not re-verified). Spec: +model-defaults SPEC Part A section A.2 (NodeVoice). + +**Known residual:** a worker row created BEFORE this change that already stores +`gpt-4.1-nano` keeps it (worker model is a caller argument, kept as stored), so +retrying it after 2026-10-23 sends that id. Rooms and all newly spawned workers +are covered. + +**Validation:** `tests/roomModelMigration.test.ts` (10 tests) fails on the +previous code (request carried `gpt-4.1-nano`) and passes now, with one +request-shape test per changed coordinator role (turn, steer interpreter, +planning worker spawned through the real steer path); `npm run check`, +`check:client`, `check:convex`, `check:citations` and `npm test` pass. No +provider calls made. ## 2026-10-07 — Make setup and proof limits reachable diff --git a/convex/rooms.ts b/convex/rooms.ts index 628571a..35bc935 100644 --- a/convex/rooms.ts +++ b/convex/rooms.ts @@ -177,7 +177,7 @@ async function spawnAgentOsGoal( ? [{ kind: "deterministic_count", title: "Prepare exact count sequence", model: undefined as string | undefined }] : [ { kind: "web_research", title: "Research current external context", model: "gpt-4.1-mini" }, - { kind: "execution_plan", title: "Draft execution plan", model: room.model }, + { kind: "execution_plan", title: "Draft execution plan", model: validModel(room.model) }, ]; for (const spec of workerSpecs) { const budgetBlocked = workersUsed >= policy.budgetMaxWorkers; diff --git a/scripts/model-eval.mjs b/scripts/model-eval.mjs index 8f4939b..7293765 100644 --- a/scripts/model-eval.mjs +++ b/scripts/model-eval.mjs @@ -22,7 +22,11 @@ if (!KEY) throw new Error("OPENAI_API_KEY missing from .env.local"); // gpt-4.1-nano dropped: retires 2026-10-23. gpt-5-nano/gpt-5-mini retire 2026-12-11 (drop in the follow-up PR). // No new row without a measured run (paid, manual). -// candidate coordinator models — [inputPrice, outputPrice] per 1M tokens +// candidate coordinator models — [inputPrice, outputPrice] per 1M tokens. +// PRICING_AS_OF = the day these prices were typed for the measured run behind docs/model-eval-results.json +// (commit 40cd6b6, 2026-07-04). They have NOT been re-read from the vendor since: SPEC A.2 requires a re-read +// on the day of the next paid run, then this date moves. It is not a "verified today" stamp. +const PRICING_AS_OF = "2026-07-04"; const MODELS = [ { id: "gpt-4o-mini", nextgen: false, price: [0.15, 0.6] }, { id: "gpt-4.1-mini", nextgen: false, price: [0.4, 1.6] }, @@ -162,7 +166,7 @@ async function main() { }; }).sort((a, b) => b.quality - a.quality); - const out = { ranAt: new Date().toISOString?.() ?? Date.now(), judge: JUDGE, byModel, detail: judged }; + const out = { ranAt: new Date().toISOString?.() ?? Date.now(), pricingAsOf: PRICING_AS_OF, judge: JUDGE, byModel, detail: judged }; fs.writeFileSync(resolve(root, "docs/model-eval-results.json"), JSON.stringify(out, null, 2)); console.log("model quality p50 latency $/turn reasoning fails"); diff --git a/src/client/live/LiveRoom.tsx b/src/client/live/LiveRoom.tsx index 510358d..e3ac81b 100644 --- a/src/client/live/LiveRoom.tsx +++ b/src/client/live/LiveRoom.tsx @@ -1054,7 +1054,7 @@ function V3WorkPanel({ ); } -function ModelSelect({ rm }: { rm: ReturnType }) { +export function ModelSelect({ rm }: { rm: ReturnType }) { const room = rm.room!; const cur = room.models.find((m) => m.id === room.state.model); return ( @@ -1070,7 +1070,7 @@ function ModelSelect({ rm }: { rm: ReturnType }) { > {room.models.map((m) => ( ))} diff --git a/tests/roomModelMigration.test.ts b/tests/roomModelMigration.test.ts index c124f0b..d2daca7 100644 --- a/tests/roomModelMigration.test.ts +++ b/tests/roomModelMigration.test.ts @@ -1,3 +1,5 @@ +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; import { afterEach, describe, expect, it, vi } from "vitest"; import { DEFAULT_MODEL, ROUTER_MODELS, validModel } from "../src/core/routerModels.js"; @@ -7,6 +9,7 @@ import { DEFAULT_MODEL, ROUTER_MODELS, validModel } from "../src/core/routerMode const load = (path: string) => import(/* @vite-ignore */ path); const rooms = await load("../convex/rooms.ts"); const coordinator = await load("../convex/coordinator.ts"); +const liveRoom = await load("../src/client/live/LiveRoom.tsx"); // OpenAI retires gpt-4.1-nano on 2026-10-23 (platform.openai.com deprecations page). // A host who created a room before the row was removed still has it stored on `rooms.model`; @@ -54,10 +57,14 @@ describe("retired router model: gpt-4.1-nano", () => { expect(ROUTER_MODELS).toHaveLength(5); }); - it("marks the gpt-5 rows as retiring 2026-12-11 in the picker", () => { - for (const id of ["gpt-5-nano", "gpt-5-mini"]) { - expect(ROUTER_MODELS.find((m) => m.id === id)?.note, id).toContain("retires 2026-12-11"); - } + it("the picker options a host chooses from carry the retirement date for the gpt-5 rows only", () => { + // render the real ; its