diff --git a/src/server/responses/core-codex-account.ts b/src/server/responses/core-codex-account.ts index 99796846d8f..ca507c6dde0 100644 --- a/src/server/responses/core-codex-account.ts +++ b/src/server/responses/core-codex-account.ts @@ -173,8 +173,9 @@ export function normalizeCodexUnsupportedModelDetail(value: string): string { * that comparison fail for the one model that is still account-gated, which silently disabled * both the alternate-account retry and the same-account ladder built for exactly that case. * - * The envelope is unchanged and stays exact: a top-level `detail` string, whitespace-collapsed - * and case-folded, matching the whole sentence with nothing before or after it. No prose is + * Accept the HTTP `detail` envelope and the `error.message` envelope emitted by the + * WebSocket refused-create projection. Both must match the whole sentence, whitespace-collapsed + * and case-folded, with nothing before or after it. Competing envelopes are ambiguous. No prose is * inferred and no other 400 shape is admitted, because a 400 is also what a malformed request * earns and that must never read as an entitlement fact. */ @@ -186,7 +187,18 @@ export function codexUnsupportedModelFromDetail( try { const payload = JSON.parse(bodyText) as unknown; if (!payload || typeof payload !== "object" || Array.isArray(payload)) return undefined; - const detail = (payload as { detail?: unknown }).detail; + const record = payload as Record; + const hasDetail = Object.hasOwn(record, "detail"); + const hasError = Object.hasOwn(record, "error"); + if (hasDetail === hasError) return undefined; + let detail: unknown = record.detail; + if (hasError) { + const error = record.error; + if (!error || typeof error !== "object" || Array.isArray(error)) return undefined; + const fields = error as Record; + if ([fields.type, fields.code].some(value => value != null && typeof value !== "string")) return undefined; + detail = fields.message; + } if (typeof detail !== "string") return undefined; const matched = /^the '([^']{1,256})' model is not supported when using codex with a chatgpt account\.$/u .exec(normalizeCodexUnsupportedModelDetail(detail)); diff --git a/src/server/responses/request-spend.ts b/src/server/responses/request-spend.ts index 16537cc3e98..d017eb22f6e 100644 --- a/src/server/responses/request-spend.ts +++ b/src/server/responses/request-spend.ts @@ -1,6 +1,7 @@ import { randomUUID } from "node:crypto"; import type { RequestSendObserver } from "../../lib/request-execution-budget"; import { sharedSpendLedger, type SpendReservationLedger } from "../../lib/spend-reservation-ledger"; +import { SpendLedgerOwnerError } from "../../lib/spend-ledger-owner"; import { markLocalRequestLogRefusal, type RequestLogContext } from "../request-log"; import { recordWorkflowRefusalEvent, workflowDenialSummary } from "../../lib/workflow-budget"; @@ -137,19 +138,28 @@ export function createRequestSpendTracker( if (resolved) return; resolved = true; const terminal = live.pop(); - if (terminal !== undefined) { - const reported = typeof usage?.inputTokens === "number" || typeof usage?.outputTokens === "number"; - if (reported) { - ledger().settle(terminal, { - inputTokens: usage?.inputTokens ?? 0, - outputTokens: usage?.outputTokens ?? 0, - }); - } else { - // The response never reported usage. It may still have been billed. - ledger().markLost(terminal); + try { + if (terminal !== undefined) { + const reported = typeof usage?.inputTokens === "number" || typeof usage?.outputTokens === "number"; + if (reported) { + ledger().settle(terminal, { + inputTokens: usage?.inputTokens ?? 0, + outputTokens: usage?.outputTokens ?? 0, + }); + } else { + // The response never reported usage. It may still have been billed. + ledger().markLost(terminal); + } } + for (const sendId of live.splice(0)) ledger().markLost(sendId); + } catch (error) { + // Settlement is the last thing a request does, and a deferred one can outlive the + // ledger's ownership window: a post-cancel drain that finishes after server.stop + // released the owner finds the journal already closed, leaving the outstanding + // sends nobody to book against and no caller alive to refuse. They die with the + // discarded ledger; anything that is not an ownership lapse still propagates. + if (!(error instanceof SpendLedgerOwnerError)) throw error; } - for (const sendId of live.splice(0)) ledger().markLost(sendId); }, get refusals(): number { return refusals; }, }; diff --git a/structure/providers/openai-tiers.md b/structure/providers/openai-tiers.md index 0fde5683acd..d46c5b9ed2a 100644 --- a/structure/providers/openai-tiers.md +++ b/structure/providers/openai-tiers.md @@ -484,6 +484,11 @@ Native Spark membership and its model-specific request/tool exceptions are remov `gpt-5.6-sol` before dispatch; comparing against the route model alone never matched for the one model that is still account-gated, which disabled both its alternate-account retry and its same-account ladder. + Refusal detection accepts the HTTP `detail` envelope and the WebSocket refused-create + projection's `error.message` envelope. Both require HTTP 400 and the complete model-specific + refusal sentence; malformed or competing envelopes, unrelated errors, and postcommit stream + errors authorize no replay. The same evidence feeds the bounded alternate attempt and later + automatic selection without changing a manual pin or the threshold-zero quota policy. `getEligiblePoolAccounts` is not the only door, so `preferModelEntitledAccount` applies the same evidence to an already-active shared cursor: the replacement is drawn from the eligible list, the active account is returned unchanged when no entitled alternative exists, and the correction is diff --git a/structure/transports/responses.md b/structure/transports/responses.md index a82e8f8cb08..5cfe6efc972 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -366,6 +366,12 @@ readable user text, and records `conversationStateScrub: "account-change"` on th without account identifiers. Once the new account issues its own state, later turns carry it normally. `canPortConversationState` is local until `src/routing/identity-domains.ts` lands. +Precommit Codex model refusals use the same bounded account recovery for HTTP `detail` and +WebSocket-projected `error.message` bodies. Only an exact HTTP 400 refusal naming the requested +or wire model establishes denial evidence; ordinary malformed requests and committed stream +errors do not authorize another send. Account selectors, uploaded files and send budgets retain +their existing restrictions. + ### Uploaded files do not move between accounts An uploaded `file_id` has always been classified as account-bound, and the scrub has always @@ -1245,7 +1251,10 @@ during this process's lifetime can still be released for free. Settlement follows what the request learned. The terminal usage belongs to the last send that left, so that one settles with the real figure; every earlier send failed without reporting usage of its own and may still have been billed, so it becomes unresolved spend rather than free. A -request that reports no usage at all leaves all of them unresolved. +request that reports no usage at all leaves all of them unresolved. A settlement deferred past +the ledger's ownership window — a post-cancel drain that outlives `server.stop` and its owner +release — drops the outstanding sends with the closed journal instead of throwing, because there +is nothing durable left to book against and no caller alive to refuse. Replay resolves what nobody is left to settle, and resolves it as unresolved spend whatever state it was in. Giving an undispatched one its tokens back would assume the journal is complete up to diff --git a/tests/codex-integration/codex-account-selection-preferences.test.ts b/tests/codex-integration/codex-account-selection-preferences.test.ts index db5d61c0e05..cb0ebce4da0 100644 --- a/tests/codex-integration/codex-account-selection-preferences.test.ts +++ b/tests/codex-integration/codex-account-selection-preferences.test.ts @@ -6,11 +6,18 @@ import { CODEX_THREAD_AFFINITY_REEVAL_INTERVAL_MS, clearCodexUpstreamHealth, clearThreadAccountMap, + previewCodexAccountForRequest, resolveCodexAccountForThread, resolveCodexAccountForThreadDetailed, } from "../../src/codex/routing"; import { clearPoolRotationState } from "../../src/codex/pool-rotation"; -import { saveCodexAccountCredential } from "../../src/codex/account-store"; +import { readCodexAccountRecord, saveCodexAccountCredential } from "../../src/codex/account-store"; +import { getEffectiveActiveCodexAccountId, rememberActiveCodexAccount } from "../../src/codex/routing/active-account"; +import { + cachedDeniedCodexAccountIdsForModel, + recordCodexModelDenialEvidence, + resetCodexModelEntitlementCacheForTests, +} from "../../src/codex/model-entitlements"; import { clearAccountNeedsReauth, clearAccountQuota, @@ -102,6 +109,8 @@ function installScratchState(): void { clearCodexUpstreamHealth(); clearAccountQuota(); clearPoolRotationState(); + resetCodexModelEntitlementCacheForTests(); + clearAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); clearAccountNeedsReauth("a"); clearAccountNeedsReauth("b"); saveTestCredential("a"); @@ -114,6 +123,8 @@ async function removeScratchState(): Promise { clearCodexUpstreamHealth(); clearThreadAccountMap(); clearPoolRotationState(); + resetCodexModelEntitlementCacheForTests(); + clearAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); clearAccountNeedsReauth("a"); clearAccountNeedsReauth("b"); } finally { @@ -125,6 +136,44 @@ describe("model entitlement ordering (#4768)", () => { beforeEach(installScratchState); afterEach(removeScratchState); + test.each([ + ["observed Astra denial selects available main", true, MAIN_CODEX_ACCOUNT_ID], + ["unknown Astra access preserves the exhausted automatic cursor", false, "a"], + ] as const)("fill-first with threshold zero: %s", (_name, observedDenial, expectedAccountId) => { + const now = Date.now(); + const modelId = "gpt-6-astra"; + const config = makeConfig({ + accountPoolStrategy: "fill-first", + autoSwitchThreshold: 0, + activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID, + codexAccounts: [{ id: "a", email: "a@test", isMain: false, plan: "free" }], + }); + updateAccountQuota("a", undefined, undefined, 100); // Free's governing monthly window. + updateAccountQuota(MAIN_CODEX_ACCOUNT_ID, 6); // 94% remaining. + // An automatic cursor is not a manual pin, including when persisted active is main. + rememberActiveCodexAccount(config, "a"); + expect(config.activeCodexAccountPinned).toBeUndefined(); + expect(cachedDeniedCodexAccountIdsForModel(modelId, now)).toBeUndefined(); + if (observedDenial) { + recordCodexModelDenialEvidence("a", modelId, readCodexAccountRecord("a")!.generation, now); + } + const deniedModelAccountIds = cachedDeniedCodexAccountIdsForModel(modelId, now); + if (observedDenial) expect([...(deniedModelAccountIds ?? [])]).toEqual(["a"]); + else expect(deniedModelAccountIds).toBeUndefined(); + const selectionOptions = { isMainAccountTokenLive: () => true, deniedModelAccountIds }; + + // Preview must neither advance the automatic cursor nor manufacture a user selection. + expect(previewCodexAccountForRequest("initial-astra", config, now, "shared", selectionOptions, modelId)) + .toBe(expectedAccountId); + expect(getEffectiveActiveCodexAccountId(config)).toBe("a"); + expect(resolveCodexAccountForThreadDetailed( + "initial-astra", config, now, "shared", selectionOptions, modelId, + )).toMatchObject({ status: "selected", accountId: expectedAccountId }); + expect(getEffectiveActiveCodexAccountId(config)).toBe(expectedAccountId); + expect(config.activeCodexAccountId).toBe(MAIN_CODEX_ACCOUNT_ID); + expect(config.activeCodexAccountPinned).toBeUndefined(); + }); + /** `a` is ordered above `b`; the persisted operator selection is the lower tier. */ function orderedConfig(overrides: Partial = {}): OcxConfig { return makeConfig({ diff --git a/tests/codex-integration/codex-model-denial-evidence.test.ts b/tests/codex-integration/codex-model-denial-evidence.test.ts index 18e12f7d44b..140af84774d 100644 --- a/tests/codex-integration/codex-model-denial-evidence.test.ts +++ b/tests/codex-integration/codex-model-denial-evidence.test.ts @@ -135,6 +135,18 @@ describe("upstream refusal as per-account model denial evidence", () => { * same-account ladder that exists specifically for it. */ describe("unsupported-model refusal detection", () => { + test("recognizes the exact refusal in the WebSocket HTTP error envelope", async () => { + const body = JSON.stringify({ error: { + type: "invalid_request_error", + code: "invalid_request_error", + message: `The '${ASTRA}' model is not supported when using Codex with a ChatGPT account.`, + } }); + expect(codexUnsupportedModelFromDetail(400, body)).toBe(ASTRA); + expect(await shouldRetryCodexPoolAccountModel400(new Response(body, { status: 400 }), ASTRA)).toBe(true); + expect(isAllowListedCodexAccountModel400(400, body, SOL)).toBe(false); + expect(codexUnsupportedModelFromDetail(403, body)).toBeUndefined(); + }); + test("extracts the model upstream named", () => { expect(codexUnsupportedModelFromDetail(400, refusalBody(SOL))).toBe(SOL); // Case and whitespace are normalized exactly as before. @@ -143,7 +155,18 @@ describe("unsupported-model refusal detection", () => { }))).toBe(SOL); }); - test("admits nothing but that exact envelope", () => { + test("rejects malformed, competing, and non-refusal error envelopes", () => { + const message = `The '${ASTRA}' model is not supported when using Codex with a ChatGPT account.`; + for (const payload of [ + { error: message }, { error: [message] }, { error: null }, + { error: { message: 400 } }, { error: { message: { detail: message } } }, + { error: { message, code: 42 } }, { error: { message, type: [] } }, + { error: { message: `note: ${message}` } }, { error: { message: "Invalid tool schema" } }, + { detail: message, error: { message } }, { detail: null, error: { message } }, + ]) expect(codexUnsupportedModelFromDetail(400, JSON.stringify(payload))).toBeUndefined(); + }); + + test("admits nothing but an exact refusal envelope", () => { expect(codexUnsupportedModelFromDetail(400, JSON.stringify({ detail: "Bad request" }))) .toBeUndefined(); // Prose around the sentence is not the sentence. diff --git a/tests/helpers/codex-pool-retry.ts b/tests/helpers/codex-pool-retry.ts new file mode 100644 index 00000000000..0702ff72ce3 --- /dev/null +++ b/tests/helpers/codex-pool-retry.ts @@ -0,0 +1,224 @@ +import { expect } from "bun:test"; +import { existsSync, mkdirSync } from "node:fs"; +import { removeTreeWithRetry } from "./remove-tree"; +import { saveConfig } from "../../src/config"; +import { saveCodexAccountCredential } from "../../src/codex/account-store"; +import { clearCodexWebSocketRegistry } from "../../src/codex/websocket-registry"; +import { clearAccountNeedsReauth, clearAccountQuota, markAccountNeedsReauth, updateAccountQuota } from "../../src/codex/auth-api"; +import { clearCodexUpstreamHealth, clearThreadAccountMap } from "../../src/codex/routing"; +import { resetCodexModelEntitlementCacheForTests } from "../../src/codex/model-entitlements"; +import { clearRequestLogsForTests } from "../../src/server/request-log"; +import { startServer } from "../../src/server"; +import type { OcxConfig, OcxProviderConfig } from "../../src/types"; + +export const POOL_RETRY_MODEL = "gpt-5.5"; + +/** Reuse the server-auth suite's owned home and transport redirection unchanged. */ +export function createPoolRetryHarness(context: { + testDir: string; + originalFetch: typeof fetch; + redirectCanonicalCodexTo: (baseUrl: string) => void; + canonicalDirect: OcxProviderConfig; +}) { + const { testDir: TEST_DIR, originalFetch: originalGlobalFetch, + redirectCanonicalCodexTo, canonicalDirect } = context; + + function unsupportedModelBody(model = POOL_RETRY_MODEL): string { + return JSON.stringify({ + detail: `The '${model}' model is not supported when using Codex with a ChatGPT account.`, + }); + } + + type PoolRetryHarness = { + config: OcxConfig; + dispatches: string[]; + request: (init?: { + stream?: boolean; + signal?: AbortSignal; + model?: string; + path?: "/v1/responses" | "/v1/responses/compact"; + callerBearer?: boolean; + headers?: Record; + extraBody?: Record; + }) => Promise; + restoreFetch: () => void; + server: ReturnType; + upstream: ReturnType; + }; + + async function removeTestDirBestEffort(dir: string): Promise { + if (!existsSync(dir)) return; + // Windows can keep the prior harness's ACL/icacls handles for a beat after + // stop; a single EBUSY must not take down the rest of the file. + for (let attempt = 0; attempt < 8; attempt++) { + try { + removeTreeWithRetry(dir); + return; + } catch (err) { + const code = err && typeof err === "object" && "code" in err ? String((err as { code: unknown }).code) : ""; + if (code !== "EBUSY" && code !== "EPERM" && code !== "ENOTEMPTY") throw err; + await Bun.sleep(25 * (attempt + 1)); + } + } + removeTreeWithRetry(dir); + } + + async function startPoolRetryHarness( + reply: (accountId: string, request: Request) => Response | Promise, + options: { + secondAccount?: boolean; + streamMode?: "legacy-tee" | "eager-relay"; + accountMode?: "direct" | "pool"; + activeAccountId?: string; + accountNamespaces?: Record; + noVisionModels?: string[]; + visionSidecarModel?: string; + websockets?: boolean; + forwardApiKey?: string; + pausedAccountIds?: string[]; + reauthAccountIds?: string[]; + omitCredentialAccountIds?: string[]; + combos?: OcxConfig["combos"]; + modelRosterByAccount?: Record; + } = {}, + ): Promise { + await removeTestDirBestEffort(TEST_DIR); + mkdirSync(TEST_DIR, { recursive: true }); + process.env.OPENCODEX_HOME = TEST_DIR; + clearCodexUpstreamHealth(); + clearThreadAccountMap(); + clearAccountQuota(); + resetCodexModelEntitlementCacheForTests(); + clearRequestLogsForTests(); + clearAccountNeedsReauth("pool-a"); + clearAccountNeedsReauth("pool-b"); + // The registry is process-global and survives a harness teardown. WS-REBIND-01 + // asserts exact per-account socket counts, so a socket leaked by any earlier test + // in this file shifts its snapshots and fails it in milliseconds — which reads as + // a flake next to the timeouts, but is ordinary shared state. Reset it with the + // rest rather than leaving one of six kinds of state uncleaned. + clearCodexWebSocketRegistry(); + + const dispatches: string[] = []; + const upstream = Bun.serve({ + port: 0, + async fetch(request) { + const accountId = request.headers.get("chatgpt-account-id") ?? "missing"; + if (new URL(request.url).pathname === "/models") { + return Response.json({ + models: (options.modelRosterByAccount?.[accountId] ?? []).map(slug => ({ + slug, + supported_in_api: true, + visibility: "list", + })), + }); + } + dispatches.push(accountId); + return reply(accountId, request); + }, + }); + redirectCanonicalCodexTo(upstream.url.toString()); + const redirectedFetch = globalThis.fetch; + + const secondAccount = options.secondAccount ?? true; + const config = { + port: 0, + defaultProvider: "openai", + openaiProviderTierVersion: 2, + providers: { + openai: { + ...canonicalDirect, + codexAccountMode: options.accountMode ?? "pool", + ...(options.noVisionModels ? { noVisionModels: options.noVisionModels } : {}), + ...(options.forwardApiKey ? { apiKey: options.forwardApiKey } : {}), + }, + }, + codexAccounts: [ + { id: "main", email: "main@example.test", isMain: true }, + { id: "pool-a", email: "pool-a@example.test", isMain: false, chatgptAccountId: "acct-pool-a" }, + ...(secondAccount + ? [{ id: "pool-b", email: "pool-b@example.test", isMain: false, chatgptAccountId: "acct-pool-b" }] + : []), + ], + activeCodexAccountId: options.activeAccountId ?? "pool-a", + ...(options.accountNamespaces ? { codexAccountNamespaces: options.accountNamespaces } : {}), + ...(options.pausedAccountIds ? { pausedCodexAccountIds: options.pausedAccountIds } : {}), + ...(options.visionSidecarModel ? { visionSidecar: { model: options.visionSidecarModel } } : {}), + ...(options.websockets ? { websockets: true } : {}), + ...(options.streamMode ? { streamMode: options.streamMode } : {}), + ...(options.combos ? { combos: options.combos } : {}), + } as OcxConfig; + saveConfig(config); + if (!options.omitCredentialAccountIds?.includes("pool-a")) { + saveCodexAccountCredential("pool-a", { + accessToken: "pool-a-token", + refreshToken: "pool-a-refresh", + expiresAt: Date.now() + 10 * 60_000, + chatgptAccountId: "acct-pool-a", + }); + } + updateAccountQuota("pool-a", 10); + if (secondAccount) { + if (!options.omitCredentialAccountIds?.includes("pool-b")) { + saveCodexAccountCredential("pool-b", { + accessToken: "pool-b-token", + refreshToken: "pool-b-refresh", + expiresAt: Date.now() + 10 * 60_000, + chatgptAccountId: "acct-pool-b", + }); + } + updateAccountQuota("pool-b", 20); + } + for (const accountId of options.reauthAccountIds ?? []) markAccountNeedsReauth(accountId); + + const server = startServer(0); + return { + config, + dispatches, + restoreFetch: () => { + if (globalThis.fetch === redirectedFetch) globalThis.fetch = originalGlobalFetch; + }, + server, + upstream, + request: ({ + stream = false, + signal, + model = POOL_RETRY_MODEL, + path = "/v1/responses", + callerBearer = true, + headers = {}, + extraBody = {}, + } = {}) => originalGlobalFetch(new URL(path, server.url), { + method: "POST", + headers: { + "content-type": "application/json", + ...(callerBearer ? { authorization: "Bearer inbound-token" } : {}), + ...headers, + }, + body: JSON.stringify({ model, input: path.endsWith("/compact") ? [] : "hello", stream, ...extraBody }), + signal, + }), + }; + } + + async function stopPoolRetryHarness(harness: PoolRetryHarness): Promise { + harness.restoreFetch(); + await harness.server.stop(true); + await harness.upstream.stop(true); + } + + function rejectionResponse(body: BodyInit, headers: Record = {}): Response { + return new Response(body, { + status: 400, + statusText: "Account Model Rejected", + headers: { "content-type": "application/json", "x-pool-retry-test": "original", ...headers }, + }); + } + + async function expectOriginal400(response: Response, body: string): Promise { + expect(response.status).toBe(400); + expect(response.headers.get("x-pool-retry-test")).toBe("original"); + expect(await response.text()).toBe(body); + } + return { startPoolRetryHarness, stopPoolRetryHarness, rejectionResponse, expectOriginal400, unsupportedModelBody }; +} diff --git a/tests/responses/responses-spend-ledger-wiring.test.ts b/tests/responses/responses-spend-ledger-wiring.test.ts index 592410a3618..a8e72686c07 100644 --- a/tests/responses/responses-spend-ledger-wiring.test.ts +++ b/tests/responses/responses-spend-ledger-wiring.test.ts @@ -1,4 +1,7 @@ import { describe, expect, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { createSpendReservationLedger, DEFAULT_SPEND_RESERVATION_POLICY, @@ -6,6 +9,8 @@ import { } from "../../src/lib/spend-reservation-ledger"; import { createRequestExecutionBudget } from "../../src/lib/request-execution-budget"; import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; /** * The durable spend ledger had no production caller (#4707). @@ -211,4 +216,29 @@ describe("the request path books every physical send on the durable ledger", () ); expect(observing.reserveDispatch({ sendClass: "initial", targetKey: "p|m" }).allowed).toBe(true); }); + + test("a settle deferred past the ledger's ownership window drops the sends instead of throwing", () => { + // addFinalRequestLog settles the tracker when the terminal row is written, and a + // post-cancel drain can defer that write past server.stop, which releases the owner. + // The released journal is already closed to this process, so the outstanding sends die + // with it rather than crashing a finalize that has no caller left to refuse. + const dir = mkdtempSync(join(tmpdir(), "ocx-spend-wiring-")); + const previousHome = process.env.OPENCODEX_HOME; + process.env.OPENCODEX_HOME = dir; + const release = acquireOwnedSpendHome(); + try { + const tracker = createRequestSpendTracker(logContext(), "root-i"); + const budget = createRequestExecutionBudget(undefined, "lr-released", tracker); + expect(budget.reserveDispatch({ sendClass: "initial", targetKey: "p|m" }).allowed).toBe(true); + + release(); + tracker.settle({ inputTokens: 10, outputTokens: 5 }); + tracker.settle(undefined); + } finally { + release(); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + removeTreeWithRetry(dir); + } + }); }); diff --git a/tests/responses/ws-upstream.test.ts b/tests/responses/ws-upstream.test.ts index 8ea36d1349b..f306942bc47 100644 --- a/tests/responses/ws-upstream.test.ts +++ b/tests/responses/ws-upstream.test.ts @@ -9,6 +9,7 @@ import { isEagerRelaySseResponse } from "../../src/server/relay"; import { isWin32EagerRewrite } from "../../src/lib/bun-stream-caps"; import { fetchWithTransientRetry, isNonReplayableResponse } from "../../src/lib/upstream-retry"; import { codexWsExchange } from "../../src/server/responses/codex-ws-exchange"; +import { shouldRetryCodexPoolAccountModel400 } from "../../src/server/responses/core-codex-account"; import { CodexWsSession } from "../../src/server/responses/codex-ws-session"; import { prepareCodexWsRequest } from "../../src/server/responses/codex-ws-request"; import { readCodexWsStage } from "../../src/server/responses/codex-ws-wire"; @@ -898,6 +899,14 @@ describe("codexWsUpstreamFetch", () => { } // Independent oracle: openai/codex d2d5b702, responses_websocket.rs:1016-1064 + test("a wrapped Astra refusal reaches the alternate-account recovery predicate", async () => { + const message = "The 'gpt-6-astra' model is not supported when using Codex with a ChatGPT account."; + const response = await receive({ type: "error", status_code: 400, + error: { type: "invalid_request_error", code: "invalid_request_error", message } }); + expect(await shouldRetryCodexPoolAccountModel400(response, "gpt-6-astra")).toBe(true); + expect(await response.json()).toEqual({ error: { type: "invalid_request_error", code: "invalid_request_error", message } }); + }); + // explicitly accepts numeric window-minutes as the HTTP header string "15". test.each(["status", "status_code"])("returns %s 429 as bounded HTTP JSON with scalar quota headers", async field => { const { status_code, ...frame } = refusal; diff --git a/tests/server/server-auth.test.ts b/tests/server/server-auth.test.ts index aec00ee0be2..85ce4875be3 100644 --- a/tests/server/server-auth.test.ts +++ b/tests/server/server-auth.test.ts @@ -1,3 +1,5 @@ +import { createPoolRetryHarness, POOL_RETRY_MODEL } from "../helpers/codex-pool-retry"; +import * as boundedBody from "../../src/lib/bounded-body"; import { waitForNativeMainStartupGate } from "../../src/codex/native-profile-startup"; import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; import { createHash } from "node:crypto"; @@ -188,205 +190,9 @@ afterEach(() => { if (existsSync(TEST_DIR)) removeTreeWithRetry(TEST_DIR); }); -const POOL_RETRY_MODEL = "gpt-5.5"; - -function unsupportedModelBody(model = POOL_RETRY_MODEL): string { - return JSON.stringify({ - detail: `The '${model}' model is not supported when using Codex with a ChatGPT account.`, - }); -} - -type PoolRetryHarness = { - config: OcxConfig; - dispatches: string[]; - request: (init?: { - stream?: boolean; - signal?: AbortSignal; - model?: string; - path?: "/v1/responses" | "/v1/responses/compact"; - callerBearer?: boolean; - headers?: Record; - extraBody?: Record; - }) => Promise; - restoreFetch: () => void; - server: ReturnType; - upstream: ReturnType; -}; - -async function removeTestDirBestEffort(dir: string): Promise { - if (!existsSync(dir)) return; - // Windows can keep the prior harness's ACL/icacls handles for a beat after - // stop; a single EBUSY must not take down the rest of the file. - for (let attempt = 0; attempt < 8; attempt++) { - try { - removeTreeWithRetry(dir); - return; - } catch (err) { - const code = err && typeof err === "object" && "code" in err ? String((err as { code: unknown }).code) : ""; - if (code !== "EBUSY" && code !== "EPERM" && code !== "ENOTEMPTY") throw err; - await Bun.sleep(25 * (attempt + 1)); - } - } - removeTreeWithRetry(dir); -} - -async function startPoolRetryHarness( - reply: (accountId: string, request: Request) => Response | Promise, - options: { - secondAccount?: boolean; - streamMode?: "legacy-tee" | "eager-relay"; - accountMode?: "direct" | "pool"; - activeAccountId?: string; - accountNamespaces?: Record; - noVisionModels?: string[]; - visionSidecarModel?: string; - websockets?: boolean; - forwardApiKey?: string; - pausedAccountIds?: string[]; - reauthAccountIds?: string[]; - omitCredentialAccountIds?: string[]; - combos?: OcxConfig["combos"]; - modelRosterByAccount?: Record; - } = {}, -): Promise { - await removeTestDirBestEffort(TEST_DIR); - mkdirSync(TEST_DIR, { recursive: true }); - process.env.OPENCODEX_HOME = TEST_DIR; - clearCodexUpstreamHealth(); - clearThreadAccountMap(); - clearAccountQuota(); - resetCodexModelEntitlementCacheForTests(); - clearRequestLogsForTests(); - clearAccountNeedsReauth("pool-a"); - clearAccountNeedsReauth("pool-b"); - // The registry is process-global and survives a harness teardown. WS-REBIND-01 - // asserts exact per-account socket counts, so a socket leaked by any earlier test - // in this file shifts its snapshots and fails it in milliseconds — which reads as - // a flake next to the timeouts, but is ordinary shared state. Reset it with the - // rest rather than leaving one of six kinds of state uncleaned. - clearCodexWebSocketRegistry(); - - const dispatches: string[] = []; - const upstream = Bun.serve({ - port: 0, - async fetch(request) { - const accountId = request.headers.get("chatgpt-account-id") ?? "missing"; - if (new URL(request.url).pathname === "/models") { - return Response.json({ - models: (options.modelRosterByAccount?.[accountId] ?? []).map(slug => ({ - slug, - supported_in_api: true, - visibility: "list", - })), - }); - } - dispatches.push(accountId); - return reply(accountId, request); - }, - }); - redirectCanonicalCodexTo(upstream.url.toString()); - const redirectedFetch = globalThis.fetch; - - const secondAccount = options.secondAccount ?? true; - const config = { - port: 0, - defaultProvider: "openai", - openaiProviderTierVersion: 2, - providers: { - openai: { - ...canonicalDirect, - codexAccountMode: options.accountMode ?? "pool", - ...(options.noVisionModels ? { noVisionModels: options.noVisionModels } : {}), - ...(options.forwardApiKey ? { apiKey: options.forwardApiKey } : {}), - }, - }, - codexAccounts: [ - { id: "main", email: "main@example.test", isMain: true }, - { id: "pool-a", email: "pool-a@example.test", isMain: false, chatgptAccountId: "acct-pool-a" }, - ...(secondAccount - ? [{ id: "pool-b", email: "pool-b@example.test", isMain: false, chatgptAccountId: "acct-pool-b" }] - : []), - ], - activeCodexAccountId: options.activeAccountId ?? "pool-a", - ...(options.accountNamespaces ? { codexAccountNamespaces: options.accountNamespaces } : {}), - ...(options.pausedAccountIds ? { pausedCodexAccountIds: options.pausedAccountIds } : {}), - ...(options.visionSidecarModel ? { visionSidecar: { model: options.visionSidecarModel } } : {}), - ...(options.websockets ? { websockets: true } : {}), - ...(options.streamMode ? { streamMode: options.streamMode } : {}), - ...(options.combos ? { combos: options.combos } : {}), - } as OcxConfig; - saveConfig(config); - if (!options.omitCredentialAccountIds?.includes("pool-a")) { - saveCodexAccountCredential("pool-a", { - accessToken: "pool-a-token", - refreshToken: "pool-a-refresh", - expiresAt: Date.now() + 10 * 60_000, - chatgptAccountId: "acct-pool-a", - }); - } - updateAccountQuota("pool-a", 10); - if (secondAccount) { - if (!options.omitCredentialAccountIds?.includes("pool-b")) { - saveCodexAccountCredential("pool-b", { - accessToken: "pool-b-token", - refreshToken: "pool-b-refresh", - expiresAt: Date.now() + 10 * 60_000, - chatgptAccountId: "acct-pool-b", - }); - } - updateAccountQuota("pool-b", 20); - } - for (const accountId of options.reauthAccountIds ?? []) markAccountNeedsReauth(accountId); - - const server = startServer(0); - return { - config, - dispatches, - restoreFetch: () => { - if (globalThis.fetch === redirectedFetch) globalThis.fetch = originalGlobalFetch; - }, - server, - upstream, - request: ({ - stream = false, - signal, - model = POOL_RETRY_MODEL, - path = "/v1/responses", - callerBearer = true, - headers = {}, - extraBody = {}, - } = {}) => originalGlobalFetch(new URL(path, server.url), { - method: "POST", - headers: { - "content-type": "application/json", - ...(callerBearer ? { authorization: "Bearer inbound-token" } : {}), - ...headers, - }, - body: JSON.stringify({ model, input: path.endsWith("/compact") ? [] : "hello", stream, ...extraBody }), - signal, - }), - }; -} - -async function stopPoolRetryHarness(harness: PoolRetryHarness): Promise { - harness.restoreFetch(); - await harness.server.stop(true); - await harness.upstream.stop(true); -} - -function rejectionResponse(body: BodyInit, headers: Record = {}): Response { - return new Response(body, { - status: 400, - statusText: "Account Model Rejected", - headers: { "content-type": "application/json", "x-pool-retry-test": "original", ...headers }, - }); -} - -async function expectOriginal400(response: Response, body: string): Promise { - expect(response.status).toBe(400); - expect(response.headers.get("x-pool-retry-test")).toBe("original"); - expect(await response.text()).toBe(body); -} +const { startPoolRetryHarness, stopPoolRetryHarness, rejectionResponse, expectOriginal400, unsupportedModelBody } = + createPoolRetryHarness({ testDir: TEST_DIR, originalFetch: originalGlobalFetch, + redirectCanonicalCodexTo, canonicalDirect }); describe("Responses request identity handoff", () => { test("returns the generated request id and overwrites an upstream value", async () => { @@ -2571,6 +2377,59 @@ describe("server local API auth", () => { } }); + test("Astra error-envelope refusal recovers once and avoids the refused account next turn", async () => { + const model = "gpt-6-astra"; + const body = JSON.stringify({ error: { type: "invalid_request_error", code: "invalid_request_error", + message: `The '${model}' model is not supported when using Codex with a ChatGPT account.` } }); + const harness = await startPoolRetryHarness(accountId => accountId === "acct-pool-a" + ? rejectionResponse(body) + : Response.json({ id: "alternate-success", status: "completed", output: [] })); + try { + const first = await harness.request({ model }); + expect(first.status).toBe(200); + expect((await first.json() as { id: string }).id).toBe("alternate-success"); + expect(harness.dispatches).toEqual(["acct-pool-a", "acct-pool-b"]); + const next = await harness.request({ model }); + expect(next.status).toBe(200); + await next.text(); + expect(harness.dispatches).toEqual(["acct-pool-a", "acct-pool-b", "acct-pool-b"]); + expect(harness.config.activeCodexAccountId).toBe("pool-a"); + } finally { await stopPoolRetryHarness(harness); } + }); + + test("Astra error-envelope refusal can recover to the request-owned main account", async () => { + const model = "gpt-6-astra"; + const body = JSON.stringify({ error: { type: "invalid_request_error", + message: `The '${model}' model is not supported when using Codex with a ChatGPT account.` } }); + const seen: Array<{ account: string; authorization: string | null }> = []; + const harness = await startPoolRetryHarness((account, request) => { + seen.push({ account, authorization: request.headers.get("authorization") }); + return account === "acct-pool-a" ? rejectionResponse(body) + : Response.json({ id: "main-success", status: "completed", output: [] }); + }, { secondAccount: false }); + try { + const response = await harness.request({ model, headers: { "chatgpt-account-id": "acct-caller-main" } }); + expect(response.status).toBe(200); + expect((await response.json() as { id: string }).id).toBe("main-success"); + expect(seen).toEqual([ + { account: "acct-pool-a", authorization: "Bearer pool-a-token" }, + { account: "acct-caller-main", authorization: "Bearer inbound-token" }, + ]); + expect(loadConfig().activeCodexAccountId).toBe("pool-a"); + } finally { await stopPoolRetryHarness(harness); } + }); + + test("Astra error-envelope recovery still stops after one refused alternate", async () => { + const model = "gpt-6-astra"; + const body = JSON.stringify({ error: { type: "invalid_request_error", + message: `The '${model}' model is not supported when using Codex with a ChatGPT account.` } }); + const harness = await startPoolRetryHarness(() => rejectionResponse(body)); + try { + await expectOriginal400(await harness.request({ model }), body); + expect(harness.dispatches).toEqual(["acct-pool-a", "acct-pool-b"]); + } finally { await stopPoolRetryHarness(harness); } + }); + test("#2097: account-gated model selection skips an unentitled active Pool account", async () => { const model = "gpt-daybreak-blue-latest"; const harness = await startPoolRetryHarness( @@ -3555,29 +3414,42 @@ describe("server local API auth", () => { } }); - // Stall past BOUNDED_BODY_TIMEOUT_MS (5s). The old 7s test budget left ~1.9s of - // headroom and timed out on windows-latest under runner contention. + // Release only after the real inspector reports timeout. A producer's 5.1s + // clock can expire before a contended Windows consumer starts its 5s clock. test("stalled 400 body timeout never authorizes a pool retry", async () => { const prefix = unsupportedModelBody().slice(0, -1); const suffix = "}"; const body = prefix + suffix; + const releases: Array<() => void> = []; + let observedTimeout = false; const harness = await startPoolRetryHarness(() => rejectionResponse(new ReadableStream({ start(controller) { controller.enqueue(new TextEncoder().encode(prefix)); - setTimeout(() => { + releases.push(() => { controller.enqueue(new TextEncoder().encode(suffix)); controller.close(); - }, 5_100); + }); }, }))); + const inspectBody = boundedBody.readBoundedResponseBody; + const inspection = spyOn(boundedBody, "readBoundedResponseBody").mockImplementation(async (response, options) => { + const result = await inspectBody(response, options); + if (response.headers.get("x-pool-retry-test") === "original" && result.timedOut) { + observedTimeout = true; + for (const release of releases.splice(0)) release(); + } + return result; + }); try { const response = await harness.request(); expect(response.status).toBe(400); expect(response.headers.get("x-pool-retry-test")).toBe("original"); expect(await response.text()).toBe(body); expect(harness.dispatches).toEqual(["acct-pool-a"]); + expect(observedTimeout).toBe(true); } finally { - await stopPoolRetryHarness(harness); + try { for (const release of releases.splice(0)) release(); } + finally { inspection.mockRestore(); await stopPoolRetryHarness(harness); } } }, { timeout: SERVER_BUDGET_MS });