From 4bcbf5330d5fabb15d8834102cb1bbb4732a949c Mon Sep 17 00:00:00 2001 From: Ingwannu Date: Mon, 28 Sep 2026 16:48:25 +0000 Subject: [PATCH 01/26] feat(oauth): support Anthropic account pause and resume --- .../docs/reference/cli/providers-accounts.md | 11 +- .../content/docs/reference/management-api.md | 4 +- .../zh-tw/reference/cli/providers-accounts.md | 12 +- .../docs/zh-tw/reference/management-api.md | 4 +- .../provider-quota-refresh-controls.test.tsx | 8 +- scripts/test-layout/layout.json | 1 + .../ocx/references/01_management_surface.md | 6 +- src/cli/account-extended.ts | 3 +- src/cli/capabilities.ts | 6 +- src/oauth/anthropic-routing.ts | 45 ++++- src/oauth/index.ts | 6 +- src/oauth/store.ts | 4 +- src/server/management/oauth-account-routes.ts | 15 +- src/server/messages-native-oauth.ts | 2 +- src/server/messages-native.ts | 7 +- src/server/responses/request-transport.ts | 3 +- .../ADR-6013-anthropic-account-pause.md | 26 +++ structure/gui-and-management-api.md | 8 + structure/providers-and-adapters.md | 25 +++ .../anthropic/anthropic-account-pause.test.ts | 175 ++++++++++++++++++ .../anthropic/anthropic-model-routes.test.ts | 63 ++++++- .../messages-native-oauth.test.ts | 32 +++- tests/cli/cli-account-pool-verbs.test.ts | 31 ++++ tests/fixtures/test-layout-expected.json | 1 + tests/oauth/oauth-accounts-api.test.ts | 34 +++- 25 files changed, 483 insertions(+), 49 deletions(-) create mode 100644 structure/decisions/ADR-6013-anthropic-account-pause.md create mode 100644 tests/adapters/anthropic/anthropic-account-pause.test.ts diff --git a/docs-site/src/content/docs/reference/cli/providers-accounts.md b/docs-site/src/content/docs/reference/cli/providers-accounts.md index b8ce75d6486..02edd827e13 100644 --- a/docs-site/src/content/docs/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/reference/cli/providers-accounts.md @@ -469,16 +469,21 @@ Clear the manual Codex account selection without resolving an account id, so it ### `ocx account pause|resume [--json]` -Pause or resume one account in the Codex pool or a generic OAuth provider pool, including +Pause or resume one account in the Codex, Anthropic, or generic OAuth provider pool, including `google-antigravity`. For the Codex pool, `main` identifies only the built-in Codex account; -generic OAuth accounts must be identified by id or a unique alias. A paused generic OAuth account +OAuth accounts must be identified by id or a unique alias. A paused OAuth account is excluded from request selection, 429 failover, and proactive token refresh, and cannot be selected manually. Pausing the active account switches to the next usable account when one exists. If every account is paused, requests that need that pool return 403 until an account is resumed. -For a generic OAuth provider, identify the account by id or by a unique exact or case-insensitive +For Anthropic and generic OAuth providers, identify the account by id or by a unique exact or case-insensitive alias. The JSON response reports the account id, pause state, and active account id. +Anthropic pause applies even when proactive pooling is disabled, including session affinity and +429 successors. It survives restart and reauthentication, preserves credentials and health, +and does not interrupt a turn already sent. Removing the account removes its pause state. +Per-account Anthropic auto-switch thresholds are not part of this control. + ```bash ocx account pause google-antigravity ocx account resume google-antigravity diff --git a/docs-site/src/content/docs/reference/management-api.md b/docs-site/src/content/docs/reference/management-api.md index 5e2ea2b0602..3c60b3505db 100644 --- a/docs-site/src/content/docs/reference/management-api.md +++ b/docs-site/src/content/docs/reference/management-api.md @@ -557,10 +557,10 @@ outcome fields from an older server do not establish successful recovery. | `POST /api/oauth/login/cancel` | Cancel a public in-progress OAuth flow | 400 unknown provider | | `GET /api/oauth/status` | Poll one provider's OAuth flow | 400 unknown provider | | `POST /api/oauth/logout` | Remove the selected provider credential | 400 unknown provider; `oauth_mutation_busy` | -| `GET /api/oauth/accounts` | List masked accounts; generic OAuth account rows include their `paused` state. Kiro rows include `autoSelectable` and a closed `skipReason` when excluded from automatic selection; an active singleton may still send. Quota remains opt-in. | 400 invalid provider | +| `GET /api/oauth/accounts` | List masked accounts; Anthropic and generic OAuth account rows include their `paused` state. Kiro rows include `autoSelectable` and a closed `skipReason` when excluded from automatic selection; an active singleton may still send. Quota remains opt-in. | 400 invalid provider | | `DELETE /api/oauth/accounts` | Remove one account | 400 invalid provider/id; 404 account missing; `oauth_mutation_busy` | | `PUT /api/oauth/accounts/active` | Select the active OAuth account | 400 invalid provider/account; 404 account missing; 409 account paused; `oauth_mutation_busy` | -| `PUT /api/oauth/accounts/pause` | Pause or resume one generic OAuth account. Body `{ provider, accountId, paused }`; pausing the active account selects the next usable account when available | 400 unsupported provider or invalid body; 404 account missing; `oauth_mutation_busy` | +| `PUT /api/oauth/accounts/pause` | Pause or resume one Anthropic or generic OAuth account. Body `{ provider, accountId, paused }`; pausing the active account selects the next usable account when available. Pause is durable and independent of pool enablement; resume preserves health and credentials. | 400 unsupported provider or invalid body; 404 account missing; `oauth_mutation_busy` | | `GET, PUT, PATCH /api/pool/settings` | Read or update pool policy for any kind (codex, anthropic, generic); answers with the same keys for all three and declares in `supported` which the kind honours | 400 unknown provider, a field the kind does not support, or an invalid value | | `GET, PUT, PATCH /api/oauth/accounts/pool` | Legacy per-pool policy for Anthropic and generic OAuth providers; superseded by `/api/pool/settings` and kept for existing clients | 400 codex or api-key provider, or invalid policy | | `POST /api/oauth/accounts/clear-cooldown` | Clear one OAuth account's runtime cooldown | 400 invalid provider/account | diff --git a/docs-site/src/content/docs/zh-tw/reference/cli/providers-accounts.md b/docs-site/src/content/docs/zh-tw/reference/cli/providers-accounts.md index 141467062a6..6abdcc7e7bc 100644 --- a/docs-site/src/content/docs/zh-tw/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/zh-tw/reference/cli/providers-accounts.md @@ -144,15 +144,19 @@ Codex 池選擇套用於清除既有親和性後的下一個請求；進行中 ### `ocx account pause|resume [--json]` -暫停或恢復 Codex 帳號池或通用 OAuth 供應商池中的單一帳號，包括 -`google-antigravity`。在 Codex 池中，`main` 僅代表 Codex 內建帳號；通用 OAuth 帳號必須用 id 或唯一別名識別。 -已暫停的通用 OAuth 帳號不會參與請求選帳、429 輪替或主動 Token 刷新，也不能手動選取。 +暫停或恢復 Codex、Anthropic 或通用 OAuth 供應商池中的單一帳號，包括 +`google-antigravity`。在 Codex 池中，`main` 僅代表 Codex 內建帳號；OAuth 帳號必須用 id 或唯一別名識別。 +已暫停的 OAuth 帳號不會參與請求選帳、429 輪替或主動 Token 刷新，也不能手動選取。 若暫停目前使用中的帳號，系統會在有其他可用帳號時切換過去。若全部帳號都已暫停， 需要該池的請求會回覆 403，直到恢復其中一個帳號。 -通用 OAuth 供應商可用帳號 id，或唯一且完全相符／不區分大小寫的別名識別帳號。 +Anthropic 和通用 OAuth 供應商可用帳號 id，或唯一且完全相符／不區分大小寫的別名識別帳號。 JSON 回應會提供帳號 id、暫停狀態與目前 active 帳號 id。 +Anthropic 暫停不受帳號池啟用開關影響，包含工作階段綁定與 429 後繼選帳。 +重新啟動或登入仍保留暫停，憑證與健康狀態不會清除，已送出的請求不會中斷。 +刪除帳號會一併刪除暫停狀態；個別帳號的自動切換門檻不在此功能範圍內。 + ```bash ocx account pause google-antigravity ocx account resume google-antigravity diff --git a/docs-site/src/content/docs/zh-tw/reference/management-api.md b/docs-site/src/content/docs/zh-tw/reference/management-api.md index 9e3549dbb42..e987e1f8a72 100644 --- a/docs-site/src/content/docs/zh-tw/reference/management-api.md +++ b/docs-site/src/content/docs/zh-tw/reference/management-api.md @@ -233,10 +233,10 @@ Aside 設定檔的變更在這種情況下仍會儲存一件事：確認之後 | `POST /api/oauth/login/cancel` | 取消公開進行中的 OAuth 流程 | 400 未知供應商 | | `GET /api/oauth/status` | 輪詢一個供應商的 OAuth 流程 | 400 未知供應商 | | `POST /api/oauth/logout` | 移除所選的供應商憑證 | 400 未知供應商；`oauth_mutation_busy` | -| `GET /api/oauth/accounts` | 列出遮罩帳號；通用 OAuth 帳號列也會提供 `paused` 狀態。Kiro 列包含自動選取狀態 `autoSelectable`，排除時還包含封閉集合的 `skipReason`。唯一的有效帳號仍可傳送請求，配額查詢仍為選用。 | 400 無效供應商 | +| `GET /api/oauth/accounts` | 列出遮罩帳號；Anthropic 與通用 OAuth 帳號列也會提供 `paused` 狀態。Kiro 列包含自動選取狀態 `autoSelectable`，排除時還包含封閉集合的 `skipReason`。唯一的有效帳號仍可傳送請求，配額查詢仍為選用。 | 400 無效供應商 | | `DELETE /api/oauth/accounts` | 移除一個帳號 | 400 無效供應商/id；404 帳號缺失；`oauth_mutation_busy` | | `PUT /api/oauth/accounts/active` | 選擇現用 OAuth 帳號 | 400 無效供應商／帳號；404 帳號缺失；409 帳號已暫停；`oauth_mutation_busy` | -| `PUT /api/oauth/accounts/pause` | 暫停或恢復一個通用 OAuth 帳號。Body `{ provider, accountId, paused }`；若暫停現用帳號，且有可用帳號，會切換至下一個 | 400 不支援的供應商或無效 body；404 帳號缺失；`oauth_mutation_busy` | +| `PUT /api/oauth/accounts/pause` | 暫停或恢復一個 Anthropic 或通用 OAuth 帳號。Body `{ provider, accountId, paused }`；若暫停現用帳號，且有可用帳號，會切換至下一個。暫停會持久儲存且不受帳號池開關影響，恢復保留健康狀態與憑證。 | 400 不支援的供應商或無效 body；404 帳號缺失；`oauth_mutation_busy` | | `GET, PUT, PATCH /api/oauth/accounts/pool` | 讀取或更新 Anthropic OAuth 池政策 | 400 非 Anthropic 供應商或無效政策 | | `POST /api/oauth/accounts/clear-cooldown` | 清除一個 OAuth 帳號的 runtime 冷卻 | 400 無效供應商／帳號 | | `PUT /api/oauth/accounts/alias` | 設定或清除 OAuth 帳號別名 | 400 無效供應商／帳號／別名 | diff --git a/gui/tests/provider-quota-refresh-controls.test.tsx b/gui/tests/provider-quota-refresh-controls.test.tsx index 5766221a13f..2c1d868550a 100644 --- a/gui/tests/provider-quota-refresh-controls.test.tsx +++ b/gui/tests/provider-quota-refresh-controls.test.tsx @@ -172,12 +172,12 @@ test("the accounts surface omits the control when the page cannot force a read", expect(findButton("Refresh quotas")).toBeNull(); }); -test("generic OAuth accounts expose pause and resume controls", async () => { +test.each(["google-antigravity", "anthropic"])("%s OAuth accounts expose pause and resume controls", async provider => { const calls: Array<{ provider: string; accountId: string; paused: boolean }> = []; const handlers = authHandlers({ onPauseAccount: async (provider, row, paused) => { calls.push({ provider, accountId: row.id, paused }); }, }); - const item = { ...oauthItem, name: "google-antigravity" }; + const item = { ...oauthItem, name: provider }; await render( { const pause = findButton("Pause"); expect(pause).not.toBeNull(); await act(async () => { pause!.click(); }); - expect(calls).toEqual([{ provider: "google-antigravity", accountId: "ga-active", paused: true }]); + expect(calls).toEqual([{ provider, accountId: "ga-active", paused: true }]); await render( { expect(host.textContent).not.toContain(en["codexAuth.pausedHint"]); expect(en["pws.accountPausedHint"]).not.toBe(en["codexAuth.pausedHint"]); await act(async () => { resume!.click(); }); - expect(calls[1]).toEqual({ provider: "google-antigravity", accountId: "ga-active", paused: false }); + expect(calls[1]).toEqual({ provider, accountId: "ga-active", paused: false }); }); test("API-key rows use independent shared credit readings and the same awaited refresh control", async () => { diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 07142b90e3a..c8ff7e50269 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -99,6 +99,7 @@ "ambiguous-resend-composition.test.ts": "lib", "ambiguous-resend-gate.test.ts": "lib", "anthropic-account-pool.test.ts": "adapters/anthropic", + "anthropic-account-pause.test.ts": "adapters/anthropic", "anthropic-model-routes.test.ts": "adapters/anthropic", "anthropic-agentrouter-language-framing.test.ts": "adapters/anthropic", "anthropic-baseurl-override.test.ts": "adapters/anthropic", diff --git a/skills/ocx/references/01_management_surface.md b/skills/ocx/references/01_management_surface.md index 636a2b88e8e..beb9e651e92 100644 --- a/skills/ocx/references/01_management_surface.md +++ b/skills/ocx/references/01_management_surface.md @@ -803,7 +803,7 @@ JSON mode: `payload`. ### `ocx account pause` -Exclude one account in a Codex or supported generic OAuth pool from automatic selection. +Exclude one account in a Codex, Anthropic or supported generic OAuth pool from automatic selection. | Method | Route | |---|---| @@ -817,11 +817,11 @@ Exclude one account in a Codex or supported generic OAuth pool from automatic se JSON mode: `envelope`. -- Codex pause unbinds pinned threads and selects a fallback when possible; with no fallback, a paused-but-selected Codex account still receives requests. Generic OAuth pause never dispatches to that account: it is excluded from new requests, failover and refresh, and an all-paused pool answers 403. Anthropic is unsupported. +- Codex pause unbinds pinned threads and selects a fallback when possible; with no fallback, a paused-but-selected Codex account still receives requests. Anthropic and generic OAuth pause exclude the account from new requests, failover and refresh, and an all-paused pool answers 403. Credentials and health are preserved; already-sent turns are not cancelled. ### `ocx account resume` -Return a paused account to a Codex or supported generic OAuth pool. +Return a paused account to a Codex, Anthropic or supported generic OAuth pool. | Method | Route | |---|---| diff --git a/src/cli/account-extended.ts b/src/cli/account-extended.ts index 4fce819000d..5260b979a99 100644 --- a/src/cli/account-extended.ts +++ b/src/cli/account-extended.ts @@ -787,7 +787,7 @@ function resolveGenericOAuthPauseTarget(accounts: unknown[], requested: string): return { error: `Account not found: no OAuth account has the id or alias "${requested}"` }; } -/** Pause or resume a Codex account or a generic OAuth provider account. */ +/** Pause or resume a Codex or OAuth provider account, including Anthropic. */ export async function cmdPause(args: string[], deps: AccountDeps, paused: boolean): Promise { const wantsJson = flag(args, "--json"); const name = args.shift(); @@ -800,7 +800,6 @@ export async function cmdPause(args: string[], deps: AccountDeps, paused: boolea if (!baseUrl) return proxyUnreachable(); if (classified.type === "oauth") { - if (name === "anthropic") return usage(`Error: ${verb} is not supported for the Anthropic OAuth pool`); const list = await apiJson(deps, baseUrl, "GET", `/api/oauth/accounts?provider=${encodeURIComponent(name)}`); if (list.status === 0) return proxyUnreachable(list.transportError); if (list.status !== 200) return apiError(list.json, `failed to list ${name} OAuth accounts`, list.status); diff --git a/src/cli/capabilities.ts b/src/cli/capabilities.ts index 04d802e56ff..723e4b4f605 100644 --- a/src/cli/capabilities.ts +++ b/src/cli/capabilities.ts @@ -541,7 +541,7 @@ export const CAPABILITIES: readonly Capability[] = [ }, { command: ["account", "pause"], - summary: "Exclude one account in a Codex or supported generic OAuth pool from automatic selection.", + summary: "Exclude one account in a Codex, Anthropic or supported generic OAuth pool from automatic selection.", // Resume uses the same endpoints with `paused: false`. routes: [ { method: "PUT", path: "/api/codex-auth/accounts/pause" }, @@ -552,12 +552,12 @@ export const CAPABILITIES: readonly Capability[] = [ mutates: true, json: "envelope", details: [ - "Codex pause unbinds pinned threads and selects a fallback when possible; with no fallback, a paused-but-selected Codex account still receives requests. Generic OAuth pause never dispatches to that account: it is excluded from new requests, failover and refresh, and an all-paused pool answers 403. Anthropic is unsupported.", + "Codex pause unbinds pinned threads and selects a fallback when possible; with no fallback, a paused-but-selected Codex account still receives requests. Anthropic and generic OAuth pause exclude the account from new requests, failover and refresh, and an all-paused pool answers 403. Credentials and health are preserved; already-sent turns are not cancelled.", ], }, { command: ["account", "resume"], - summary: "Return a paused account to a Codex or supported generic OAuth pool.", + summary: "Return a paused account to a Codex, Anthropic or supported generic OAuth pool.", routes: [ { method: "PUT", path: "/api/codex-auth/accounts/pause" }, { method: "GET", path: "/api/oauth/accounts" }, diff --git a/src/oauth/anthropic-routing.ts b/src/oauth/anthropic-routing.ts index 36327c6d43d..fdcbb98929c 100644 --- a/src/oauth/anthropic-routing.ts +++ b/src/oauth/anthropic-routing.ts @@ -34,6 +34,7 @@ import type { OcxAccountPoolQuotaWindow, OcxAccountPoolRotationStrategy, OcxConf import { sweepExpiredOnWrite } from "../lib/state-store-sweeper"; import { retainedUtf8Bytes } from "../lib/admission"; import { routeCandidates, type AnthropicRouteDecision } from "./anthropic-model-routes"; +import { subscribeOAuthAccountPauseChanges } from "../lib/account-selection-events"; /** * The read side of a `Headers` object, so a caller can pass the live upstream response's @@ -266,7 +267,7 @@ export function getEligibleAnthropicAccounts(now = Date.now()): string[] { if (!set) return []; return set.accounts .filter(account => - account.needsReauth !== true + account.paused !== true && account.needsReauth !== true && !isCooled(account.id, now) && isPoolCredentialUsable(account.id, now)) .map(account => account.id); @@ -301,6 +302,8 @@ export function getEligibleAnthropicAccounts(now = Date.now()): string[] { const QUORUM_CACHE_TTL_MS = 2_000; let quorumCache: { value: boolean; readAt: number } | null = null; +// Pause changes eligibility, not health. Do not reset cooldowns or cancel sent turns. +subscribeOAuthAccountPauseChanges(provider => { if (provider === PROVIDER) quorumCache = null; }); /** * Whether a 429 has somewhere to go: two or more accounts that could serve traffic if asked. @@ -331,7 +334,7 @@ export function hasAnthropicFailoverQuorum(now = Date.now()): boolean { if (set) { let usable = 0; for (const account of set.accounts) { - if (account.needsReauth === true) continue; + if (account.paused === true || account.needsReauth === true) continue; if (!isPoolCredentialUsable(account.id, now)) continue; if (++usable >= 2) { value = true; break; } } @@ -355,6 +358,7 @@ export function getAnthropicPoolRetryAfterSeconds(now = Date.now(), decision: An && !getEligibleAnthropicAccounts(now).some(id => decision.accounts.includes(id)); let earliest: number | null = null; for (const account of set.accounts) { + if (account.paused === true) continue; if (decision && !fallbackExpanded && !decision.accounts.includes(account.id)) continue; if (fallbackExpanded && (account.needsReauth === true || !isPoolCredentialUsable(account.id, now))) continue; const snap = getAnthropicAccountHealthSnapshot(account.id, now); @@ -486,6 +490,7 @@ export type AnthropicAccountSelectionReason = | "manual" | "fill-first" | "none" + | "paused" | "all-cooled"; export interface AnthropicAccountSelection { @@ -568,7 +573,7 @@ function pickUnboundStrategyAccount( /** * Resolve which Anthropic OAuth account should serve this session. - * When the pool is disabled, always returns the store's active account. + * When the pool is disabled, retain the active account unless the operator paused it. */ export function resolveAnthropicAccountForSession( sessionKey: string | null | undefined, @@ -579,6 +584,11 @@ export function resolveAnthropicAccountForSession( pruneExpiredAffinity(now); const set = getAccountSet(PROVIDER); if (!set || set.accounts.length === 0) return { accountId: null, reason: "none", ...(decision ? { routePosition: decision.position } : {}) }; + const scoped = decision && !decision.fallback + ? set.accounts.filter(account => decision.accounts.includes(account.id)) : set.accounts; + if (scoped.length > 0 && scoped.every(account => account.paused === true)) { + return { accountId: null, reason: "paused", routePosition: decision?.position }; + } if (manualPreference === undefined) { manualPreference = set.selectionRevision !== undefined @@ -587,7 +597,9 @@ export function resolveAnthropicAccountForSession( } if (!isAnthropicAccountPoolEnabled(config)) { - return { accountId: set.activeAccountId, reason: "pool-disabled" }; + const active = set.accounts.find(account => account.id === set.activeAccountId); + // Disabled proactive rotation does not grant permission to use an operator-paused slot. + return { accountId: active?.paused ? getEligibleAnthropicAccounts(now)[0] ?? null : set.activeAccountId, reason: "pool-disabled" }; } const eligible = routeCandidates(getEligibleAnthropicAccounts(now), decision); @@ -597,7 +609,7 @@ export function resolveAnthropicAccountForSession( // A removed route member is not a candidate, but fallback can still use the // ordinary pool once those accounts recover. Only usable stored accounts count. const ordinary = set.accounts.filter(account => - account.needsReauth !== true && isPoolCredentialUsable(account.id, now)); + account.paused !== true && account.needsReauth !== true && isPoolCredentialUsable(account.id, now)); cooled = ordinary.length > 0 && ordinary.every(account => isCooled(account.id, now)); } else { cooled = decision.accounts.every(id => set.accounts.some(account => account.id === id && isCooled(id, now))); @@ -626,7 +638,7 @@ export function resolveAnthropicAccountForSession( if (key) { const affined = sessionAffinity.get(key); if (affined && now - affined.lastUsedAt <= AFFINITY_IDLE_TTL_MS) { - const stillThere = set.accounts.some(a => a.id === affined.accountId && a.needsReauth !== true); + const stillThere = set.accounts.some(a => a.id === affined.accountId && a.paused !== true && a.needsReauth !== true); const stillUsable = stillThere && !isCooled(affined.accountId, now) && isPoolCredentialUsable(affined.accountId, now); if (stillUsable && eligible.includes(affined.accountId)) { @@ -694,7 +706,7 @@ export function resolveAnthropicAccountForSession( } if (!accountId) { - const anyCooled = set.accounts.some(a => (!decision || decision.accounts.includes(a.id)) && isCooled(a.id, now)); + const anyCooled = set.accounts.some(a => !a.paused && (!decision || decision.accounts.includes(a.id)) && isCooled(a.id, now)); return { accountId: null, reason: anyCooled ? "all-cooled" : "none", routePosition: decision?.position }; } @@ -742,7 +754,10 @@ export function rotateAnthropicAccountOn429( // just rate-limited is a different thing: it only ever runs after a refusal, and stranding a // 429 while a second logged-in account sits idle is a defect, not a configuration choice. // Presence is the activation rule, the same one an apiKeyPool of two keys already uses. - if (!isAnthropicAccountPoolEnabled(config) && !hasAnthropicFailoverQuorum(now)) return null; + // A sent turn may finish after its account is paused; its one remaining successor is + // still a valid reactive recovery even though the current unpaused quorum is now one. + if (!isAnthropicAccountPoolEnabled(config) && !hasAnthropicFailoverQuorum(now) + && !getAccountCredentialWithStatus(PROVIDER, failedAccountId)?.paused) return null; // Retry-After first: it is the header written FOR this decision. The rejected window's // reset is the fallback, because a 429 that omits Retry-After still carries it -- and @@ -813,7 +828,7 @@ export function commitAnthropicSelectionRouting( committed: OAuthAccountSelection, options: AnthropicSelectionRoutingOptions, ): boolean { - if (committed.accountId !== accountId || (options.routeDecision + if (getAccountCredentialWithStatus(PROVIDER, accountId)?.paused || committed.accountId !== accountId || (options.routeDecision && !routeCandidates(getEligibleAnthropicAccounts(), options.routeDecision).includes(accountId))) return false; const current = captureOAuthAccountSelection(PROVIDER); if (current?.accountId !== committed.accountId || current.revision !== committed.revision) return false; @@ -857,7 +872,12 @@ export function resetAnthropicRoutingForManualSelection(accountId: string): void * as quota probes). */ export async function getAnthropicPoolAccessToken(accountId: string): Promise { - const stored = getAccountCredential(PROVIDER, accountId); + const row = getAccountCredentialWithStatus(PROVIDER, accountId); + if (row?.paused) { + const { OAuthAccountPausedError } = await import("./index"); + throw new OAuthAccountPausedError(); + } + const stored = row?.credential; if (!stored) { const { OAuthLoginRequiredError } = await import("./index"); throw new OAuthLoginRequiredError(PROVIDER); @@ -874,6 +894,10 @@ export async function getAnthropicPoolAccessToken(accountId: string): Promise { const accessToken = await getAnthropicPoolAccessToken(accountId); const row = getAccountCredentialWithStatus(PROVIDER, accountId); + if (row?.paused) { + const { OAuthAccountPausedError } = await import("./index"); + throw new OAuthAccountPausedError(); + } if (!row || row.needsReauth || row.credential.access !== accessToken || row.credential.expires <= Date.now()) { throw new Error("Anthropic pool credential changed during account selection"); @@ -887,6 +911,7 @@ export async function getAnthropicPoolAccessSnapshot(accountId: string): Promise */ export function canRefreshAnthropicPoolAccount(accountId: string): boolean { const set = getAccountSet(PROVIDER); + if (set?.accounts.find(account => account.id === accountId)?.paused) return false; const cred = getAccountCredential(PROVIDER, accountId); if (!cred) return false; if (cred.source !== "local-cli") return true; diff --git a/src/oauth/index.ts b/src/oauth/index.ts index db7e5b7c5ad..3d6a0efcf68 100644 --- a/src/oauth/index.ts +++ b/src/oauth/index.ts @@ -925,8 +925,10 @@ export async function refreshAnthropicAccountWithLock( const now = deps.now ?? Date.now; const guard = await (deps.intentLock ?? createOAuthRefreshIntentLock(provider, accountId)).acquire(); try { - const stored = getAccountCredential(provider, accountId); - if (!stored) throw new OAuthLoginRequiredError(provider); + const row = getAccountCredentialWithStatus(provider, accountId); + if (!row) throw new OAuthLoginRequiredError(provider); + if (row.paused) throw new OAuthAccountPausedError(); + const stored = row.credential; const account = getAccountSet(provider)?.accounts.find(candidate => candidate.id === accountId); const generation = credentialGeneration(stored); let pendingIntent = readOAuthRefreshIntent(provider, accountId); diff --git a/src/oauth/store.ts b/src/oauth/store.ts index 5c9fff453fe..7d085c96034 100644 --- a/src/oauth/store.ts +++ b/src/oauth/store.ts @@ -1380,4 +1380,6 @@ export async function markAccountNeedsReauth( } export async function mergeAccountCredential(provider:string,accountId:string,credential:OAuthCredentials,opts:{expectedGeneration?:string;afterPrePersistRead?:()=>void|Promise;assertOwnership?: (store: AuthStore) => void}={}):Promise<{superseded:false}|{superseded:true;stored:OAuthCredentials}>{const safe=normalizeCredential(credential);if(!safe)throw new Error("Refusing to persist invalid OAuth credential");return await mutateStore(async store=>{await opts.afterPrePersistRead?.();const account=store[provider]?.accounts.find(x=>x.id===accountId);if(!account)throw new Error(`OAuth account disappeared before persist: ${provider}`);if(opts.expectedGeneration!==undefined&&credentialGeneration(account.credential)!==opts.expectedGeneration)return{superseded:true,stored:account.credential};opts.assertOwnership?.(store);account.credential=safe;delete account.needsReauth;return{superseded:false};},[provider,accountId,safe,opts.expectedGeneration]);} -export async function markAccountNeedsReauthIfGeneration(provider:string,accountId:string,generation:string,writerGeneration=captureConfigGeneration()):Promise{const key=oauthAccountKey(provider,accountId);if(writerGeneration{const account=store[provider]?.accounts.find(x=>x.id===accountId);if(!account?.credential||credentialGeneration(account.credential)!==generation)return false;if(writerGeneration{const key=oauthAccountKey(provider,accountId);if(writerGeneration{const account=store[provider]?.accounts.find(x=>x.id===accountId);if(!account?.credential||account.paused||credentialGeneration(account.credential)!==generation)return false;if(writerGeneration Date.now() + && !!row && !row.paused && !row.needsReauth && row.credential.expires > Date.now() && credentialGeneration(row.credential) === binding.snapshot.generation; } diff --git a/src/server/messages-native.ts b/src/server/messages-native.ts index 5a88dc50aa3..4bbaeed84a3 100644 --- a/src/server/messages-native.ts +++ b/src/server/messages-native.ts @@ -59,7 +59,7 @@ import { transientRetryPolicyFor, } from "../providers/key-failover"; import { stampApiKeyAccountLabel, stampOAuthAccountLabel } from "../providers/label"; -import { publicOAuthAuthenticationErrorMessage } from "../oauth"; +import { OAuthAccountPausedError, publicOAuthAuthenticationErrorMessage } from "../oauth"; import { hasAnthropicFailoverQuorum } from "../oauth/anthropic-routing"; import { resolveProtocolSettings } from "../protocols/settings"; import { addProtocolEntryReason, markProtocolBlocked } from "../protocols/trace"; @@ -367,6 +367,7 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) upstream.abort(); if (req.signal.aborted) return fail(499, "Client cancelled request", "api_error"); if (error instanceof NativeOAuthSelectionChangedError) return fail(409, error.message, "api_error"); + if (error instanceof OAuthAccountPausedError) return fail(403, publicOAuthAuthenticationErrorMessage(error), "permission_error"); return fail(401, publicOAuthAuthenticationErrorMessage(error), "authentication_error"); } } else { @@ -463,7 +464,8 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) if (!nativeOAuthBindingIsCurrent(oauthBinding)) { try { oauthBinding = await resolveNativeOAuthBinding(config); - } catch { + } catch (error) { + if (error instanceof OAuthAccountPausedError) throw error; throw new NativeOAuthSelectionChangedError(); } rebuildFor(oauthProvider(oauthBinding)); @@ -570,6 +572,7 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) return refusal; } if (sendError instanceof NativeOAuthSelectionChangedError) return fail(409, sendError.message, "api_error"); + if (sendError instanceof OAuthAccountPausedError) return fail(403, publicOAuthAuthenticationErrorMessage(sendError), "permission_error"); if (sendError instanceof NativeOpaqueStateRefusal) { logCtx.errorCode = "unsupported_feature"; return fail(400, sendError.message, "invalid_request_error", "unsupported_feature"); diff --git a/src/server/responses/request-transport.ts b/src/server/responses/request-transport.ts index 206a923963d..f2262d4fa96 100644 --- a/src/server/responses/request-transport.ts +++ b/src/server/responses/request-transport.ts @@ -337,7 +337,7 @@ export async function prepareResponsesTransport( const selected = captureOAuthAccountSelection(route.providerName); const row = getAccountCredentialWithStatus(route.providerName, binding.snapshot.accountId); return selected?.accountId === binding.selection.accountId && selected?.revision === binding.selection.revision - && !!row && !row.needsReauth && row.credential.expires > Date.now() + && !!row && !row.paused && !row.needsReauth && row.credential.expires > Date.now() && credentialGeneration(row.credential) === binding.snapshot.generation; }; const resolveSelectionAdapter = (provider: OcxProviderConfig, retention = config.cacheRetention): ProviderAdapter => { @@ -561,6 +561,7 @@ export async function prepareResponsesTransport( anthropicRouteDecision = routeResult.decision; const selection = resolveAnthropicAccountForSession(anthropicSessionKey, config, Date.now(), anthropicRouteDecision); if (!selection.accountId) { + if (selection.reason === "paused") return formatErrorResponse(403, "permission_error", "Anthropic OAuth accounts are paused. Resume an account in account settings and retry."); // Route names may resemble account IDs; log only the matched rule position. if (anthropicRouteDecision) console.warn(`[anthropic-pool] route:#${anthropicRouteDecision.position} ${selection.reason}; answering locally`); if (selection.reason === "all-cooled") { diff --git a/structure/decisions/ADR-6013-anthropic-account-pause.md b/structure/decisions/ADR-6013-anthropic-account-pause.md new file mode 100644 index 00000000000..6b01aaf5b1b --- /dev/null +++ b/structure/decisions/ADR-6013-anthropic-account-pause.md @@ -0,0 +1,26 @@ +# ADR-6013 — decision recorded under "Anthropic account pause" + +- Contract owner: [Providers and adapters](../providers-and-adapters.md#anthropic-account-pause) + +## Decision Log + +- Purpose and intent: Temporarily exclude one Claude OAuth account through the API, CLI and + dashboard without deleting its login or conflating operator intent with runtime health. +- Existing implementation and constraints: Anthropic has its own manual/affinity/quota/model + route selector; generic OAuth pause already owns persistence, DTOs and dashboard controls. + Request preparation and refresh can await while the operator changes the selection. +- Alternatives considered: Add `pausedAnthropicAccountIds` to config, implement a second + mutation endpoint, or merely expose the generic pause switch without selector changes. +- Selected approach: Reuse the protected auth store's optional `paused: true`, shared locked + mutation, endpoint and translated controls. Filter Anthropic eligibility at each selector + and validate live pause state at credential, commit and physical-send boundaries. Persist + successful in-flight refresh rotation, but leave paused health unchanged on late failure. +- Why this approach: One authoritative row avoids config/auth split-write races and automatic + cleanup follows account deletion. A UI-only toggle would still allow affinity, pool-off + failover or a pre-wait bearer to select the paused account. +- Benefits, tradeoffs, and impact: No new config migration or translation keys; restart and + reauthentication preserve pause. Already-sent turns finish normally. Additional store reads + are limited to existing admission boundaries, and all-paused pools fail locally with 403. + Per-account thresholds remain a separate slice of #6013. This is the TypeScript `dev` + implementation; the maintainer must assess/port the corresponding Go paths on `dev2-go` + at integration, rather than treating this local commit as cross-branch completion. diff --git a/structure/gui-and-management-api.md b/structure/gui-and-management-api.md index 205de32668e..b96f866d664 100644 --- a/structure/gui-and-management-api.md +++ b/structure/gui-and-management-api.md @@ -1,5 +1,13 @@ # GUI And Management API +Anthropic account rows now expose the shared boolean `paused` DTO and use the existing +`PUT /api/oauth/accounts/pause` body `{ provider, accountId, paused }`. The dashboard's +`ProviderAuthPanel` and `useProviderAccountPools` reuse the translated pause/resume actions, +disabled manual selection, and separate mutation-versus-refresh failure notices. CLI +`ocx account pause|resume anthropic [--json]` uses that same endpoint. +Only OAuth routes support this operation; API-key routes remain outside this switch. +Selection and persistence semantics: [Anthropic account pause](providers-and-adapters.md#anthropic-account-pause). + `src/server/management/oauth-account-routes.ts` exposes Anthropic `routes` through both unified `/api/pool/settings` and legacy `/api/oauth/accounts/pool`. Omitted rules survive other setting writes, `null` clears them, and other pool kinds reject supplied rules. The unified DTO declares `routes` supported only for Anthropic and reports null otherwise. Both Anthropic settings GETs validate saved rules before projection: malformed hand edits yield `routes: null` plus `routesError` without changing the stored value; valid and absent rules omit that diagnostic. Config and management responses retain route names; request logs use only the rule’s 1-based `route:#` position. `src/cli/account-extended.ts` reads, replaces and clears these rules with `ocx account routes anthropic`; the server validates content. The provider management API validates `modelContextTiers` as a strict per-model map, diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index c55750f8314..a508f9bd202 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -1,5 +1,30 @@ # Providers And Adapters +## Anthropic account pause + +Anthropic OAuth shares `ProviderAccount.paused` in the protected auth store with generic +OAuth, not a second list in provider config. `setAccountPaused` serializes pause/resume +with credential and selection writes, advances the selection revision, and only publishes +invalidation after persistence. Removing an account removes its pause; reauthentication +preserves it. The store moves active selection to an unpaused, non-reauth row if available. +Pause does not clear cooldowns, quota, or credentials and does not cancel an already-sent turn. + +`src/oauth/anthropic-routing.ts` excludes paused rows from quota, round-robin, fill-first, +manual, affinity, model-route and reactive 429 candidates, including when proactive pooling +is off. All-paused requests return 403 with resume guidance. Quorum is invalidated on pause +and resume; a sent account paused before its 429 may still recover on its sole unpaused +successor. Credential resolution, refresh-lock acquisition, selection commit and physical +dispatch recheck live eligibility after asynchronous waits. An already-dispatched refresh +retains a successful rotated credential without unpausing; a late failure cannot mark the +paused row for reauthentication. Token Guardian and quota probes use their existing pause +guards. + +> Decision record: [ADR-6013](decisions/ADR-6013-anthropic-account-pause.md) + +Regression coverage: `tests/adapters/anthropic/anthropic-account-pause.test.ts`, +`tests/adapters/anthropic/anthropic-model-routes.test.ts`, `tests/oauth/oauth-accounts-api.test.ts`, +`tests/cli/cli-account-pool-verbs.test.ts`, and `gui/tests/provider-quota-refresh-controls.test.tsx`. + For Anthropic OAuth, `src/oauth/anthropic-routing.ts` applies the first matching `anthropicAccountPool.routes` rule to every eligible pick. The declared account order is stable while its candidates remain eligible; active, manual, affinity, quota and strategy preferences only choose inside that set. A healthy session affinity outside a model route is ignored for that request and retained for later unrouted or differently routed models; the routed commit does not overwrite it. An explicit fallback widens an empty route to the ordinary pool, and fill-first then advances in ordinary pool order from the active account. A missing eligible route fails locally without that fallback. The rules are operator allowlists, not provider entitlement evidence. Request logs use `route:#` for the 1-based rule position, not the operator name. GitHub Copilot `modelContextTiers` is selected per upstream model. The Chat and Responses diff --git a/tests/adapters/anthropic/anthropic-account-pause.test.ts b/tests/adapters/anthropic/anthropic-account-pause.test.ts new file mode 100644 index 00000000000..631ecb15e62 --- /dev/null +++ b/tests/adapters/anthropic/anthropic-account-pause.test.ts @@ -0,0 +1,175 @@ +import { afterEach, beforeEach, expect, spyOn, test } from "bun:test"; +import { mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { clearPoolRotationState } from "../../../src/codex/pool-rotation"; +import { OAuthAccountPausedError, OAuthLoginRequiredError, OAUTH_PROVIDERS, refreshAnthropicAccountWithLock } from "../../../src/oauth"; +import { AnthropicTokenError } from "../../../src/oauth/anthropic"; +import { + bindAnthropicSessionAffinity, clearAnthropicAccountPoolState, getAnthropicAccountHealthSnapshot, + getAnthropicPoolAccessSnapshot, getEligibleAnthropicAccounts, hasAnthropicFailoverQuorum, + promoteAnthropicActiveAccount, resolveAnthropicAccountForSession, rotateAnthropicAccountOn429, +} from "../../../src/oauth/anthropic-routing"; +import { + captureOAuthAccountSelection, createOAuthRefreshIntentLock, getAccountCredential, getAccountSet, + removeAccount, saveAccountCredential, saveCredential, setAccountPaused, setActiveAccount, +} from "../../../src/oauth/store"; +import { clearAccountQuotaCache, setCachedProviderAccountQuotaForTests } from "../../../src/providers/quota"; +import type { OcxAccountPoolQuotaWindow, OcxAccountPoolRotationStrategy, OcxConfig } from "../../../src/types"; +import { removeTreeWithRetry } from "../../helpers/remove-tree"; + +const previousHome = process.env.OPENCODEX_HOME; +let home: string; +let ids: string[]; +beforeEach(async () => { + home = mkdtempSync(join(tmpdir(), "ocx-anthropic-pause-")); + process.env.OPENCODEX_HOME = home; + clearAnthropicAccountPoolState(); clearPoolRotationState(); clearAccountQuotaCache(); + for (let i = 0; i < 3; i++) await saveCredential("anthropic", { + access: `synthetic-access-${i}`, refresh: `synthetic-refresh-${i}`, + expires: Date.now() + 3_600_000, accountId: `pause-${i}`, + }); + ids = getAccountSet("anthropic")!.accounts.map(account => account.id); + await setActiveAccount("anthropic", ids[0]!); +}); +afterEach(() => { + clearAnthropicAccountPoolState(); clearPoolRotationState(); clearAccountQuotaCache(); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + removeTreeWithRetry(home); +}); +function config(enabled = true, strategy: OcxAccountPoolRotationStrategy = "quota", quotaWindow: OcxAccountPoolQuotaWindow = "five-hour"): OcxConfig { + return { port: 0, defaultProvider: "anthropic", providers: { + anthropic: { adapter: "anthropic", baseUrl: "https://api.anthropic.com", authMode: "oauth" }, + }, anthropicAccountPool: { enabled, strategy, quotaWindow } }; +} + +for (const strategy of ["quota", "round-robin", "fill-first"] as const) { + for (const window of ["five-hour", "weekly", "max-utilization"] as const) { + test(`${strategy}/${window}: paused active, manual and affined accounts cannot win`, async () => { + const [a, b, c] = ids as [string, string, string]; + const cfg = config(true, strategy, window); + resolveAnthropicAccountForSession("session", cfg); // Cache the initial manual preference. + bindAnthropicSessionAffinity("session", a); + setCachedProviderAccountQuotaForTests("anthropic", a, { fiveHourPercent: 0, weeklyPercent: 0, updatedAt: Date.now() }); + await setAccountPaused("anthropic", a, true); + await setAccountPaused("anthropic", b, true); + expect(getEligibleAnthropicAccounts()).toEqual([c]); + expect(resolveAnthropicAccountForSession("session", cfg).accountId).toBe(c); + expect(resolveAnthropicAccountForSession(null, cfg).accountId).toBe(c); + expect(rotateAnthropicAccountOn429(cfg, b, "60")).toBe(c); + }); + } +} + +test("pause invalidates quorum immediately; disabled-pool reactive failover skips paused successors", async () => { + const [a, b, c] = ids as [string, string, string]; + expect(hasAnthropicFailoverQuorum()).toBe(true); + await setAccountPaused("anthropic", b, true); + expect(rotateAnthropicAccountOn429(config(false), a, "60")).toBe(c); + await setAccountPaused("anthropic", a, true); + expect(hasAnthropicFailoverQuorum()).toBe(false); + // The turn on A was already sent when it was paused; B stays excluded, C can recover it. + expect(rotateAnthropicAccountOn429(config(false), a, "60")).toBe(c); + await setAccountPaused("anthropic", b, false); + expect(hasAnthropicFailoverQuorum()).toBe(true); +}); + +test("all-paused refusal and resume preserve credentials and cooldown, independent of pool enable", async () => { + const a = ids[0]!; + const credential = getAccountCredential("anthropic", a); + rotateAnthropicAccountOn429(config(), a, "60"); + const health = getAnthropicAccountHealthSnapshot(a); + for (const id of ids) await setAccountPaused("anthropic", id, true); + for (const enabled of [true, false]) expect(resolveAnthropicAccountForSession("old", config(enabled))).toMatchObject({ accountId: null, reason: "paused" }); + await expect(getAnthropicPoolAccessSnapshot(a)).rejects.toBeInstanceOf(OAuthAccountPausedError); + expect(await setActiveAccount("anthropic", a)).toBe(false); + await setAccountPaused("anthropic", a, false); + expect(getAccountCredential("anthropic", a)).toEqual(credential); + expect(getAnthropicAccountHealthSnapshot(a)).toEqual(health); + expect(getAccountSet("anthropic")!.activeAccountId).toBe(a); +}); + +test("model routes cannot restore a paused account, but explicit fallback can widen", async () => { + const [a, b, c] = ids as [string, string, string]; + await setAccountPaused("anthropic", a, true); + await setAccountPaused("anthropic", b, true); + const route = { position: 1, accounts: [a, b], fallback: false }; + expect(resolveAnthropicAccountForSession("", config(), Date.now(), route).reason).toBe("paused"); + expect(resolveAnthropicAccountForSession("", config(), Date.now(), { ...route, fallback: true }).accountId).toBe(c); +}); + +test("pause and pause-resume ABA both invalidate an earlier credential proposal", async () => { + const a = ids[0]!; + const selection = captureOAuthAccountSelection("anthropic"); + const snapshot = await getAnthropicPoolAccessSnapshot(a); + await setAccountPaused("anthropic", a, true); + expect(await promoteAnthropicActiveAccount(a, selection, { config: config(), expectedCredentialGeneration: snapshot.generation })).toBeNull(); + await setAccountPaused("anthropic", a, false); + // Resume preserves the credential but cannot authorize a proposal captured before pause. + expect(await promoteAnthropicActiveAccount(a, selection, { config: config() })).toBeNull(); + expect(await promoteAnthropicActiveAccount(a, captureOAuthAccountSelection("anthropic"), { config: config() })).not.toBeNull(); +}); + +test("pause during an asynchronous credential refresh prevents returning its bearer", async () => { + const a = ids[0]!; + const credential = getAccountCredential("anthropic", a)!; + await saveAccountCredential("anthropic", a, { ...credential, expires: 0 }); + const refresh = spyOn(OAUTH_PROVIDERS.anthropic!, "refresh").mockImplementation(async () => { + await setAccountPaused("anthropic", a, true); + return { ...credential, access: "synthetic-after-wait", refresh: "synthetic-after-wait-refresh" }; + }); + try { + await expect(getAnthropicPoolAccessSnapshot(a)).rejects.toBeInstanceOf(OAuthAccountPausedError); + expect(refresh).toHaveBeenCalledTimes(1); + // Keep an already-rotated refresh token for resume without authorizing this request. + expect(getAccountCredential("anthropic", a)?.access).toBe("synthetic-after-wait"); + } finally { refresh.mockRestore(); } +}); + +test("pause is durable, idempotent, preserved on relogin and removed with its account", async () => { + const a = ids[0]!; + const credential = getAccountCredential("anthropic", a)!; + await Promise.all([setAccountPaused("anthropic", a, true), setAccountPaused("anthropic", ids[1]!, true)]); + expect((await setAccountPaused("anthropic", a, true)).status).toBe("unchanged"); + clearAnthropicAccountPoolState(); + expect(JSON.parse(readFileSync(join(home, "auth.json"), "utf8")).anthropic.accounts.filter((row: { paused?: boolean }) => row.paused)).toHaveLength(2); + await saveCredential("anthropic", credential); + expect(getAccountSet("anthropic")!.accounts.find(row => row.id === a)?.paused).toBe(true); + await removeAccount("anthropic", a); + await saveCredential("anthropic", credential); + expect(getAccountSet("anthropic")!.accounts.find(row => row.credential.accountId === credential.accountId)?.paused).toBeUndefined(); +}); + +test("refresh waiting for its lock observes a newly persisted pause without upstream use", async () => { + const a = ids[0]!; + const lock = createOAuthRefreshIntentLock("anthropic", a); + const originalAcquire = lock.acquire.bind(lock); + lock.acquire = async () => { await setAccountPaused("anthropic", a, true); return originalAcquire(); }; + let sends = 0; + await expect(refreshAnthropicAccountWithLock("anthropic", a, { + ...OAUTH_PROVIDERS.anthropic!, refresh: async () => { sends++; throw new Error("unexpected refresh"); }, + }, getAccountCredential("anthropic", a)!, { intentLock: lock })).rejects.toBeInstanceOf(OAuthAccountPausedError); + expect(sends).toBe(0); +}); + +test("refresh finishing after pause preserves rotated credentials; late failure preserves health", async () => { + const a = ids[0]!; + const original = getAccountCredential("anthropic", a)!; + await refreshAnthropicAccountWithLock("anthropic", a, { + ...OAUTH_PROVIDERS.anthropic!, refresh: async () => { + await setAccountPaused("anthropic", a, true); + return { ...original, access: "synthetic-rotated", refresh: "synthetic-rotated-refresh" }; + }, + }, original); + expect(getAccountCredential("anthropic", a)?.access).toBe("synthetic-rotated"); + expect(getAccountSet("anthropic")!.accounts.find(row => row.id === a)?.paused).toBe(true); + await setAccountPaused("anthropic", a, false); + await expect(refreshAnthropicAccountWithLock("anthropic", a, { + ...OAUTH_PROVIDERS.anthropic!, refresh: async () => { + await setAccountPaused("anthropic", a, true); + throw new AnthropicTokenError("synthetic rejection", 400, "invalid_grant"); + }, + }, getAccountCredential("anthropic", a)!)).rejects.toBeInstanceOf(OAuthLoginRequiredError); + expect(getAccountSet("anthropic")!.accounts.find(row => row.id === a)?.needsReauth).not.toBe(true); +}); diff --git a/tests/adapters/anthropic/anthropic-model-routes.test.ts b/tests/adapters/anthropic/anthropic-model-routes.test.ts index 9daa0d7c316..76b22368cc1 100644 --- a/tests/adapters/anthropic/anthropic-model-routes.test.ts +++ b/tests/adapters/anthropic/anthropic-model-routes.test.ts @@ -6,7 +6,8 @@ import { acquireOwnedSpendHome } from "../../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../../helpers/remove-tree"; import { clearAnthropicAccountPoolState, bindAnthropicSessionAffinity, getAnthropicPoolAccessSnapshot, getAnthropicPoolRetryAfterSeconds, promoteAnthropicActiveAccount, resolveAnthropicAccountForSession, rotateAnthropicAccountOn429 } from "../../../src/oauth/anthropic-routing"; import { parseAnthropicModelRoutes, resolveAnthropicModelRoute } from "../../../src/oauth/anthropic-model-routes"; -import { captureOAuthAccountSelection, getAccountSet, saveCredential, setActiveAccount } from "../../../src/oauth/store"; +import { captureOAuthAccountSelection, getAccountSet, saveCredential, setAccountPaused, setActiveAccount } from "../../../src/oauth/store"; +import { providerRequestPacingStatus, resetProviderRequestPacingForTest, waitForProviderRequestSlot } from "../../../src/providers/request-pacing"; import { clearAccountQuotaCache, setCachedProviderAccountQuotaForTests } from "../../../src/providers/quota"; import { clearResponseStateForTests } from "../../../src/responses/state"; import { handleResponses } from "../../../src/server/responses"; @@ -30,6 +31,7 @@ beforeEach(() => { clearResponseStateForTests(); }); afterEach(() => { + resetProviderRequestPacingForTest(); releaseSpend(); clearAnthropicAccountPoolState(); clearAccountQuotaCache(); @@ -49,7 +51,7 @@ async function seed(): Promise { await setActiveAccount("anthropic", ids[0]!); return ids; } -function config(ids: string[], reply: (token: string) => Response): OcxConfig { +function config(ids: string[], reply: (token: string) => Response | Promise): OcxConfig { const fetcher = (async (_url, init) => { const token = new Headers(init?.headers).get("authorization") ?? new Headers(init?.headers).get("x-api-key") ?? ""; sends.push(token); @@ -87,6 +89,63 @@ test("bounded first-match globs and invalid rules", () => { expect(parseAnthropicModelRoutes([{ name: "a", match: "[bad]", accounts: ["a"] }]).ok).toBe(false); }); +test.each([true, false])("all-paused pool returns 403 without any send (enabled=%s)", async enabled => { + const ids = await seed(); + for (const id of ids) await setAccountPaused("anthropic", id, true); + const cfg = config(ids, () => answer()); + cfg.anthropicAccountPool!.enabled = enabled; + const response = await post(cfg); + expect(response.status).toBe(403); + expect(await response.text()).toContain("Resume"); + expect(sends).toEqual([]); +}); + +test("a paused route successor is skipped on disabled-pool 429 failover", async () => { + const ids = await seed(); + await setAccountPaused("anthropic", ids[1]!, true); + const cfg = config(ids, async token => { + if (token.includes("synthetic-access-0")) { + await setAccountPaused("anthropic", ids[0]!, true); + return Response.json({ error: { type: "rate_limit_error", message: "synthetic refusal" } }, { status: 429, headers: { "retry-after": "60" } }); + } + return answer(); + }); + cfg.anthropicAccountPool!.enabled = false; + expect((await post(cfg)).status).toBe(200); + expect(sends).toHaveLength(2); + expect(sends[0]).toContain("synthetic-access-0"); + expect(sends[1]).toContain("synthetic-access-2"); +}); + +test("pausing an already-sent successful turn does not cancel its result", async () => { + const ids = await seed(); + const cfg = config(ids, async () => { + for (const id of ids) await setAccountPaused("anthropic", id, true); + return answer(); + }); + expect((await post(cfg)).status).toBe(200); + expect(sends).toHaveLength(1); +}); + +test.each([true, false])("pause while queued for pacing never sends the cached bearer (enabled=%s)", async enabled => { + const ids = await seed(); + const cfg = config(ids, () => answer()); + cfg.anthropicAccountPool = { enabled }; + cfg.providers.anthropic!.requestPacing = { enabled: true, maxConcurrentRequests: 1 }; + const slot = await waitForProviderRequestSlot("anthropic", cfg.providers.anthropic!, "claude-sonnet-4-5"); + const pending = post(cfg); + try { + for (let i = 0; i < 100 && providerRequestPacingStatus("anthropic", cfg.providers.anthropic!).queued === 0; i++) { + await new Promise(resolve => setTimeout(resolve, 5)); + } + expect(providerRequestPacingStatus("anthropic", cfg.providers.anthropic!).queued).toBe(1); + await setAccountPaused("anthropic", ids[0]!, true); + } finally { slot.release(); } + expect((await pending).status).toBe(200); + expect(sends).toHaveLength(1); + expect(sends[0]).not.toContain("synthetic-access-0"); +}); + test("matched route excludes active outsider before an upstream send", async () => { const ids = await seed(); const cfg = config(ids, () => answer()); diff --git a/tests/claude-integration/messages-native-oauth.test.ts b/tests/claude-integration/messages-native-oauth.test.ts index 5d86a9001bf..948ffc82a12 100644 --- a/tests/claude-integration/messages-native-oauth.test.ts +++ b/tests/claude-integration/messages-native-oauth.test.ts @@ -13,9 +13,10 @@ import { join } from "node:path"; import { saveConfig } from "../../src/config"; import { ANTHROPIC_OAUTH_BETA, CLAUDE_CODE_SYSTEM_INSTRUCTION } from "../../src/oauth/anthropic"; import { clearAnthropicAccountPoolState, forgetAnthropicFailoverQuorum } from "../../src/oauth/anthropic-routing"; -import { getAccountSet, markAccountNeedsReauth, saveCredential, setActiveAccount } from "../../src/oauth/store"; +import { getAccountSet, markAccountNeedsReauth, saveCredential, setAccountPaused, setActiveAccount } from "../../src/oauth/store"; import { handleClaudeMessages } from "../../src/server/claude-messages"; import { getRequestLogEntries } from "../../src/server/request-log"; +import { providerRequestPacingStatus, resetProviderRequestPacingForTest, waitForProviderRequestSlot } from "../../src/providers/request-pacing"; import type { OcxConfig, OcxProviderConfig } from "../../src/types"; import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; @@ -54,6 +55,7 @@ beforeEach(() => { }); afterEach(() => { + resetProviderRequestPacingForTest(); releaseSpendHome?.(); releaseSpendHome = undefined; try { @@ -169,6 +171,34 @@ async function send(config: OcxConfig, body: Record) { } describe("managed native Messages over Anthropic OAuth", () => { + test("a singleton paused during native pacing returns 403 without dispatch", async () => { + const [id] = await seed(1); + const cfg = fixtureConfig(); + cfg.providers.anthropic!.requestPacing = { enabled: true, maxConcurrentRequests: 1 }; + const slot = await waitForProviderRequestSlot("anthropic", cfg.providers.anthropic!, "claude-sonnet-4-5"); + const pending = send(cfg, { ...BODY, stream: false }); + try { + for (let i = 0; i < 100 && providerRequestPacingStatus("anthropic", cfg.providers.anthropic!).queued === 0; i++) { + await new Promise(resolve => setTimeout(resolve, 5)); + } + expect(providerRequestPacingStatus("anthropic", cfg.providers.anthropic!).queued).toBe(1); + await setAccountPaused("anthropic", id!, true); + } finally { slot.release(); } + const { response, text } = await pending; + expect(response.status).toBe(403); + expect(text).toContain("Resume"); + expect(sent).toEqual([]); + }); + + test("an operator-paused singleton returns 403 and never sends", async () => { + const [id] = await seed(1); + await setAccountPaused("anthropic", id!, true); + const { response, text } = await send(fixtureConfig(), { ...BODY, stream: false }); + expect(response.status).toBe(403); + expect(text).toContain("Resume"); + expect(sent).toEqual([]); + }); + test("sends with the selected account's token and the OAuth shape, never the caller's credential", async () => { await seed(1); const { response, text, row } = await send(fixtureConfig(), { ...BODY, stream: true }); diff --git a/tests/cli/cli-account-pool-verbs.test.ts b/tests/cli/cli-account-pool-verbs.test.ts index 7d7542e6b92..a6b2613fbcf 100644 --- a/tests/cli/cli-account-pool-verbs.test.ts +++ b/tests/cli/cli-account-pool-verbs.test.ts @@ -57,6 +57,37 @@ function capture(): { lines: string[]; errors: string[]; restore: () => void } { } describe("ocx account pause / resume", () => { + test("Anthropic ambiguous aliases fail without a write; exact ids still win", async () => { + const out = capture(); + const writes: unknown[] = []; + const d: AccountDeps = { + baseUrl: "http://127.0.0.1:10100", + loadConfigImpl: () => ({ providers: { anthropic: { adapter: "anthropic", authMode: "oauth" } } }) as never, + fetchImpl: (async (_url, init) => { + if (init?.method === "PUT") { writes.push(JSON.parse(String(init.body))); return Response.json({ ok: true }); } + return Response.json({ accounts: [{ id: "a", alias: "reserve" }, { id: "b", alias: "reserve" }] }); + }) as typeof fetch, + }; + try { + expect(await cmdPause(["anthropic", "reserve"], d, true)).toBe(1); + expect(writes).toEqual([]); + expect(await cmdPause(["anthropic", "a"], d, true)).toBe(0); + expect(writes).toEqual([{ provider: "anthropic", accountId: "a", paused: true }]); + } finally { out.restore(); } + }); + + test.each([true, false])("Anthropic pause=%s resolves a unique alias and preserves the JSON contract", async paused => { + const calls: Captured[] = []; + const out = capture(); + const d = deps(() => ({ json: { ok: true, activeAccountId: "acct_2" } }), calls); + d.loadConfigImpl = () => ({ providers: { anthropic: { adapter: "anthropic", authMode: "oauth" } } }) as never; + try { + expect(await cmdPause(["anthropic", "GEM-PRO", "--json"], d, paused)).toBe(0); + expect(calls.find(call => call.method === "PUT")?.body).toEqual({ provider: "anthropic", accountId: "acct_1", paused }); + expect(JSON.parse(out.lines.join("\n"))).toEqual({ ok: true, provider: "anthropic", id: "acct_1", paused, activeAccountId: "acct_2" }); + } finally { out.restore(); } + }); + test("generic OAuth pause resolves aliases and uses the OAuth account route", async () => { const calls: Captured[] = []; const out = capture(); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 1d05e0afd5c..1f0daa84ca9 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -106,6 +106,7 @@ "ambiguous-resend-composition.test.ts": "lib", "ambiguous-resend-gate.test.ts": "lib", "anthropic-account-pool.test.ts": "adapters/anthropic", + "anthropic-account-pause.test.ts": "adapters/anthropic", "anthropic-model-routes.test.ts": "adapters/anthropic", "anthropic-agentrouter-language-framing.test.ts": "adapters/anthropic", "anthropic-baseurl-override.test.ts": "adapters/anthropic", diff --git a/tests/oauth/oauth-accounts-api.test.ts b/tests/oauth/oauth-accounts-api.test.ts index d1c46e0ede8..c1d4a564d74 100644 --- a/tests/oauth/oauth-accounts-api.test.ts +++ b/tests/oauth/oauth-accounts-api.test.ts @@ -512,7 +512,32 @@ describe("multiauth accounts API", () => { } }); - test("pause API rejects Anthropic and a generic OAuth account behind an API-key route", async () => { + test("Anthropic pause validates input, persists resume and never alters credentials", async () => { + const before = getAccountSet("anthropic")!; + const server = startServer(0); + const pause = (accountId: string, paused: unknown) => fetch(new URL("/api/oauth/accounts/pause", server.url), { + method: "PUT", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ provider: "anthropic", accountId, paused }), + }); + try { + expect((await pause("aaaa1111", "true")).status).toBe(400); + expect((await pause("missing", true)).status).toBe(404); + expect(getAccountSet("anthropic")).toEqual(before); + expect((await pause("aaaa1111", true)).status).toBe(200); + expect((await pause("bbbb2222", true)).status).toBe(200); + const disk = JSON.parse(readFileSync(join(testDir, "auth.json"), "utf8")); + expect(disk.anthropic.accounts.every((row: { paused: boolean }) => row.paused)).toBe(true); + const resumed = await pause("aaaa1111", false); + expect(await resumed.json()).toMatchObject({ paused: false, activeAccountId: "aaaa1111", activeAccountChanged: true }); + const after = getAccountSet("anthropic")!; + expect(after.accounts.map(row => row.credential)).toEqual(before.accounts.map(row => row.credential)); + expect(after.accounts.find(row => row.id === "aaaa1111")?.paused).toBeUndefined(); + const serialized = await fetch(new URL("/api/oauth/accounts?provider=anthropic", server.url)).then(response => response.text()); + expect(serialized).not.toContain('"access"'); + expect(serialized).not.toContain('"refresh"'); + } finally { await server.stop(true); } + }); + + test("pause API supports Anthropic but rejects a generic OAuth account behind an API-key route", async () => { enableGoogleAntigravityAccounts(); const keyRouteConfig = baseConfig(); keyRouteConfig.providers["google-antigravity"] = { @@ -525,7 +550,12 @@ describe("multiauth accounts API", () => { method: "PUT", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ provider: "anthropic", accountId: "aaaa1111", paused: true }), }); - expect(anthropicPause.status).toBe(400); + expect(anthropicPause.status).toBe(200); + expect(await anthropicPause.json()).toMatchObject({ paused: true, activeAccountId: "bbbb2222" }); + const listed = await fetch(new URL("/api/oauth/accounts?provider=anthropic", server.url)); + const rows = await listed.json() as { accounts: Array<{ id: string; paused: boolean }> }; + expect(rows.accounts.find(row => row.id === "aaaa1111")?.paused).toBe(true); + expect(rows.accounts.find(row => row.id === "bbbb2222")?.paused).toBe(false); const keyRoutePause = await fetch(new URL("/api/oauth/accounts/pause", server.url), { method: "PUT", headers: { "Content-Type": "application/json" }, From aea8b5604b9dfc5b89e3d254d44f23fdc119eace Mon Sep 17 00:00:00 2001 From: Ingwannu Date: Mon, 28 Sep 2026 17:02:50 +0000 Subject: [PATCH 02/26] fix(oauth): preserve Anthropic pause refusals after pacing --- .../fr/reference/cli/providers-accounts.md | 4 ++ .../docs/fr/reference/management-api.md | 5 +++ .../ja/reference/cli/providers-accounts.md | 4 ++ .../docs/ja/reference/management-api.md | 5 +++ .../ko/reference/cli/providers-accounts.md | 4 ++ .../docs/ko/reference/management-api.md | 5 +++ .../ru/reference/cli/providers-accounts.md | 4 ++ .../docs/ru/reference/management-api.md | 5 +++ .../tr/reference/cli/providers-accounts.md | 4 ++ .../docs/tr/reference/management-api.md | 5 +++ .../zh-cn/reference/cli/providers-accounts.md | 4 ++ .../docs/zh-cn/reference/management-api.md | 5 +++ src/oauth/anthropic-routing.ts | 16 +++----- src/server/responses/adapter-dispatch.ts | 10 +++++ src/server/responses/passthrough-dispatch.ts | 13 ++++++- src/server/responses/request-transport.ts | 6 +++ .../ADR-6013-anthropic-account-pause.md | 3 ++ structure/providers-and-adapters.md | 5 ++- .../anthropic/anthropic-model-routes.test.ts | 39 +++++++++++++++++++ 19 files changed, 133 insertions(+), 13 deletions(-) diff --git a/docs-site/src/content/docs/fr/reference/cli/providers-accounts.md b/docs-site/src/content/docs/fr/reference/cli/providers-accounts.md index f4e17ef8221..4a8824336de 100644 --- a/docs-site/src/content/docs/fr/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/fr/reference/cli/providers-accounts.md @@ -214,6 +214,10 @@ faire basculer la requête vers un autre compte de pool admissible. Ces transiti { ok: true, provider, type, activeId } ``` +### `ocx account pause|resume anthropic [--json]` + +La commande CLI suspend ou reprend un compte Anthropic OAuth par id ou alias unique (correspondance exacte, puis sans distinction de casse). Utilise `PUT /api/oauth/accounts/pause` avec `{ provider: "anthropic", accountId, paused }`, également utilisé par le tableau de bord. L’état `paused` est enregistré dans le compte et exposé par `GET /api/oauth/accounts`. La suspension s’applique même si le pool proactif est désactivé : le compte est exclu de la sélection, des affinités et des successeurs 429. Si tous les comptes sont suspendus, les requêtes renvoient 403 jusqu’à une reprise. Les requêtes déjà envoyées continuent ; les identifiants et l’état de santé sont conservés. La suspension survit au redémarrage et à une nouvelle connexion, et disparaît avec la suppression du compte. Les seuils individuels ne font pas partie de cette commande. + ### `ocx account clear [--json]` Efface la sélection manuelle du compte Codex sans résoudre d'id de compte, donc fonctionne même lorsqu'un compte s'appelle littéralement `auto`. Pools Codex uniquement ; les autres types de fournisseur n'ont pas de sélection automatique à rétablir. diff --git a/docs-site/src/content/docs/fr/reference/management-api.md b/docs-site/src/content/docs/fr/reference/management-api.md index 6b0f60fc5e5..207e6a4c965 100644 --- a/docs-site/src/content/docs/fr/reference/management-api.md +++ b/docs-site/src/content/docs/fr/reference/management-api.md @@ -283,6 +283,7 @@ Tant qu’une liste initiale fiable n’est pas disponible, les requêtes PUT va | `GET, PUT, PATCH /api/oauth/accounts/pool` | Lire ou mettre à jour la stratégie du pool OAuth Anthropic | 400 fournisseur non Anthropic ou stratégie invalide | | `POST /api/oauth/accounts/clear-cooldown` | Effacer le temps de recharge d'un compte OAuth | 400 invalide provider/account | | `PUT /api/oauth/accounts/alias` | Définir ou supprimer un alias de compte OAuth | 400 invalide provider/account/alias | +| `PUT /api/oauth/accounts/pause` | Suspendre/reprendre Anthropic ou un compte OAuth générique. Body `{ provider, accountId, paused }` ; la suspension du compte actif sélectionne un autre compte utilisable s’il existe. | 400 fournisseur non pris en charge ou body invalide ; 404 compte absent ; `oauth_mutation_busy` | | `GET, POST, DELETE /api/providers/keys` | Répertorier les clés de fournisseur masquées, en ajouter ou en activer une, ou en supprimer une | 400 saisie invalide ; 404 fournisseur ou clé manquante | | `PUT /api/providers/keys/active` | Sélectionnez la clé active d'un fournisseur | 400 saisie invalide ; 404 provider/key manquant | | `PUT /api/providers/keys/alias` | Définir ou supprimer un alias de clé de fournisseur | 400 saisie invalide ; 404 provider/key manquant | @@ -291,6 +292,10 @@ Tant qu’une liste initiale fiable n’est pas disponible, les requêtes PUT va Les réponses qui répertorient les identifiants sont délibérément masquées. Les jetons d'accès OAuth et les clés API complètes des fournisseurs ne sont pas renvoyés aux clients du tableau de bord. +#### Anthropic OAuth: `pause` / `resume` + +La commande CLI suspend ou reprend un compte Anthropic OAuth par id ou alias unique (correspondance exacte, puis sans distinction de casse). Utilise `PUT /api/oauth/accounts/pause` avec `{ provider: "anthropic", accountId, paused }`, également utilisé par le tableau de bord. L’état `paused` est enregistré dans le compte et exposé par `GET /api/oauth/accounts`. La suspension s’applique même si le pool proactif est désactivé : le compte est exclu de la sélection, des affinités et des successeurs 429. Si tous les comptes sont suspendus, les requêtes renvoient 403 jusqu’à une reprise. Les requêtes déjà envoyées continuent ; les identifiants et l’état de santé sont conservés. La suspension survit au redémarrage et à une nouvelle connexion, et disparaît avec la suppression du compte. Les seuils individuels ne font pas partie de cette commande. + ### Fournisseurs | Méthode et chemin | Objectif | Erreurs notables | diff --git a/docs-site/src/content/docs/ja/reference/cli/providers-accounts.md b/docs-site/src/content/docs/ja/reference/cli/providers-accounts.md index 479fac0f0da..2d4b72755a6 100644 --- a/docs-site/src/content/docs/ja/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/ja/reference/cli/providers-accounts.md @@ -159,6 +159,10 @@ Codex pool selection applies to the next request after clearing existing affinit { ok: true, provider, type, activeId } ``` +### `ocx account pause|resume anthropic [--json]` + +CLI コマンドは Anthropic OAuth アカウントを id または一意の別名で一時停止・再開します。別名は完全一致を優先し、次に大文字と小文字を区別せず照合します。ダッシュボードと同じ `PUT /api/oauth/accounts/pause` に `{ provider: "anthropic", accountId, paused }` を送信します。`paused` はアカウントに保存され、`GET /api/oauth/accounts` にも表示されます。プロアクティブなプールが無効でも、停止中のアカウントは選択、セッションの紐付け、429 の切り替え候補から除外されます。全アカウントが停止中なら、再開するまでリクエストは 403 を返します。送信済みのリクエストは継続し、認証情報と健全性の状態は保持されます。再起動や再ログインでも停止は維持され、アカウント削除時に消えます。アカウント別のしきい値はこの操作に含まれません。 + ### `ocx account clear [--json]` アカウント id を解決せずに Codex アカウントの手動選択を解除するため、`auto` という id のアカウントが存在しても機能します。Codex プール専用です。他のプロバイダー種別には復元する自動選択がありません。 diff --git a/docs-site/src/content/docs/ja/reference/management-api.md b/docs-site/src/content/docs/ja/reference/management-api.md index 9821c5d358f..3fee2a3a801 100644 --- a/docs-site/src/content/docs/ja/reference/management-api.md +++ b/docs-site/src/content/docs/ja/reference/management-api.md @@ -245,6 +245,7 @@ Aside プロファイルの変更はこの場合でも一つだけ保存しま | `GET, PUT, PATCH /api/oauth/accounts/pool` | Anthropic OAuth プール ポリシーの読み取りまたは更新 | 400 非 Anthropic プロバイダーまたは無効なポリシー | | `POST /api/oauth/accounts/clear-cooldown` | 1 つの OAuth アカウントのランタイム クールダウンをクリアする | 400 無効なプロバイダー/アカウント | | `PUT /api/oauth/accounts/alias` | OAuth アカウント エイリアスを設定またはクリアする | 400 無効なプロバイダー/アカウント/エイリアス | +| `PUT /api/oauth/accounts/pause` | Anthropic または汎用 OAuth アカウントを一時停止・再開。Body `{ provider, accountId, paused }`。アクティブなアカウントを停止すると、利用可能な別のアカウントがあれば切り替えます。 | 400 未対応のプロバイダーまたは無効な body；404 アカウントなし；`oauth_mutation_busy` | | `GET, POST, DELETE /api/providers/keys` |マスクされたプロバイダー キーを一覧表示し、1 つを追加/アクティブ化するか、1 つを削除します。 400 無効な入力。 404 プロバイダー/キーがありません | | `PUT /api/providers/keys/active` |プロバイダーのアクティブなキーを選択します | 400 無効な入力。 404 プロバイダー/キーがありません | | `PUT /api/providers/keys/alias` |プロバイダー キー エイリアスを設定またはクリアする | 400 無効な入力。 404 プロバイダー/キーがありません | @@ -252,6 +253,10 @@ Aside プロファイルの変更はこの場合でも一つだけ保存しま 資格情報リストの応答は意図的にマスクされます。 OAuth アクセス トークンと完全なプロバイダー API キーはダッシュボード クライアントに返されません。 +#### Anthropic OAuth: `pause` / `resume` + +CLI コマンドは Anthropic OAuth アカウントを id または一意の別名で一時停止・再開します。別名は完全一致を優先し、次に大文字と小文字を区別せず照合します。ダッシュボードと同じ `PUT /api/oauth/accounts/pause` に `{ provider: "anthropic", accountId, paused }` を送信します。`paused` はアカウントに保存され、`GET /api/oauth/accounts` にも表示されます。プロアクティブなプールが無効でも、停止中のアカウントは選択、セッションの紐付け、429 の切り替え候補から除外されます。全アカウントが停止中なら、再開するまでリクエストは 403 を返します。送信済みのリクエストは継続し、認証情報と健全性の状態は保持されます。再起動や再ログインでも停止は維持され、アカウント削除時に消えます。アカウント別のしきい値はこの操作に含まれません。 + ### プロバイダー |メソッドとパス |目的 |注目すべきエラー | diff --git a/docs-site/src/content/docs/ko/reference/cli/providers-accounts.md b/docs-site/src/content/docs/ko/reference/cli/providers-accounts.md index 5f94fd837b1..856390db88c 100644 --- a/docs-site/src/content/docs/ko/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/ko/reference/cli/providers-accounts.md @@ -254,6 +254,10 @@ Codex pool selection applies to the next request after clearing existing affinit { ok: true, provider, type, activeId } ``` +### `ocx account pause|resume anthropic [--json]` + +CLI 명령은 Anthropic OAuth 계정을 id 또는 유일한 alias로 일시 정지하거나 재개합니다. alias는 정확히 일치하는 값을 먼저 찾고, 없으면 대소문자를 구분하지 않고 찾습니다. 대시보드와 같은 `PUT /api/oauth/accounts/pause`에 `{ provider: "anthropic", accountId, paused }`를 보냅니다. 계정에 저장되는 `paused` 상태는 `GET /api/oauth/accounts`에도 표시됩니다. 사전 계정 전환 풀이 꺼져 있어도 정지된 계정은 선택, 세션 바인딩, 429 대체 후보에서 제외됩니다. 모든 계정이 정지되면 하나를 재개할 때까지 요청은 403을 반환합니다. 이미 전송한 요청은 유지하며 자격 증명과 건강 상태를 지우지 않습니다. 재시작·재로그인 후에도 정지는 유지되고, 계정을 삭제하면 함께 제거됩니다. 계정별 전환 임계값은 이 기능에 포함되지 않습니다. + ### `ocx account clear [--json]` 계정 id를 해석하지 않고 Codex 계정의 수동 선택을 지우므로 `auto`라는 id의 계정이 있어도 동작합니다. Codex 풀 전용이며 다른 공급자 유형에는 복원할 자동 선택이 없습니다. diff --git a/docs-site/src/content/docs/ko/reference/management-api.md b/docs-site/src/content/docs/ko/reference/management-api.md index cd428d4ef35..1a884ba64d4 100644 --- a/docs-site/src/content/docs/ko/reference/management-api.md +++ b/docs-site/src/content/docs/ko/reference/management-api.md @@ -258,6 +258,7 @@ Aside 프로필 변경은 이때도 한 가지를 저장합니다. 확인을 보 | `GET, PUT, PATCH /api/oauth/accounts/pool` | Anthropic과 일반 OAuth provider의 기존 pool policy입니다. `/api/pool/settings`로 대체되었고 기존 클라이언트를 위해 유지합니다 | 400 codex 또는 API 키 provider, 잘못된 policy | | `POST /api/oauth/accounts/clear-cooldown` | OAuth 계정 하나의 런타임 cooldown을 지웁니다 | 400 잘못된 provider/account | | `PUT /api/oauth/accounts/alias` | OAuth 계정 alias를 설정하거나 지웁니다 | 400 잘못된 provider/account/alias | +| `PUT /api/oauth/accounts/pause` | Anthropic 또는 일반 OAuth 계정을 정지·재개합니다. Body `{ provider, accountId, paused }`. 활성 계정을 정지하면 사용 가능한 다른 계정이 있을 때 전환합니다. | 400 지원하지 않는 provider 또는 잘못된 body; 404 계정 없음; `oauth_mutation_busy` | | `GET, POST, DELETE /api/providers/keys` | 마스킹된 provider key를 나열, 추가/활성화, 또는 제거합니다 | 400 잘못된 입력; 404 provider/key 없음 | | `PUT /api/providers/keys/active` | provider의 활성 key를 선택합니다 | 400 잘못된 입력; 404 provider/key 없음 | | `PUT /api/providers/keys/alias` | provider-key alias를 설정하거나 지웁니다 | 400 잘못된 입력; 404 provider/key 없음 | @@ -265,6 +266,10 @@ Aside 프로필 변경은 이때도 한 가지를 저장합니다. 확인을 보 자격 증명 목록 응답은 의도적으로 마스킹됩니다. OAuth access token과 완전한 provider API key는 대시보드 클라이언트에 반환되지 않습니다. +#### Anthropic OAuth: `pause` / `resume` + +CLI 명령은 Anthropic OAuth 계정을 id 또는 유일한 alias로 일시 정지하거나 재개합니다. alias는 정확히 일치하는 값을 먼저 찾고, 없으면 대소문자를 구분하지 않고 찾습니다. 대시보드와 같은 `PUT /api/oauth/accounts/pause`에 `{ provider: "anthropic", accountId, paused }`를 보냅니다. 계정에 저장되는 `paused` 상태는 `GET /api/oauth/accounts`에도 표시됩니다. 사전 계정 전환 풀이 꺼져 있어도 정지된 계정은 선택, 세션 바인딩, 429 대체 후보에서 제외됩니다. 모든 계정이 정지되면 하나를 재개할 때까지 요청은 403을 반환합니다. 이미 전송한 요청은 유지하며 자격 증명과 건강 상태를 지우지 않습니다. 재시작·재로그인 후에도 정지는 유지되고, 계정을 삭제하면 함께 제거됩니다. 계정별 전환 임계값은 이 기능에 포함되지 않습니다. + ### 제공자 | HTTP 메서드와 경로 | 목적 | 주요 오류 | diff --git a/docs-site/src/content/docs/ru/reference/cli/providers-accounts.md b/docs-site/src/content/docs/ru/reference/cli/providers-accounts.md index 7339d3f53bb..bd3daa00013 100644 --- a/docs-site/src/content/docs/ru/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/ru/reference/cli/providers-accounts.md @@ -190,6 +190,10 @@ credential'а, это состояние тоже печатается, но к { ok: true, provider, type, activeId } ``` +### `ocx account pause|resume anthropic [--json]` + +Команда CLI приостанавливает или возобновляет аккаунт Anthropic OAuth по id или уникальному alias: сначала точное совпадение, затем без учёта регистра. CLI и дашборд используют `PUT /api/oauth/accounts/pause` с `{ provider: "anthropic", accountId, paused }`. Поле `paused` сохраняется в аккаунте и возвращается через `GET /api/oauth/accounts`. Пауза действует и при отключённом проактивном пуле: аккаунт исключается из выбора, привязок сессий и кандидатов после 429. Если приостановлены все аккаунты, запросы получают 403 до возобновления одного из них. Уже отправленные запросы продолжаются; учётные данные и состояние здоровья сохраняются. Пауза переживает перезапуск и повторный вход, но удаляется вместе с аккаунтом. Индивидуальные пороги в эту операцию не входят. + ### `ocx account clear [--json]` Снимает ручной выбор аккаунта Codex без разрешения id, поэтому работает, даже когда аккаунт буквально называется `auto`. Только для пулов Codex; у других типов провайдеров нет автоматического выбора для восстановления. diff --git a/docs-site/src/content/docs/ru/reference/management-api.md b/docs-site/src/content/docs/ru/reference/management-api.md index 30a02a70ed7..62b74ab5452 100644 --- a/docs-site/src/content/docs/ru/reference/management-api.md +++ b/docs-site/src/content/docs/ru/reference/management-api.md @@ -276,6 +276,7 @@ Endpoint'ы storage cleanup могут перемещать или навсег | `GET, PUT, PATCH /api/oauth/accounts/pool` | Прежняя policy пула для Anthropic и обычных OAuth-провайдеров; заменена на `/api/pool/settings` и сохранена для существующих клиентов | 400 codex или api-key provider, либо недопустимая policy | | `POST /api/oauth/accounts/clear-cooldown` | Очистить runtime cooldown одного OAuth-аккаунта | 400 invalid provider/account | | `PUT /api/oauth/accounts/alias` | Задать или очистить alias OAuth-аккаунта | 400 invalid provider/account/alias | +| `PUT /api/oauth/accounts/pause` | Приостановить/возобновить Anthropic или обычный OAuth-аккаунт. Body `{ provider, accountId, paused }`; при паузе активного аккаунта выбирается другой пригодный аккаунт, если он есть. | 400 неподдерживаемый provider или неверный body; 404 аккаунт не найден; `oauth_mutation_busy` | | `GET, POST, DELETE /api/providers/keys` | Показать список masked provider-key'ов, добавить/активировать один или удалить один | 400 invalid input; 404 provider/key missing | | `PUT /api/providers/keys/active` | Выбрать активный ключ провайдера | 400 invalid input; 404 provider/key missing | | `PUT /api/providers/keys/alias` | Задать или очистить alias provider-key'а | 400 invalid input; 404 provider/key missing | @@ -284,6 +285,10 @@ Endpoint'ы storage cleanup могут перемещать или навсег Ответы со списками credential'ов намеренно маскируются. OAuth access-token'ы и полные API-key'и провайдеров клиентам дашборда не возвращаются. +#### Anthropic OAuth: `pause` / `resume` + +Команда CLI приостанавливает или возобновляет аккаунт Anthropic OAuth по id или уникальному alias: сначала точное совпадение, затем без учёта регистра. CLI и дашборд используют `PUT /api/oauth/accounts/pause` с `{ provider: "anthropic", accountId, paused }`. Поле `paused` сохраняется в аккаунте и возвращается через `GET /api/oauth/accounts`. Пауза действует и при отключённом проактивном пуле: аккаунт исключается из выбора, привязок сессий и кандидатов после 429. Если приостановлены все аккаунты, запросы получают 403 до возобновления одного из них. Уже отправленные запросы продолжаются; учётные данные и состояние здоровья сохраняются. Пауза переживает перезапуск и повторный вход, но удаляется вместе с аккаунтом. Индивидуальные пороги в эту операцию не входят. + ### Провайдеры | Метод и путь | Назначение | Особые ошибки | diff --git a/docs-site/src/content/docs/tr/reference/cli/providers-accounts.md b/docs-site/src/content/docs/tr/reference/cli/providers-accounts.md index 15d2c125d1f..6c98e625a2d 100644 --- a/docs-site/src/content/docs/tr/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/tr/reference/cli/providers-accounts.md @@ -235,6 +235,10 @@ ayar yalnızca kullanıma dayalı proaktif geçişi devre dışı bırakır. { ok: true, provider, type, activeId } ``` +### `ocx account pause|resume anthropic [--json]` + +CLI komutu Anthropic OAuth hesabını id veya benzersiz takma ad ile duraklatır ya da sürdürür. Önce tam eşleşme, ardından büyük/küçük harf duyarsız eşleşme aranır. CLI ve kontrol paneli aynı `PUT /api/oauth/accounts/pause` uç noktasına `{ provider: "anthropic", accountId, paused }` gönderir. `paused` hesapta saklanır ve `GET /api/oauth/accounts` yanıtında gösterilir. Proaktif havuz kapalı olsa bile duraklatılan hesap seçimden, oturum bağlarından ve 429 sonrası adaylardan çıkarılır. Tüm hesaplar duraklatılmışsa biri sürdürülene kadar istekler 403 döndürür. Önceden gönderilmiş istekler devam eder; kimlik bilgileri ve sağlık durumu korunur. Duraklatma yeniden başlatma ve yeniden girişten sonra da sürer, hesap silinince kaldırılır. Hesaba özel eşikler bu işleme dahil değildir. + ### `ocx account clear [--json]` Bir hesap id'si çözümlemeden Codex hesabının elle seçimini temizler; `auto` adında bir hesap olsa bile çalışır. Yalnızca Codex havuzları içindir; diğer sağlayıcı türlerinde geri yüklenecek otomatik seçim yoktur. diff --git a/docs-site/src/content/docs/tr/reference/management-api.md b/docs-site/src/content/docs/tr/reference/management-api.md index 1bd688931ae..0a19213029c 100644 --- a/docs-site/src/content/docs/tr/reference/management-api.md +++ b/docs-site/src/content/docs/tr/reference/management-api.md @@ -301,6 +301,7 @@ Güvenilir ilk model listesi hazır olana kadar `/api/selected-models` ve `/api/ | `GET, PUT, PATCH /api/oauth/accounts/pool` | Anthropic OAuth havuz politikasını okuyun veya güncelleyin | 400 Anthropic olmayan sağlayıcı veya geçersiz politika | | `POST /api/oauth/accounts/clear-cooldown` | Bir OAuth hesabının çalışma zamanı soğuma süresini temizleyin | 400 geçersiz sağlayıcı/hesap | | `PUT /api/oauth/accounts/alias` | Bir OAuth hesap takma adını ayarlayın veya temizleyin | 400 geçersiz sağlayıcı/hesap/takma ad | +| `PUT /api/oauth/accounts/pause` | Anthropic veya genel OAuth hesabını duraklatın/sürdürün. Body `{ provider, accountId, paused }`; etkin hesap duraklatıldığında varsa kullanılabilir başka hesaba geçilir. | 400 desteklenmeyen sağlayıcı veya geçersiz body; 404 hesap yok; `oauth_mutation_busy` | | `GET, POST, DELETE /api/providers/keys` | Maskelenmiş sağlayıcı anahtarlarını listeleyin, bir tane ekleyin/etkinleştirin veya kaldırın | 400 geçersiz girdi; 404 sağlayıcı/anahtar eksik | | `PUT /api/providers/keys/active` | Bir sağlayıcının etkin anahtarını seçin | 400 geçersiz girdi; 404 sağlayıcı/anahtar eksik | | `PUT /api/providers/keys/alias` | Bir sağlayıcı anahtarı takma adını ayarlayın veya temizleyin | 400 geçersiz girdi; 404 sağlayıcı/anahtar eksik | @@ -310,6 +311,10 @@ Kimlik bilgisi listesi yanıtları kasıtlı olarak maskelenir. OAuth erişim belirteçleri ve eksiksiz sağlayıcı API anahtarları kontrol paneli istemcilerine döndürülmez. +#### Anthropic OAuth: `pause` / `resume` + +CLI komutu Anthropic OAuth hesabını id veya benzersiz takma ad ile duraklatır ya da sürdürür. Önce tam eşleşme, ardından büyük/küçük harf duyarsız eşleşme aranır. CLI ve kontrol paneli aynı `PUT /api/oauth/accounts/pause` uç noktasına `{ provider: "anthropic", accountId, paused }` gönderir. `paused` hesapta saklanır ve `GET /api/oauth/accounts` yanıtında gösterilir. Proaktif havuz kapalı olsa bile duraklatılan hesap seçimden, oturum bağlarından ve 429 sonrası adaylardan çıkarılır. Tüm hesaplar duraklatılmışsa biri sürdürülene kadar istekler 403 döndürür. Önceden gönderilmiş istekler devam eder; kimlik bilgileri ve sağlık durumu korunur. Duraklatma yeniden başlatma ve yeniden girişten sonra da sürer, hesap silinince kaldırılır. Hesaba özel eşikler bu işleme dahil değildir. + ### Sağlayıcılar | Yöntem ve yol | Amaç | Dikkate değer hatalar | diff --git a/docs-site/src/content/docs/zh-cn/reference/cli/providers-accounts.md b/docs-site/src/content/docs/zh-cn/reference/cli/providers-accounts.md index 2a102ecec5c..1806ae38843 100644 --- a/docs-site/src/content/docs/zh-cn/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/zh-cn/reference/cli/providers-accounts.md @@ -173,6 +173,10 @@ Codex Pool 选择会清除进程本地 affinity，并从下一次请求开始生 { ok: true, provider, type, activeId } ``` +### `ocx account pause|resume anthropic [--json]` + +CLI 命令通过 id 或唯一别名暂停或恢复 Anthropic OAuth 账户。别名先精确匹配，再进行不区分大小写的匹配。CLI 和仪表板使用同一个 `PUT /api/oauth/accounts/pause`，请求体为 `{ provider: "anthropic", accountId, paused }`。`paused` 保存在账户中，并通过 `GET /api/oauth/accounts` 返回。即使主动账户池已关闭，暂停账户也会从选择、会话绑定和 429 后继候选中排除。所有账户暂停时，请求返回 403，直到恢复一个账户。已经发送的请求继续执行，凭证和健康状态保持不变。重启或重新登录仍保留暂停，删除账户时一并清除。此操作不包含账户级自动切换阈值。 + ### `ocx account clear [--json]` 在不解析账号 id 的情况下清除 Codex 账号的手动选择，因此即使存在名为 `auto` 的账号也有效。仅适用于 Codex Pool；其他提供商类型没有可恢复的自动选择。 diff --git a/docs-site/src/content/docs/zh-cn/reference/management-api.md b/docs-site/src/content/docs/zh-cn/reference/management-api.md index 6280fb8b9d8..4abf420ad36 100644 --- a/docs-site/src/content/docs/zh-cn/reference/management-api.md +++ b/docs-site/src/content/docs/zh-cn/reference/management-api.md @@ -238,6 +238,7 @@ Aside 配置档的变更在这种情况下仍会保存一件事：确认之后 | `GET, PUT, PATCH /api/oauth/accounts/pool` | 读取或更新 Anthropic OAuth 池策略 | 400 非 Anthropic provider 或策略无效 | | `POST /api/oauth/accounts/clear-cooldown` | 清除一个 OAuth 账户的运行时冷却 | 400 provider/账户无效 | | `PUT /api/oauth/accounts/alias` | 设置或清除 OAuth 账户别名 | 400 provider/账户/别名无效 | +| `PUT /api/oauth/accounts/pause` | 暂停或恢复 Anthropic 或通用 OAuth 账户。Body `{ provider, accountId, paused }`；暂停活跃账户时，如有其他可用账户则切换过去。 | 400 不支持的 provider 或无效 body；404 账户不存在；`oauth_mutation_busy` | | `GET, POST, DELETE /api/providers/keys` | 列出已脱敏的 provider 密钥，添加/激活一个，或移除一个 | 400 输入无效；404 provider/密钥缺失 | | `PUT /api/providers/keys/active` | 选择某个 provider 的活跃密钥 | 400 输入无效；404 provider/密钥缺失 | | `PUT /api/providers/keys/alias` | 设置或清除 provider 密钥别名 | 400 输入无效；404 provider/密钥缺失 | @@ -245,6 +246,10 @@ Aside 配置档的变更在这种情况下仍会保存一件事：确认之后 凭证列表响应会刻意脱敏。OAuth 访问令牌和完整的 provider API 密钥不会返回给仪表板客户端。 +#### Anthropic OAuth: `pause` / `resume` + +CLI 命令通过 id 或唯一别名暂停或恢复 Anthropic OAuth 账户。别名先精确匹配，再进行不区分大小写的匹配。CLI 和仪表板使用同一个 `PUT /api/oauth/accounts/pause`，请求体为 `{ provider: "anthropic", accountId, paused }`。`paused` 保存在账户中，并通过 `GET /api/oauth/accounts` 返回。即使主动账户池已关闭，暂停账户也会从选择、会话绑定和 429 后继候选中排除。所有账户暂停时，请求返回 403，直到恢复一个账户。已经发送的请求继续执行，凭证和健康状态保持不变。重启或重新登录仍保留暂停，删除账户时一并清除。此操作不包含账户级自动切换阈值。 + ### Providers | 方法和路径 | 用途 | 典型错误 | diff --git a/src/oauth/anthropic-routing.ts b/src/oauth/anthropic-routing.ts index fdcbb98929c..91502cd2619 100644 --- a/src/oauth/anthropic-routing.ts +++ b/src/oauth/anthropic-routing.ts @@ -604,16 +604,12 @@ export function resolveAnthropicAccountForSession( const eligible = routeCandidates(getEligibleAnthropicAccounts(now), decision); if (decision && eligible.length === 0) { - let cooled: boolean; - if (decision.fallback) { - // A removed route member is not a candidate, but fallback can still use the - // ordinary pool once those accounts recover. Only usable stored accounts count. - const ordinary = set.accounts.filter(account => - account.paused !== true && account.needsReauth !== true && isPoolCredentialUsable(account.id, now)); - cooled = ordinary.length > 0 && ordinary.every(account => isCooled(account.id, now)); - } else { - cooled = decision.accounts.every(id => set.accounts.some(account => account.id === id && isCooled(id, now))); - } + // Pause, removal and reauthentication are not cooldown evidence. Classify only + // usable members of this strict route (or the ordinary pool after fallback widens). + const recoverable = set.accounts.filter(account => + (decision.fallback || decision.accounts.includes(account.id)) + && account.paused !== true && account.needsReauth !== true && isPoolCredentialUsable(account.id, now)); + const cooled = recoverable.length > 0 && recoverable.every(account => isCooled(account.id, now)); return { accountId: null, reason: cooled ? "all-cooled" : "none", routePosition: decision.position }; } diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index b836e619b2c..2710fd3077f 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -35,6 +35,7 @@ import { fetchWithResetRetry, applyUpstreamRecoveryInit, SendBudgetExhaustedError, + UpstreamRetryEvidenceError, prepareSameTarget429Wait, sleepWithAbort, } from "../../lib/upstream-retry"; @@ -382,6 +383,11 @@ export async function prepareAdapterExchange( cleanupUpstreamAbort(); upstream.abort(); if (options.abortSignal?.aborted) return clientCancelledResponse(); + const refusal = err instanceof UpstreamRetryEvidenceError ? err.cause : err; + // A pause committed during pacing is local admission policy, not a failed upstream. + if (refusal instanceof OAuthAccountPausedError) { + return formatErrorResponse(403, "permission_error", publicOAuthAuthenticationErrorMessage(refusal)); + } // A budget refusal is a decision this process made, not an upstream fault. Reporting it as // 502 does more than mislabel it: the Codex client retries 5xx and does not retry a 429, so // blaming the provider makes the caller send the whole turn again -- the amplification this @@ -588,6 +594,10 @@ export async function prepareAdapterExchange( if (options.abortSignal?.aborted) { return { failed: clientCancelledResponse() }; } + const refusal = err instanceof UpstreamRetryEvidenceError ? err.cause : err; + if (refusal instanceof OAuthAccountPausedError) { + return { failed: formatErrorResponse(403, "permission_error", publicOAuthAuthenticationErrorMessage(refusal)) }; + } // Same rule on the recovery leg: the ladder refused to send again, so the answer names // this proxy rather than the provider it never reached. if (err instanceof SendBudgetExhaustedError) { diff --git a/src/server/responses/passthrough-dispatch.ts b/src/server/responses/passthrough-dispatch.ts index a331cbaeaf8..0c73b678cce 100644 --- a/src/server/responses/passthrough-dispatch.ts +++ b/src/server/responses/passthrough-dispatch.ts @@ -137,7 +137,7 @@ import { resolveWireProtocolOverride } from "../adapter-resolve"; import { refreshPoolForwardAuth, refreshNativeMainForwardAuth, withClaudeNativeSession } from "./core-auth"; import { bindRouteReasoningReplayScope } from "./core-replay"; import type { OAuthAccessSnapshot } from "../../oauth"; -import { publicOAuthAuthenticationErrorMessage } from "../../oauth"; +import { OAuthAccountPausedError, publicOAuthAuthenticationErrorMessage } from "../../oauth"; import { resolveCopilotApiBaseUrl } from "../../oauth/github-copilot"; import { hasEligibleGenericOAuthFailoverTarget, @@ -881,7 +881,16 @@ export async function preparePassthroughExchange( releaseCodexAuthContextProbeLease(admissionState.authCtx); return formatErrorResponse(429, "request_send_budget_exhausted", err.message); } - const localRefusal = mapCodexAuthContextErrorToResponse(unwrapUpstreamRetryEvidenceError(err), { + const refusal = unwrapUpstreamRetryEvidenceError(err); + // Pacing may outlive the selected account's admission. No fetch occurred, so do + // not turn an operator pause into a 502 or charge it to host/account health. + if (refusal instanceof OAuthAccountPausedError) { + releaseUpstreamHostAdmission(nativeHostState.lease); + nativeHostState.lease = null; + releaseCodexAuthContextProbeLease(admissionState.authCtx); + return formatErrorResponse(403, "permission_error", publicOAuthAuthenticationErrorMessage(refusal)); + } + const localRefusal = mapCodexAuthContextErrorToResponse(refusal, { now: Date.now(), accountSelector: route.codexAccountNamespace, }); if (localRefusal) { diff --git a/src/server/responses/request-transport.ts b/src/server/responses/request-transport.ts index f2262d4fa96..a544fcdc128 100644 --- a/src/server/responses/request-transport.ts +++ b/src/server/responses/request-transport.ts @@ -400,6 +400,12 @@ export async function prepareResponsesTransport( const refreshDispatchAdapter = async (requestParsed: OcxParsedRequest): Promise => { if (route.provider.authMode === "oauth") { if (!servingOAuthSnapshot || !await applyFailoverSnapshot(servingOAuthSnapshot, requestParsed)) { + // An explicit route may reject its stale proposal before resolving a new bearer. + // Preserve a local pause refusal instead of misclassifying it as a transport error. + if (route.providerName === "anthropic" + && resolveAnthropicAccountForSession(anthropicSessionKey, config, Date.now(), anthropicRouteDecision).reason === "paused") { + throw new OAuthAccountPausedError(); + } throw new Error("OAuth account selection changed before dispatch"); } } else { diff --git a/structure/decisions/ADR-6013-anthropic-account-pause.md b/structure/decisions/ADR-6013-anthropic-account-pause.md index 6b01aaf5b1b..ed45cda59bd 100644 --- a/structure/decisions/ADR-6013-anthropic-account-pause.md +++ b/structure/decisions/ADR-6013-anthropic-account-pause.md @@ -15,6 +15,9 @@ mutation, endpoint and translated controls. Filter Anthropic eligibility at each selector and validate live pause state at credential, commit and physical-send boundaries. Persist successful in-flight refresh rotation, but leave paused health unchanged on late failure. + Keep typed local refusals across adapter/passthrough error projection after pacing, and + classify routed cooldown using only usable unpaused members so resume and retry guidance + remain distinct from login errors. - Why this approach: One authoritative row avoids config/auth split-write races and automatic cleanup follows account deletion. A UI-only toggle would still allow affinity, pool-off failover or a pre-wait bearer to select the paused account. diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index a508f9bd202..8e2ec21e617 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -14,7 +14,10 @@ manual, affinity, model-route and reactive 429 candidates, including when proact is off. All-paused requests return 403 with resume guidance. Quorum is invalidated on pause and resume; a sent account paused before its 429 may still recover on its sole unpaused successor. Credential resolution, refresh-lock acquisition, selection commit and physical -dispatch recheck live eligibility after asynchronous waits. An already-dispatched refresh +dispatch recheck live eligibility after asynchronous waits. Adapter and passthrough dispatch +preserve the 403 pause refusal after pacing; they do not report a local rejection as a 502 +upstream failure. A strict route with paused members and only cooled usable survivors +returns 429 with Retry-After, not a login error. An already-dispatched refresh retains a successful rotated credential without unpausing; a late failure cannot mark the paused row for reauthentication. Token Guardian and quota probes use their existing pause guards. diff --git a/tests/adapters/anthropic/anthropic-model-routes.test.ts b/tests/adapters/anthropic/anthropic-model-routes.test.ts index 76b22368cc1..fd2c36197cb 100644 --- a/tests/adapters/anthropic/anthropic-model-routes.test.ts +++ b/tests/adapters/anthropic/anthropic-model-routes.test.ts @@ -146,6 +146,45 @@ test.each([true, false])("pause while queued for pacing never sends the cached b expect(sends[0]).not.toContain("synthetic-access-0"); }); +for (const adapter of ["anthropic", "openai-responses"] as const) { + for (const enabled of [true, false]) { + test(`${adapter}: pausing every account during pacing returns 403, pool enabled=${enabled}`, async () => { + const ids = await seed(); + const cfg = config(ids, () => answer()); + cfg.anthropicAccountPool!.enabled = enabled; + cfg.providers.anthropic!.adapter = adapter; + cfg.providers.anthropic!.requestPacing = { enabled: true, maxConcurrentRequests: 1 }; + const slot = await waitForProviderRequestSlot("anthropic", cfg.providers.anthropic!, "claude-sonnet-4-5"); + const pending = post(cfg); + try { + for (let i = 0; i < 100 && providerRequestPacingStatus("anthropic", cfg.providers.anthropic!).queued === 0; i++) { + await new Promise(resolve => setTimeout(resolve, 5)); + } + expect(providerRequestPacingStatus("anthropic", cfg.providers.anthropic!).queued).toBe(1); + for (const id of ids) await setAccountPaused("anthropic", id, true); + } finally { slot.release(); } + const response = await pending; + expect(response.status).toBe(403); + expect(await response.json()).toMatchObject({ error: { type: "permission_error", message: expect.stringContaining("Resume") } }); + expect(sends).toEqual([]); + }); + } +} + +test("strict route with paused and cooled members returns 429 from its remaining usable member", async () => { + const ids = await seed(); + const cfg = config(ids, () => answer()); + const decision = resolveAnthropicModelRoute(cfg, "claude-sonnet-4-5").decision!; + await setAccountPaused("anthropic", ids[1]!, true); + rotateAnthropicAccountOn429(cfg, ids[2]!, "60", null, Date.now(), null, decision); + expect(resolveAnthropicAccountForSession("", cfg, Date.now(), decision).reason).toBe("all-cooled"); + const response = await post(cfg); + expect(response.status).toBe(429); + expect(Number(response.headers.get("retry-after"))).toBeGreaterThan(0); + expect(Number(response.headers.get("retry-after"))).toBeLessThanOrEqual(60); + expect(sends).toEqual([]); +}); + test("matched route excludes active outsider before an upstream send", async () => { const ids = await seed(); const cfg = config(ids, () => answer()); From 60d1d3cbfd9a6685b94a36d519554805d47717da Mon Sep 17 00:00:00 2001 From: Ingwannu Date: Mon, 28 Sep 2026 17:13:37 +0000 Subject: [PATCH 03/26] fix(oauth): retain scoped cooldown refusals after pacing --- src/oauth/anthropic-routing.ts | 31 +++++++---- src/server/responses/adapter-dispatch.ts | 9 ++++ src/server/responses/passthrough-dispatch.ts | 7 ++- src/server/responses/request-transport.ts | 30 ++++++++--- .../ADR-6013-anthropic-account-pause.md | 2 + structure/providers-and-adapters.md | 6 +-- .../anthropic/anthropic-model-routes.test.ts | 51 +++++++++++++++++++ 7 files changed, 114 insertions(+), 22 deletions(-) diff --git a/src/oauth/anthropic-routing.ts b/src/oauth/anthropic-routing.ts index 91502cd2619..361664001ba 100644 --- a/src/oauth/anthropic-routing.ts +++ b/src/oauth/anthropic-routing.ts @@ -58,6 +58,16 @@ export const ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST = 3; export type AnthropicAccountPoolConfig = NonNullable; +/** Local admission refusal across async dispatch; never treat it as provider reachability evidence. */ +export class AnthropicAccountCooldownError extends Error { + constructor(readonly retryAfterSeconds: number | null, readonly routePosition?: number) { + super(routePosition === undefined + ? "All Anthropic OAuth accounts are temporarily rate-limited" + : "Anthropic OAuth accounts for this model route are temporarily rate-limited"); + this.name = "AnthropicAccountCooldownError"; + } +} + /** * Where a cooldown's length came from. Same vocabulary as `CodexCooldownSource`, because it * answers the same question for the same reason: `retry-after` is upstream answering THIS @@ -358,9 +368,8 @@ export function getAnthropicPoolRetryAfterSeconds(now = Date.now(), decision: An && !getEligibleAnthropicAccounts(now).some(id => decision.accounts.includes(id)); let earliest: number | null = null; for (const account of set.accounts) { - if (account.paused === true) continue; + if (account.paused === true || account.needsReauth === true || !isPoolCredentialUsable(account.id, now)) continue; if (decision && !fallbackExpanded && !decision.accounts.includes(account.id)) continue; - if (fallbackExpanded && (account.needsReauth === true || !isPoolCredentialUsable(account.id, now))) continue; const snap = getAnthropicAccountHealthSnapshot(account.id, now); if (!snap?.cooldownUntil) continue; if (earliest === null || snap.cooldownUntil < earliest) earliest = snap.cooldownUntil; @@ -596,21 +605,21 @@ export function resolveAnthropicAccountForSession( : null; } - if (!isAnthropicAccountPoolEnabled(config)) { - const active = set.accounts.find(account => account.id === set.activeAccountId); - // Disabled proactive rotation does not grant permission to use an operator-paused slot. - return { accountId: active?.paused ? getEligibleAnthropicAccounts(now)[0] ?? null : set.activeAccountId, reason: "pool-disabled" }; - } - const eligible = routeCandidates(getEligibleAnthropicAccounts(now), decision); - if (decision && eligible.length === 0) { + if (eligible.length === 0) { // Pause, removal and reauthentication are not cooldown evidence. Classify only // usable members of this strict route (or the ordinary pool after fallback widens). const recoverable = set.accounts.filter(account => - (decision.fallback || decision.accounts.includes(account.id)) + (!decision || decision.fallback || decision.accounts.includes(account.id)) && account.paused !== true && account.needsReauth !== true && isPoolCredentialUsable(account.id, now)); const cooled = recoverable.length > 0 && recoverable.every(account => isCooled(account.id, now)); - return { accountId: null, reason: cooled ? "all-cooled" : "none", routePosition: decision.position }; + if (cooled || decision) return { accountId: null, reason: cooled ? "all-cooled" : "none", routePosition: decision?.position }; + } + + if (!isAnthropicAccountPoolEnabled(config)) { + const active = set.accounts.find(account => account.id === set.activeAccountId); + // Disabled proactive rotation does not authorize a paused slot or an entirely cooled pool. + return { accountId: active?.paused || isCooled(set.activeAccountId, now) ? eligible[0] ?? null : set.activeAccountId, reason: "pool-disabled" }; } // A manual choice is a one-dispatch preference, not a lower-priority quota hint. diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index 2710fd3077f..f895b0e50a2 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -55,6 +55,7 @@ import { resolveCopilotApiBaseUrl } from "../../oauth/github-copilot"; import { resolveWireProtocolOverride } from "../adapter-resolve"; import { bindRouteReasoningReplayScope } from "./core-replay"; import { + AnthropicAccountCooldownError, ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST, rotateAnthropicAccountOn429, getAnthropicPoolAccessSnapshot, @@ -388,6 +389,10 @@ export async function prepareAdapterExchange( if (refusal instanceof OAuthAccountPausedError) { return formatErrorResponse(403, "permission_error", publicOAuthAuthenticationErrorMessage(refusal)); } + if (refusal instanceof AnthropicAccountCooldownError) { + return formatErrorResponse(429, "rate_limit_error", refusal.message, + refusal.retryAfterSeconds === null ? undefined : { retryAfter: String(refusal.retryAfterSeconds) }); + } // A budget refusal is a decision this process made, not an upstream fault. Reporting it as // 502 does more than mislabel it: the Codex client retries 5xx and does not retry a 429, so // blaming the provider makes the caller send the whole turn again -- the amplification this @@ -598,6 +603,10 @@ export async function prepareAdapterExchange( if (refusal instanceof OAuthAccountPausedError) { return { failed: formatErrorResponse(403, "permission_error", publicOAuthAuthenticationErrorMessage(refusal)) }; } + if (refusal instanceof AnthropicAccountCooldownError) { + return { failed: formatErrorResponse(429, "rate_limit_error", refusal.message, + refusal.retryAfterSeconds === null ? undefined : { retryAfter: String(refusal.retryAfterSeconds) }) }; + } // Same rule on the recovery leg: the ladder refused to send again, so the answer names // this proxy rather than the provider it never reached. if (err instanceof SendBudgetExhaustedError) { diff --git a/src/server/responses/passthrough-dispatch.ts b/src/server/responses/passthrough-dispatch.ts index 0c73b678cce..07fcc155bfe 100644 --- a/src/server/responses/passthrough-dispatch.ts +++ b/src/server/responses/passthrough-dispatch.ts @@ -138,6 +138,7 @@ import { refreshPoolForwardAuth, refreshNativeMainForwardAuth, withClaudeNativeS import { bindRouteReasoningReplayScope } from "./core-replay"; import type { OAuthAccessSnapshot } from "../../oauth"; import { OAuthAccountPausedError, publicOAuthAuthenticationErrorMessage } from "../../oauth"; +import { AnthropicAccountCooldownError } from "../../oauth/anthropic-routing"; import { resolveCopilotApiBaseUrl } from "../../oauth/github-copilot"; import { hasEligibleGenericOAuthFailoverTarget, @@ -884,10 +885,14 @@ export async function preparePassthroughExchange( const refusal = unwrapUpstreamRetryEvidenceError(err); // Pacing may outlive the selected account's admission. No fetch occurred, so do // not turn an operator pause into a 502 or charge it to host/account health. - if (refusal instanceof OAuthAccountPausedError) { + if (refusal instanceof OAuthAccountPausedError || refusal instanceof AnthropicAccountCooldownError) { releaseUpstreamHostAdmission(nativeHostState.lease); nativeHostState.lease = null; releaseCodexAuthContextProbeLease(admissionState.authCtx); + if (refusal instanceof AnthropicAccountCooldownError) { + return formatErrorResponse(429, "rate_limit_error", refusal.message, + refusal.retryAfterSeconds === null ? undefined : { retryAfter: String(refusal.retryAfterSeconds) }); + } return formatErrorResponse(403, "permission_error", publicOAuthAuthenticationErrorMessage(refusal)); } const localRefusal = mapCodexAuthContextErrorToResponse(refusal, { diff --git a/src/server/responses/request-transport.ts b/src/server/responses/request-transport.ts index a544fcdc128..e637c7ef7ab 100644 --- a/src/server/responses/request-transport.ts +++ b/src/server/responses/request-transport.ts @@ -15,8 +15,10 @@ import type { AdapterEvent, OcxParsedRequest, OcxProviderConfig, OcxUsage } from import type { AnthropicAccountSelectionReason } from "../../oauth/anthropic-routing"; import { resolveAnthropicModelRoute, routeCandidates, type AnthropicRouteDecision } from "../../oauth/anthropic-model-routes"; import { + AnthropicAccountCooldownError, isAnthropicAccountPoolEnabled, getAnthropicPoolAccessSnapshot, + getAnthropicAccountHealthSnapshot, getEligibleAnthropicAccounts, commitAnthropicSelectionRouting, formatAnthropicProviderForLog, @@ -338,6 +340,7 @@ export async function prepareResponsesTransport( const row = getAccountCredentialWithStatus(route.providerName, binding.snapshot.accountId); return selected?.accountId === binding.selection.accountId && selected?.revision === binding.selection.revision && !!row && !row.paused && !row.needsReauth && row.credential.expires > Date.now() + && (route.providerName !== "anthropic" || !getAnthropicAccountHealthSnapshot(binding.snapshot.accountId)) && credentialGeneration(row.credential) === binding.snapshot.generation; }; const resolveSelectionAdapter = (provider: OcxProviderConfig, retention = config.cacheRetention): ProviderAdapter => { @@ -397,15 +400,28 @@ export async function prepareResponsesTransport( } return resolved; }; + const resolveAnthropicDispatchAccount = (): string | null => { + if (route.providerName !== "anthropic") return null; + const now = Date.now(); + const selection = resolveAnthropicAccountForSession(anthropicSessionKey, config, now, anthropicRouteDecision); + if (selection.reason === "paused") throw new OAuthAccountPausedError(); + if (selection.reason === "all-cooled") { + throw new AnthropicAccountCooldownError(getAnthropicPoolRetryAfterSeconds(now, anthropicRouteDecision), anthropicRouteDecision?.position); + } + return selection.accountId; + }; const refreshDispatchAdapter = async (requestParsed: OcxParsedRequest): Promise => { if (route.provider.authMode === "oauth") { - if (!servingOAuthSnapshot || !await applyFailoverSnapshot(servingOAuthSnapshot, requestParsed)) { - // An explicit route may reject its stale proposal before resolving a new bearer. - // Preserve a local pause refusal instead of misclassifying it as a transport error. - if (route.providerName === "anthropic" - && resolveAnthropicAccountForSession(anthropicSessionKey, config, Date.now(), anthropicRouteDecision).reason === "paused") { - throw new OAuthAccountPausedError(); - } + // Pacing may outlive admission. Preserve local pause/cooldown reasons even when + // a strict route rejects the old proposal before another bearer can be resolved. + let candidate = servingOAuthSnapshot; + if (route.providerName === "anthropic") { + oauthSelection = captureOAuthAccountSelection(route.providerName); + const accountId = resolveAnthropicDispatchAccount(); + candidate = accountId ? await getAnthropicPoolAccessSnapshot(accountId) : undefined; + } + if (!candidate || !await applyFailoverSnapshot(candidate, requestParsed)) { + resolveAnthropicDispatchAccount(); throw new Error("OAuth account selection changed before dispatch"); } } else { diff --git a/structure/decisions/ADR-6013-anthropic-account-pause.md b/structure/decisions/ADR-6013-anthropic-account-pause.md index ed45cda59bd..2db38da2a30 100644 --- a/structure/decisions/ADR-6013-anthropic-account-pause.md +++ b/structure/decisions/ADR-6013-anthropic-account-pause.md @@ -18,6 +18,8 @@ Keep typed local refusals across adapter/passthrough error projection after pacing, and classify routed cooldown using only usable unpaused members so resume and retry guidance remain distinct from login errors. + Cooldown refusals carry the route position and retry seconds across pacing and dispatch, + including disabled proactive pools; they never count as an upstream reachability failure. - Why this approach: One authoritative row avoids config/auth split-write races and automatic cleanup follows account deletion. A UI-only toggle would still allow affinity, pool-off failover or a pre-wait bearer to select the paused account. diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index 8e2ec21e617..e6fb12839af 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -15,9 +15,9 @@ is off. All-paused requests return 403 with resume guidance. Quorum is invalidat and resume; a sent account paused before its 429 may still recover on its sole unpaused successor. Credential resolution, refresh-lock acquisition, selection commit and physical dispatch recheck live eligibility after asynchronous waits. Adapter and passthrough dispatch -preserve the 403 pause refusal after pacing; they do not report a local rejection as a 502 -upstream failure. A strict route with paused members and only cooled usable survivors -returns 429 with Retry-After, not a login error. An already-dispatched refresh +preserve typed 403 pause and 429 cooldown refusals after pacing, including pool-off recovery; +they do not report local rejection as 502. Only cooled usable survivors of a strict route +produce its scoped 429 and Retry-After, not a login error. An already-dispatched refresh retains a successful rotated credential without unpausing; a late failure cannot mark the paused row for reauthentication. Token Guardian and quota probes use their existing pause guards. diff --git a/tests/adapters/anthropic/anthropic-model-routes.test.ts b/tests/adapters/anthropic/anthropic-model-routes.test.ts index fd2c36197cb..2229c607d37 100644 --- a/tests/adapters/anthropic/anthropic-model-routes.test.ts +++ b/tests/adapters/anthropic/anthropic-model-routes.test.ts @@ -148,6 +148,57 @@ test.each([true, false])("pause while queued for pacing never sends the cached b for (const adapter of ["anthropic", "openai-responses"] as const) { for (const enabled of [true, false]) { + test(`${adapter}: a pacing-time pause skips a cooled successor for a healthy account, pool enabled=${enabled}`, async () => { + const ids = await seed(); + const cfg = config(ids, () => answer()); + cfg.anthropicAccountPool = { enabled, routes: [{ name: "all", match: "claude-*", accounts: ids }] }; + cfg.providers.anthropic!.adapter = adapter; + cfg.providers.anthropic!.requestPacing = { enabled: true, maxConcurrentRequests: 1 }; + const slot = await waitForProviderRequestSlot("anthropic", cfg.providers.anthropic!, "claude-sonnet-4-5"); + const pending = post(cfg); + try { + for (let i = 0; i < 100 && providerRequestPacingStatus("anthropic", cfg.providers.anthropic!).queued === 0; i++) { + await new Promise(resolve => setTimeout(resolve, 5)); + } + expect(providerRequestPacingStatus("anthropic", cfg.providers.anthropic!).queued).toBe(1); + rotateAnthropicAccountOn429(cfg, ids[1]!, "60"); + await setAccountPaused("anthropic", ids[0]!, true); + } finally { slot.release(); } + expect((await pending).status).toBe(200); + expect(sends).toHaveLength(1); + expect(sends[0]).toContain("synthetic-access-2"); + }); + + test(`${adapter}: paused plus cooled accounts during pacing return scoped 429, pool enabled=${enabled}`, async () => { + const ids = await seed(); + const cfg = config(ids, () => answer()); + cfg.anthropicAccountPool = { enabled, routes: [{ name: "private-scope", match: "claude-*", accounts: [ids[0]!, ids[1]!] }] }; + cfg.providers.anthropic!.adapter = adapter; + cfg.providers.anthropic!.requestPacing = { enabled: true, maxConcurrentRequests: 1 }; + if (!enabled) await setAccountPaused("anthropic", ids[2]!, true); + const slot = await waitForProviderRequestSlot("anthropic", cfg.providers.anthropic!, "claude-sonnet-4-5"); + const pending = post(cfg); + try { + for (let i = 0; i < 100 && providerRequestPacingStatus("anthropic", cfg.providers.anthropic!).queued === 0; i++) { + await new Promise(resolve => setTimeout(resolve, 5)); + } + expect(providerRequestPacingStatus("anthropic", cfg.providers.anthropic!).queued).toBe(1); + rotateAnthropicAccountOn429(cfg, ids[1]!, "60"); + // A shorter outsider cooldown must not change the strict route's Retry-After. + if (enabled) rotateAnthropicAccountOn429(cfg, ids[2]!, "5"); + await setAccountPaused("anthropic", ids[0]!, true); + } finally { slot.release(); } + const response = await pending; + expect(response.status).toBe(429); + expect(Number(response.headers.get("retry-after"))).toBeGreaterThan(50); + expect(Number(response.headers.get("retry-after"))).toBeLessThanOrEqual(60); + const body = await response.json() as { error: { type: string; message: string } }; + expect(body.error.type).toBe("rate_limit_error"); + expect(body.error.message.includes("model route")).toBe(enabled); + expect(body.error.message).not.toContain("private-scope"); + expect(sends).toEqual([]); + }); + test(`${adapter}: pausing every account during pacing returns 403, pool enabled=${enabled}`, async () => { const ids = await seed(); const cfg = config(ids, () => answer()); From 33e2cab3e7ca7a3c45db2cfdc06e702a423ccbff Mon Sep 17 00:00:00 2001 From: Ingwannu Date: Mon, 28 Sep 2026 17:21:32 +0000 Subject: [PATCH 04/26] fix(oauth): preserve mixed-state Anthropic admission errors --- src/server/responses/adapter-dispatch.ts | 6 +++ src/server/responses/passthrough-dispatch.ts | 7 ++- src/server/responses/request-transport.ts | 10 ++++ .../ADR-6013-anthropic-account-pause.md | 3 ++ structure/providers-and-adapters.md | 2 +- .../anthropic/anthropic-model-routes.test.ts | 52 ++++++++++++++++++- 6 files changed, 75 insertions(+), 5 deletions(-) diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index f895b0e50a2..34994a37c9d 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -389,6 +389,9 @@ export async function prepareAdapterExchange( if (refusal instanceof OAuthAccountPausedError) { return formatErrorResponse(403, "permission_error", publicOAuthAuthenticationErrorMessage(refusal)); } + if (refusal instanceof OAuthLoginRequiredError) { + return formatErrorResponse(401, "authentication_error", publicOAuthAuthenticationErrorMessage(refusal)); + } if (refusal instanceof AnthropicAccountCooldownError) { return formatErrorResponse(429, "rate_limit_error", refusal.message, refusal.retryAfterSeconds === null ? undefined : { retryAfter: String(refusal.retryAfterSeconds) }); @@ -603,6 +606,9 @@ export async function prepareAdapterExchange( if (refusal instanceof OAuthAccountPausedError) { return { failed: formatErrorResponse(403, "permission_error", publicOAuthAuthenticationErrorMessage(refusal)) }; } + if (refusal instanceof OAuthLoginRequiredError) { + return { failed: formatErrorResponse(401, "authentication_error", publicOAuthAuthenticationErrorMessage(refusal)) }; + } if (refusal instanceof AnthropicAccountCooldownError) { return { failed: formatErrorResponse(429, "rate_limit_error", refusal.message, refusal.retryAfterSeconds === null ? undefined : { retryAfter: String(refusal.retryAfterSeconds) }) }; diff --git a/src/server/responses/passthrough-dispatch.ts b/src/server/responses/passthrough-dispatch.ts index 07fcc155bfe..ca1f92e473c 100644 --- a/src/server/responses/passthrough-dispatch.ts +++ b/src/server/responses/passthrough-dispatch.ts @@ -137,7 +137,7 @@ import { resolveWireProtocolOverride } from "../adapter-resolve"; import { refreshPoolForwardAuth, refreshNativeMainForwardAuth, withClaudeNativeSession } from "./core-auth"; import { bindRouteReasoningReplayScope } from "./core-replay"; import type { OAuthAccessSnapshot } from "../../oauth"; -import { OAuthAccountPausedError, publicOAuthAuthenticationErrorMessage } from "../../oauth"; +import { OAuthAccountPausedError, OAuthLoginRequiredError, publicOAuthAuthenticationErrorMessage } from "../../oauth"; import { AnthropicAccountCooldownError } from "../../oauth/anthropic-routing"; import { resolveCopilotApiBaseUrl } from "../../oauth/github-copilot"; import { @@ -885,10 +885,13 @@ export async function preparePassthroughExchange( const refusal = unwrapUpstreamRetryEvidenceError(err); // Pacing may outlive the selected account's admission. No fetch occurred, so do // not turn an operator pause into a 502 or charge it to host/account health. - if (refusal instanceof OAuthAccountPausedError || refusal instanceof AnthropicAccountCooldownError) { + if (refusal instanceof OAuthAccountPausedError || refusal instanceof OAuthLoginRequiredError || refusal instanceof AnthropicAccountCooldownError) { releaseUpstreamHostAdmission(nativeHostState.lease); nativeHostState.lease = null; releaseCodexAuthContextProbeLease(admissionState.authCtx); + if (refusal instanceof OAuthLoginRequiredError) { + return formatErrorResponse(401, "authentication_error", publicOAuthAuthenticationErrorMessage(refusal)); + } if (refusal instanceof AnthropicAccountCooldownError) { return formatErrorResponse(429, "rate_limit_error", refusal.message, refusal.retryAfterSeconds === null ? undefined : { retryAfter: String(refusal.retryAfterSeconds) }); diff --git a/src/server/responses/request-transport.ts b/src/server/responses/request-transport.ts index e637c7ef7ab..0c46b96a533 100644 --- a/src/server/responses/request-transport.ts +++ b/src/server/responses/request-transport.ts @@ -408,6 +408,16 @@ export async function prepareResponsesTransport( if (selection.reason === "all-cooled") { throw new AnthropicAccountCooldownError(getAnthropicPoolRetryAfterSeconds(now, anthropicRouteDecision), anthropicRouteDecision?.position); } + // No usable credential is an authentication refusal, not failed host reachability. + // Keep it distinct from the explicit all-paused and all-cooled policy above. + if (!selection.accountId || !getEligibleAnthropicAccounts(now).includes(selection.accountId)) { + const active = getAccountSet("anthropic")?.activeAccountId; + // Pool-off fresh admission resolves the active credential first; retain its pause + // refusal when unusable survivors left that paused account selected. + if (!isAnthropicAccountPoolEnabled(config) && active + && getAccountCredentialWithStatus("anthropic", active)?.paused) throw new OAuthAccountPausedError(); + throw new OAuthLoginRequiredError("anthropic"); + } return selection.accountId; }; const refreshDispatchAdapter = async (requestParsed: OcxParsedRequest): Promise => { diff --git a/structure/decisions/ADR-6013-anthropic-account-pause.md b/structure/decisions/ADR-6013-anthropic-account-pause.md index 2db38da2a30..4e76e56a4b0 100644 --- a/structure/decisions/ADR-6013-anthropic-account-pause.md +++ b/structure/decisions/ADR-6013-anthropic-account-pause.md @@ -20,6 +20,9 @@ remain distinct from login errors. Cooldown refusals carry the route position and retry seconds across pacing and dispatch, including disabled proactive pools; they never count as an upstream reachability failure. + If only reauthentication-required or unusable credentials remain after pause, dispatch + matches fresh admission: 401 with pooling enabled, or the existing 403 paused-active + refusal when pooling is disabled. Neither refusal changes account or host health. - Why this approach: One authoritative row avoids config/auth split-write races and automatic cleanup follows account deletion. A UI-only toggle would still allow affinity, pool-off failover or a pre-wait bearer to select the paused account. diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index e6fb12839af..22b3c72d954 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -15,7 +15,7 @@ is off. All-paused requests return 403 with resume guidance. Quorum is invalidat and resume; a sent account paused before its 429 may still recover on its sole unpaused successor. Credential resolution, refresh-lock acquisition, selection commit and physical dispatch recheck live eligibility after asynchronous waits. Adapter and passthrough dispatch -preserve typed 403 pause and 429 cooldown refusals after pacing, including pool-off recovery; +preserve typed 401 authentication, 403 pause and 429 cooldown refusals after pacing, including pool-off recovery; they do not report local rejection as 502. Only cooled usable survivors of a strict route produce its scoped 429 and Retry-After, not a login error. An already-dispatched refresh retains a successful rotated credential without unpausing; a late failure cannot mark the diff --git a/tests/adapters/anthropic/anthropic-model-routes.test.ts b/tests/adapters/anthropic/anthropic-model-routes.test.ts index 2229c607d37..faaae0d123f 100644 --- a/tests/adapters/anthropic/anthropic-model-routes.test.ts +++ b/tests/adapters/anthropic/anthropic-model-routes.test.ts @@ -4,9 +4,10 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { acquireOwnedSpendHome } from "../../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../../helpers/remove-tree"; -import { clearAnthropicAccountPoolState, bindAnthropicSessionAffinity, getAnthropicPoolAccessSnapshot, getAnthropicPoolRetryAfterSeconds, promoteAnthropicActiveAccount, resolveAnthropicAccountForSession, rotateAnthropicAccountOn429 } from "../../../src/oauth/anthropic-routing"; +import { clearAnthropicAccountPoolState, bindAnthropicSessionAffinity, getAnthropicAccountHealthSnapshot, getAnthropicPoolAccessSnapshot, getAnthropicPoolRetryAfterSeconds, promoteAnthropicActiveAccount, resolveAnthropicAccountForSession, rotateAnthropicAccountOn429 } from "../../../src/oauth/anthropic-routing"; import { parseAnthropicModelRoutes, resolveAnthropicModelRoute } from "../../../src/oauth/anthropic-model-routes"; -import { captureOAuthAccountSelection, getAccountSet, saveCredential, setAccountPaused, setActiveAccount } from "../../../src/oauth/store"; +import { captureOAuthAccountSelection, getAccountSet, markAccountNeedsReauth, replaceProviderAccountSet, saveAccountCredential, saveCredential, setAccountPaused, setActiveAccount } from "../../../src/oauth/store"; +import { clearUpstreamHostHealth, getUpstreamHostHealth, upstreamHostHealthKey } from "../../../src/codex/upstream-host-health"; import { providerRequestPacingStatus, resetProviderRequestPacingForTest, waitForProviderRequestSlot } from "../../../src/providers/request-pacing"; import { clearAccountQuotaCache, setCachedProviderAccountQuotaForTests } from "../../../src/providers/quota"; import { clearResponseStateForTests } from "../../../src/responses/state"; @@ -27,10 +28,12 @@ beforeEach(() => { releaseSpend = acquireOwnedSpendHome(); sends = []; clearAnthropicAccountPoolState(); + clearUpstreamHostHealth(); clearAccountQuotaCache(); clearResponseStateForTests(); }); afterEach(() => { + clearUpstreamHostHealth(); resetProviderRequestPacingForTest(); releaseSpend(); clearAnthropicAccountPoolState(); @@ -148,6 +151,51 @@ test.each([true, false])("pause while queued for pacing never sends the cached b for (const adapter of ["anthropic", "openai-responses"] as const) { for (const enabled of [true, false]) { + for (const remaining of ["needs-reauth", "unusable"] as const) { + test(`${adapter}: pause with ${remaining} survivors during pacing matches fresh admission, pool enabled=${enabled}`, async () => { + const ids = await seed(); + await setAccountPaused("anthropic", ids[2]!, true); + const cfg = config(ids, () => answer()); + cfg.anthropicAccountPool = { enabled, routes: [{ name: "private-auth-scope", match: "claude-*", accounts: [ids[0]!, ids[1]!] }] }; + cfg.providers.anthropic!.adapter = adapter; + cfg.providers.anthropic!.requestPacing = { enabled: true, maxConcurrentRequests: 1 }; + const hostKey = upstreamHostHealthKey("anthropic", "anthropic-routes.test"); + const slot = await waitForProviderRequestSlot("anthropic", cfg.providers.anthropic!, "claude-sonnet-4-5"); + const pending = post(cfg); + let rosterBeforeDispatch: ReturnType; + try { + for (let i = 0; i < 100 && providerRequestPacingStatus("anthropic", cfg.providers.anthropic!).queued === 0; i++) { + await new Promise(resolve => setTimeout(resolve, 5)); + } + expect(providerRequestPacingStatus("anthropic", cfg.providers.anthropic!).queued).toBe(1); + if (remaining === "needs-reauth") await markAccountNeedsReauth("anthropic", ids[1]!, true); + else { + const credential = getAccountSet("anthropic")!.accounts.find(row => row.id === ids[1])!.credential; + await saveAccountCredential("anthropic", ids[1]!, { ...credential, source: "local-cli", expires: 0 }); + } + await setAccountPaused("anthropic", ids[0]!, true); + if (remaining === "unusable") { + // A persisted background local-CLI slot must not adopt the foreground CLI identity. + await replaceProviderAccountSet("anthropic", { ...getAccountSet("anthropic")!, activeAccountId: ids[0]! }); + } + rosterBeforeDispatch = getAccountSet("anthropic"); + expect(getUpstreamHostHealth(hostKey)).toBeNull(); + } finally { slot.release(); } + const response = await pending; + const initial = await post(cfg); + expect(initial.status).toBe(enabled ? 401 : 403); + expect(response.status).toBe(initial.status); + const body = await response.json() as { error: { type: string; message: string } }; + const initialBody = await initial.json() as { error: { type: string } }; + expect(body.error.type).toBe(initialBody.error.type); + expect(body.error.message).not.toContain("private-auth-scope"); + expect(sends).toEqual([]); + expect(getUpstreamHostHealth(hostKey)).toBeNull(); + expect(ids.map(id => getAnthropicAccountHealthSnapshot(id))).toEqual([null, null, null]); + expect(getAccountSet("anthropic")).toEqual(rosterBeforeDispatch!); + }); + } + test(`${adapter}: a pacing-time pause skips a cooled successor for a healthy account, pool enabled=${enabled}`, async () => { const ids = await seed(); const cfg = config(ids, () => answer()); From 932000958d2c4cf692e1bc41d49b9d1acfb8b242 Mon Sep 17 00:00:00 2001 From: Ingwannu Date: Mon, 28 Sep 2026 17:33:14 +0000 Subject: [PATCH 05/26] test(oauth): drain failed pacing setup --- .../messages-native-oauth.test.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/claude-integration/messages-native-oauth.test.ts b/tests/claude-integration/messages-native-oauth.test.ts index 948ffc82a12..127c9e48f42 100644 --- a/tests/claude-integration/messages-native-oauth.test.ts +++ b/tests/claude-integration/messages-native-oauth.test.ts @@ -177,13 +177,24 @@ describe("managed native Messages over Anthropic OAuth", () => { cfg.providers.anthropic!.requestPacing = { enabled: true, maxConcurrentRequests: 1 }; const slot = await waitForProviderRequestSlot("anthropic", cfg.providers.anthropic!, "claude-sonnet-4-5"); const pending = send(cfg, { ...BODY, stream: false }); + let setupFailed = false; + let setupError: unknown; try { for (let i = 0; i < 100 && providerRequestPacingStatus("anthropic", cfg.providers.anthropic!).queued === 0; i++) { await new Promise(resolve => setTimeout(resolve, 5)); } expect(providerRequestPacingStatus("anthropic", cfg.providers.anthropic!).queued).toBe(1); await setAccountPaused("anthropic", id!, true); - } finally { slot.release(); } + } catch (error) { + setupFailed = true; + setupError = error; + } finally { + slot.release(); + // A failed queue assertion must not leave this request running into afterEach. Drain it, + // but keep the assertion as the useful failure if teardown also rejects. + if (setupFailed) await pending.catch(() => undefined); + } + if (setupFailed) throw setupError; const { response, text } = await pending; expect(response.status).toBe(403); expect(text).toContain("Resume"); From 132770d6145cdc4abadb625cbe840486234126d2 Mon Sep 17 00:00:00 2001 From: Ingwannu Date: Mon, 28 Sep 2026 17:46:59 +0000 Subject: [PATCH 06/26] fix(oauth): align native Messages account admission --- src/oauth/anthropic-routing.ts | 28 +++++++- src/server/messages-native-oauth.ts | 39 +++++------ src/server/messages-native.ts | 32 ++++++--- src/server/responses/request-transport.ts | 28 ++------ .../ADR-6013-anthropic-account-pause.md | 10 +++ structure/providers-and-adapters.md | 4 +- .../anthropic/anthropic-account-pool.test.ts | 13 +++- .../messages-native-oauth.test.ts | 69 ++++++++++++++++++- 8 files changed, 164 insertions(+), 59 deletions(-) diff --git a/src/oauth/anthropic-routing.ts b/src/oauth/anthropic-routing.ts index 361664001ba..dc7bcbc40fa 100644 --- a/src/oauth/anthropic-routing.ts +++ b/src/oauth/anthropic-routing.ts @@ -619,7 +619,10 @@ export function resolveAnthropicAccountForSession( if (!isAnthropicAccountPoolEnabled(config)) { const active = set.accounts.find(account => account.id === set.activeAccountId); // Disabled proactive rotation does not authorize a paused slot or an entirely cooled pool. - return { accountId: active?.paused || isCooled(set.activeAccountId, now) ? eligible[0] ?? null : set.activeAccountId, reason: "pool-disabled" }; + if (active?.paused || isCooled(set.activeAccountId, now)) { + return { accountId: eligible[0] ?? null, reason: eligible.length > 0 ? "only-eligible" : "none" }; + } + return { accountId: set.activeAccountId, reason: "pool-disabled" }; } // A manual choice is a one-dispatch preference, not a lower-priority quota hint. @@ -718,6 +721,29 @@ export function resolveAnthropicAccountForSession( return { accountId, reason, routePosition: decision?.position }; } +/** Shared local refusal policy for Responses and native Messages after asynchronous waits. */ +export async function resolveAnthropicDispatchAccountId( + config: OcxConfig, + sessionKey: string | null = null, + decision: AnthropicRouteDecision | null = null, +): Promise { + const now = Date.now(); + const selection = resolveAnthropicAccountForSession(sessionKey, config, now, decision); + if (selection.reason === "all-cooled") { + throw new AnthropicAccountCooldownError(getAnthropicPoolRetryAfterSeconds(now, decision), decision?.position); + } + if (selection.reason === "paused" || !selection.accountId || !getEligibleAnthropicAccounts(now).includes(selection.accountId)) { + const { OAuthAccountPausedError, OAuthLoginRequiredError } = await import("./index"); + const active = getAccountSet(PROVIDER)?.activeAccountId; + // Pool-off fresh admission resolves the active credential first. Preserve that + // paused-active refusal when no usable survivor could take over from it. + if (selection.reason === "paused" || (!isAnthropicAccountPoolEnabled(config) && active + && getAccountCredentialWithStatus(PROVIDER, active)?.paused)) throw new OAuthAccountPausedError(); + throw new OAuthLoginRequiredError(PROVIDER); + } + return selection.accountId; +} + export function bindAnthropicSessionAffinity( sessionKey: string | null | undefined, accountId: string, diff --git a/src/server/messages-native-oauth.ts b/src/server/messages-native-oauth.ts index 9d69543fca9..d6b5f3faa5b 100644 --- a/src/server/messages-native-oauth.ts +++ b/src/server/messages-native-oauth.ts @@ -4,8 +4,8 @@ * * Credential. The account is the one the existing OAuth selection chooses for this request, by * the same steps the Responses pipeline's transport takes for an unpooled Anthropic OAuth route - * (`prepareResponsesTransport`): capture the committed selection, resolve the active account's - * access snapshot (refreshing it through the OAuth owner when it is due), and commit that + * (`prepareResponsesTransport`): capture the committed selection, resolve an eligible account's + * access snapshot (including pause/cooldown recovery), and commit that * proposal against the captured selection so a concurrent manual switch wins. Nothing here picks * an account of its own, and nothing runs at planning time: the planner and eligibility read * config only, and this module is reached from the lane's dispatch alone. @@ -13,7 +13,8 @@ * Pools. A pooled account set (the opt-in Anthropic account pool, or two or more usable accounts, * which turns on reactive 429 rotation) is declined before this lane is chosen * (`oauth-account-pool`). Should one appear between that decision and dispatch, resolution fails - * closed rather than serve a pooled account without the pool's rotation and affinity. + * closed rather than serve a pooled account without the pool's rotation and affinity. This + * lane-change 409 is distinct from the shared local authentication/pause/cooldown refusals. * * Tool names. An OAuth request carries client tool names under the Claude OAuth prefix, as the * adapter sends them; the answer's `tool_use` names are mapped back here, for exactly the names @@ -21,10 +22,12 @@ * * No token, account id or body content is logged or returned in an error message. */ -import { getValidAccessTokenSnapshot, type OAuthAccessSnapshot } from "../oauth"; +import type { OAuthAccessSnapshot } from "../oauth"; import { commitAnthropicSelectionRouting, getAnthropicPoolAccessSnapshot, + getAnthropicAccountHealthSnapshot, + resolveAnthropicDispatchAccountId, hasAnthropicFailoverQuorum, isAnthropicAccountPoolEnabled, } from "../oauth/anthropic-routing"; @@ -72,17 +75,14 @@ function pooled(config: OcxConfig): boolean { * selection could not be committed. */ export async function resolveNativeOAuthBinding(config: OcxConfig): Promise { - if (pooled(config)) throw new NativeOAuthSelectionChangedError(); - let selection: Selection | null = captureOAuthAccountSelection(PROVIDER); - let candidate = await getValidAccessTokenSnapshot(PROVIDER); for (let attempt = 0; attempt < MAX_SELECTION_ATTEMPTS; attempt++) { - if (!selection) break; - if (candidate.accountId !== selection.accountId) { - // The active account moved between capture and resolution: serve the committed one. - selection = captureOAuthAccountSelection(PROVIDER); - if (!selection) break; - candidate = await getAnthropicPoolAccessSnapshot(selection.accountId); - } + const selection = captureOAuthAccountSelection(PROVIDER); + // Resolve eligibility before the lane guard: a local 401/403/429 must not turn + // into a generic lane-change 409 merely because it appeared during pacing. + const accountId = await resolveAnthropicDispatchAccountId(config); + if (pooled(config)) throw new NativeOAuthSelectionChangedError(); + if (!selection) continue; + const candidate = await getAnthropicPoolAccessSnapshot(accountId); const committed = await commitOAuthAccountSelection(PROVIDER, candidate.accountId, { expectedSelection: selection, expectedCredentialGeneration: candidate.generation, @@ -93,14 +93,12 @@ export async function resolveNativeOAuthBinding(config: OcxConfig): Promise Date.now() + && !getAnthropicAccountHealthSnapshot(binding.snapshot.accountId) && credentialGeneration(row.credential) === binding.snapshot.generation; } diff --git a/src/server/messages-native.ts b/src/server/messages-native.ts index 4bbaeed84a3..80d698efc4d 100644 --- a/src/server/messages-native.ts +++ b/src/server/messages-native.ts @@ -58,9 +58,9 @@ import { selectProactiveApiKeyTransport, transientRetryPolicyFor, } from "../providers/key-failover"; -import { stampApiKeyAccountLabel, stampOAuthAccountLabel } from "../providers/label"; -import { OAuthAccountPausedError, publicOAuthAuthenticationErrorMessage } from "../oauth"; -import { hasAnthropicFailoverQuorum } from "../oauth/anthropic-routing"; +import { stampApiKeyAccountLabel } from "../providers/label"; +import { OAuthAccountPausedError, OAuthLoginRequiredError, publicOAuthAuthenticationErrorMessage } from "../oauth"; +import { AnthropicAccountCooldownError, formatAnthropicProviderForLog, hasAnthropicFailoverQuorum } from "../oauth/anthropic-routing"; import { resolveProtocolSettings } from "../protocols/settings"; import { addProtocolEntryReason, markProtocolBlocked } from "../protocols/trace"; import type { OcxProviderTransport } from "../providers/xai-transport"; @@ -328,6 +328,16 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) finishLog(status, safeMessage); return anthropicErrorResponse(status, safeMessage, type, code); }; + const localOAuthFailure = (error: unknown): Response | null => { + if (error instanceof AnthropicAccountCooldownError) { + const response = fail(429, error.message, "rate_limit_error"); + if (error.retryAfterSeconds !== null) response.headers.set("retry-after", String(error.retryAfterSeconds)); + return response; + } + if (error instanceof OAuthAccountPausedError) return fail(403, publicOAuthAuthenticationErrorMessage(error), "permission_error"); + if (error instanceof OAuthLoginRequiredError) return fail(401, publicOAuthAuthenticationErrorMessage(error), "authentication_error"); + return null; + }; try { await prepareNativeBody(body, req.signal); @@ -366,8 +376,9 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) cleanupAbort(); upstream.abort(); if (req.signal.aborted) return fail(499, "Client cancelled request", "api_error"); + const localRefusal = localOAuthFailure(error); + if (localRefusal) return localRefusal; if (error instanceof NativeOAuthSelectionChangedError) return fail(409, error.message, "api_error"); - if (error instanceof OAuthAccountPausedError) return fail(403, publicOAuthAuthenticationErrorMessage(error), "permission_error"); return fail(401, publicOAuthAuthenticationErrorMessage(error), "authentication_error"); } } else { @@ -377,7 +388,7 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) } let activeProvider: OcxProviderConfig = oauthBinding ? oauthProvider(oauthBinding) : route.provider; stampApiKeyAccountLabel(logCtx, route.providerName, activeProvider); - if (oauthBinding) stampOAuthAccountLabel(logCtx, route.providerName, activeProvider, oauthBinding.snapshot.accountId); + if (oauthBinding) logCtx.provider = formatAnthropicProviderForLog(route.providerName, oauthBinding.snapshot.accountId, config); const spendTracker = attachRequestSpendTracker(req, logCtx); let activeRequest: AnthropicMessagesPassthroughRequest; let retainedRequestBytes = 0; @@ -461,15 +472,17 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) if (oauthBinding) { // The OAuth twin of the key check below: re-resolve through the same selection // owner when the committed account or its credential moved since the build. - if (!nativeOAuthBindingIsCurrent(oauthBinding)) { + for (let attempt = 0; !nativeOAuthBindingIsCurrent(oauthBinding); attempt++) { + if (attempt >= 3) throw new NativeOAuthSelectionChangedError(); try { oauthBinding = await resolveNativeOAuthBinding(config); } catch (error) { - if (error instanceof OAuthAccountPausedError) throw error; + if (error instanceof OAuthAccountPausedError || error instanceof OAuthLoginRequiredError + || error instanceof AnthropicAccountCooldownError) throw error; throw new NativeOAuthSelectionChangedError(); } rebuildFor(oauthProvider(oauthBinding)); - stampOAuthAccountLabel(logCtx, route.providerName, activeProvider, oauthBinding.snapshot.accountId); + logCtx.provider = formatAnthropicProviderForLog(route.providerName, oauthBinding.snapshot.accountId, config); } } else if (!providerApiKeySelectionIsCurrent(config, route.providerName, activeProvider)) { const current = resolveCurrentProviderApiKeyTransport(config, route.providerName, activeProvider); @@ -571,8 +584,9 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) finishLog(429); return refusal; } + const localRefusal = localOAuthFailure(sendError); + if (localRefusal) return localRefusal; if (sendError instanceof NativeOAuthSelectionChangedError) return fail(409, sendError.message, "api_error"); - if (sendError instanceof OAuthAccountPausedError) return fail(403, publicOAuthAuthenticationErrorMessage(sendError), "permission_error"); if (sendError instanceof NativeOpaqueStateRefusal) { logCtx.errorCode = "unsupported_feature"; return fail(400, sendError.message, "invalid_request_error", "unsupported_feature"); diff --git a/src/server/responses/request-transport.ts b/src/server/responses/request-transport.ts index 0c46b96a533..c340c0612a9 100644 --- a/src/server/responses/request-transport.ts +++ b/src/server/responses/request-transport.ts @@ -15,7 +15,7 @@ import type { AdapterEvent, OcxParsedRequest, OcxProviderConfig, OcxUsage } from import type { AnthropicAccountSelectionReason } from "../../oauth/anthropic-routing"; import { resolveAnthropicModelRoute, routeCandidates, type AnthropicRouteDecision } from "../../oauth/anthropic-model-routes"; import { - AnthropicAccountCooldownError, + resolveAnthropicDispatchAccountId, isAnthropicAccountPoolEnabled, getAnthropicPoolAccessSnapshot, getAnthropicAccountHealthSnapshot, @@ -400,26 +400,6 @@ export async function prepareResponsesTransport( } return resolved; }; - const resolveAnthropicDispatchAccount = (): string | null => { - if (route.providerName !== "anthropic") return null; - const now = Date.now(); - const selection = resolveAnthropicAccountForSession(anthropicSessionKey, config, now, anthropicRouteDecision); - if (selection.reason === "paused") throw new OAuthAccountPausedError(); - if (selection.reason === "all-cooled") { - throw new AnthropicAccountCooldownError(getAnthropicPoolRetryAfterSeconds(now, anthropicRouteDecision), anthropicRouteDecision?.position); - } - // No usable credential is an authentication refusal, not failed host reachability. - // Keep it distinct from the explicit all-paused and all-cooled policy above. - if (!selection.accountId || !getEligibleAnthropicAccounts(now).includes(selection.accountId)) { - const active = getAccountSet("anthropic")?.activeAccountId; - // Pool-off fresh admission resolves the active credential first; retain its pause - // refusal when unusable survivors left that paused account selected. - if (!isAnthropicAccountPoolEnabled(config) && active - && getAccountCredentialWithStatus("anthropic", active)?.paused) throw new OAuthAccountPausedError(); - throw new OAuthLoginRequiredError("anthropic"); - } - return selection.accountId; - }; const refreshDispatchAdapter = async (requestParsed: OcxParsedRequest): Promise => { if (route.provider.authMode === "oauth") { // Pacing may outlive admission. Preserve local pause/cooldown reasons even when @@ -427,11 +407,11 @@ export async function prepareResponsesTransport( let candidate = servingOAuthSnapshot; if (route.providerName === "anthropic") { oauthSelection = captureOAuthAccountSelection(route.providerName); - const accountId = resolveAnthropicDispatchAccount(); - candidate = accountId ? await getAnthropicPoolAccessSnapshot(accountId) : undefined; + const accountId = await resolveAnthropicDispatchAccountId(config, anthropicSessionKey, anthropicRouteDecision); + candidate = await getAnthropicPoolAccessSnapshot(accountId); } if (!candidate || !await applyFailoverSnapshot(candidate, requestParsed)) { - resolveAnthropicDispatchAccount(); + if (route.providerName === "anthropic") await resolveAnthropicDispatchAccountId(config, anthropicSessionKey, anthropicRouteDecision); throw new Error("OAuth account selection changed before dispatch"); } } else { diff --git a/structure/decisions/ADR-6013-anthropic-account-pause.md b/structure/decisions/ADR-6013-anthropic-account-pause.md index 4e76e56a4b0..5d5fb1f005c 100644 --- a/structure/decisions/ADR-6013-anthropic-account-pause.md +++ b/structure/decisions/ADR-6013-anthropic-account-pause.md @@ -23,6 +23,16 @@ If only reauthentication-required or unusable credentials remain after pause, dispatch matches fresh admission: 401 with pooling enabled, or the existing 403 paused-active refusal when pooling is disabled. Neither refusal changes account or host health. + Native Messages uses the same dispatch eligibility resolver, checks cooldown after each + await, and preserves typed 401/403/429 responses. Its existing 409 remains reserved for a + healthy roster becoming pooled and requiring the bridge lane, or repeated selection races. +- Pool-off policy evidence: #6013's accepted contract retains reactive 429 successors with + pooling disabled. The existing `anthropicAccountPool.enabled` reference likewise gates + only proactive routing, not 429 recovery. Cooldown is recorded from an upstream 429; using + a healthy successor for a paused or already-cooled active account is recovery, not quota + rotation. A healthy active account remains selected regardless of quota/strategy settings. + Recovery proposals say `only-eligible`; native request logs carry the committed account's + ordinal rather than a provider-only label. The native/bridge pacing matrix pins this policy. - Why this approach: One authoritative row avoids config/auth split-write races and automatic cleanup follows account deletion. A UI-only toggle would still allow affinity, pool-off failover or a pre-wait bearer to select the paused account. diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index 22b3c72d954..e0d4a49f68c 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -14,13 +14,13 @@ manual, affinity, model-route and reactive 429 candidates, including when proact is off. All-paused requests return 403 with resume guidance. Quorum is invalidated on pause and resume; a sent account paused before its 429 may still recover on its sole unpaused successor. Credential resolution, refresh-lock acquisition, selection commit and physical -dispatch recheck live eligibility after asynchronous waits. Adapter and passthrough dispatch +dispatch recheck live eligibility after asynchronous waits. Responses and native Messages preserve typed 401 authentication, 403 pause and 429 cooldown refusals after pacing, including pool-off recovery; they do not report local rejection as 502. Only cooled usable survivors of a strict route produce its scoped 429 and Retry-After, not a login error. An already-dispatched refresh retains a successful rotated credential without unpausing; a late failure cannot mark the paused row for reauthentication. Token Guardian and quota probes use their existing pause -guards. +guards. Pool-off keeps a healthy active account; pause/prior-429 recovery uses `only-eligible`, and logs name the committed account. > Decision record: [ADR-6013](decisions/ADR-6013-anthropic-account-pause.md) diff --git a/tests/adapters/anthropic/anthropic-account-pool.test.ts b/tests/adapters/anthropic/anthropic-account-pool.test.ts index 61396bd71e9..d56f869a070 100644 --- a/tests/adapters/anthropic/anthropic-account-pool.test.ts +++ b/tests/adapters/anthropic/anthropic-account-pool.test.ts @@ -252,7 +252,7 @@ describe("anthropic account pool", () => { expect(resolveAnthropicAccountForSession("uncommitted", quota).accountId).toBe(bId); }); - test("default off always returns the active account", async () => { + test("default off retains the healthy active account", async () => { const { aId, bId } = await seedTwoAccounts(); expect(isAnthropicAccountPoolEnabled(cfg(false))).toBe(false); const sel = resolveAnthropicAccountForSession("session-1", cfg(false)); @@ -261,6 +261,17 @@ describe("anthropic account pool", () => { expect(sel.accountId).not.toBe(bId); }); + test("pool-off ignores proactive preferences but names recovery from an upstream 429", async () => { + const { aId, bId } = await seedTwoAccounts(); + const config = cfg(false, 80, { strategy: "round-robin" }); + setCachedProviderAccountQuotaForTests("anthropic", aId, { fiveHourPercent: 95 }); + setCachedProviderAccountQuotaForTests("anthropic", bId, { fiveHourPercent: 5 }); + expect(resolveAnthropicAccountForSession("new", config)).toMatchObject({ accountId: aId, reason: "pool-disabled" }); + expect(rotateAnthropicAccountOn429(config, aId, "60")).toBe(bId); + expect(resolveAnthropicAccountForSession("new", config)).toMatchObject({ accountId: bId, reason: "only-eligible" }); + expect(getAccountSet("anthropic")!.activeAccountId).toBe(aId); // A proposal is not a committed send. + }); + test("affinity sticks across resolves until cooled", async () => { const { aId, bId } = await seedTwoAccounts(); // Force lowest-usage toward B for a cold start with high active usage. diff --git a/tests/claude-integration/messages-native-oauth.test.ts b/tests/claude-integration/messages-native-oauth.test.ts index 127c9e48f42..18f03268a78 100644 --- a/tests/claude-integration/messages-native-oauth.test.ts +++ b/tests/claude-integration/messages-native-oauth.test.ts @@ -12,8 +12,9 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { saveConfig } from "../../src/config"; import { ANTHROPIC_OAUTH_BETA, CLAUDE_CODE_SYSTEM_INSTRUCTION } from "../../src/oauth/anthropic"; -import { clearAnthropicAccountPoolState, forgetAnthropicFailoverQuorum } from "../../src/oauth/anthropic-routing"; -import { getAccountSet, markAccountNeedsReauth, saveCredential, setAccountPaused, setActiveAccount } from "../../src/oauth/store"; +import { clearAnthropicAccountPoolState, forgetAnthropicFailoverQuorum, formatAnthropicProviderForLog, getAnthropicAccountHealthSnapshot, rotateAnthropicAccountOn429 } from "../../src/oauth/anthropic-routing"; +import { getAccountSet, markAccountNeedsReauth, replaceProviderAccountSet, saveAccountCredential, saveCredential, setAccountPaused, setActiveAccount } from "../../src/oauth/store"; +import { clearUpstreamHostHealth, getUpstreamHostHealth, upstreamHostHealthKey } from "../../src/codex/upstream-host-health"; import { handleClaudeMessages } from "../../src/server/claude-messages"; import { getRequestLogEntries } from "../../src/server/request-log"; import { providerRequestPacingStatus, resetProviderRequestPacingForTest, waitForProviderRequestSlot } from "../../src/providers/request-pacing"; @@ -50,11 +51,13 @@ beforeEach(() => { throw new Error("unexpected global fetch in the native OAuth Messages test"); }) as unknown as typeof fetch; clearAnthropicAccountPoolState(); + clearUpstreamHostHealth(); forgetAnthropicFailoverQuorum(); releaseSpendHome = acquireOwnedSpendHome(); }); afterEach(() => { + clearUpstreamHostHealth(); resetProviderRequestPacingForTest(); releaseSpendHome?.(); releaseSpendHome = undefined; @@ -171,6 +174,68 @@ async function send(config: OcxConfig, body: Record) { } describe("managed native Messages over Anthropic OAuth", () => { + for (const native of [true, false]) for (const enabled of [true, false]) { + for (const state of ["needs-reauth", "unusable", "cooled", "paused-cooled", "healthy"] as const) { + test(`Messages native=${native}, pool=${enabled}: pacing ${state} preserves fresh admission`, async () => { + const ids = await seed(2); + await markAccountNeedsReauth("anthropic", ids[1]!, true); + forgetAnthropicFailoverQuorum(); + const cfg = fixtureConfig({ oauthSwitch: native }); + cfg.anthropicAccountPool = { enabled }; + cfg.providers.anthropic!.requestPacing = { enabled: true, maxConcurrentRequests: 1 }; + const slot = await waitForProviderRequestSlot("anthropic", cfg.providers.anthropic!, "claude-sonnet-4-5"); + const pending = send(cfg, { ...BODY, stream: false }); + let setupFailed = false; + let setupError: unknown; + let health: ReturnType[] = []; + let roster: ReturnType; + try { + for (let i = 0; i < 100 && providerRequestPacingStatus("anthropic", cfg.providers.anthropic!).queued === 0; i++) { + await new Promise(resolve => setTimeout(resolve, 5)); + } + expect(providerRequestPacingStatus("anthropic", cfg.providers.anthropic!).queued).toBe(1); + const coolingConfig = { ...cfg, anthropicAccountPool: { enabled: true } }; + if (state === "cooled") rotateAnthropicAccountOn429(coolingConfig, ids[0]!, "60"); + else { + await markAccountNeedsReauth("anthropic", ids[1]!, false); + await setAccountPaused("anthropic", ids[0]!, true); + if (state === "needs-reauth") await markAccountNeedsReauth("anthropic", ids[1]!, true); + if (state === "paused-cooled") rotateAnthropicAccountOn429(coolingConfig, ids[1]!, "60"); + if (state === "unusable") { + await saveAccountCredential("anthropic", ids[1]!, { ...credential(1), source: "local-cli", expires: 0 }); + await replaceProviderAccountSet("anthropic", { ...getAccountSet("anthropic")!, activeAccountId: ids[0]! }); + } + } + health = ids.map(id => getAnthropicAccountHealthSnapshot(id)); + roster = getAccountSet("anthropic"); + } catch (error) { setupFailed = true; setupError = error; } + finally { slot.release(); if (setupFailed) await pending.catch(() => undefined); } + if (setupFailed) throw setupError; + const result = await pending; + const initial = await send(cfg, { ...BODY, stream: false }); + const expected = state === "healthy" ? 200 : state === "cooled" || state === "paused-cooled" ? 429 + : state === "unusable" && !enabled ? 403 : 401; + expect(initial.response.status).toBe(expected); + expect(result.response.status).toBe(initial.response.status); + if (expected !== 200) { + expect(JSON.parse(result.text).error.type).toBe(JSON.parse(initial.text).error.type); + expect(sent).toEqual([]); + expect(getAccountSet("anthropic")).toEqual(roster!); + } else { + expect(sent).toHaveLength(2); + expect(sent.every(entry => entry.headers.get("authorization") === `Bearer ${credential(1).access}`)).toBe(true); + expect(result.row.provider).toBe(formatAnthropicProviderForLog("anthropic", ids[1])); + } + if (expected === 429) { + expect(Number(result.response.headers.get("retry-after"))).toBeGreaterThan(50); + expect(Number(result.response.headers.get("retry-after"))).toBeLessThanOrEqual(60); + } + expect(ids.map(id => getAnthropicAccountHealthSnapshot(id))).toEqual(health); + expect(getUpstreamHostHealth(upstreamHostHealthKey("anthropic", "api.anthropic.com"))).toBeNull(); + }); + } + } + test("a singleton paused during native pacing returns 403 without dispatch", async () => { const [id] = await seed(1); const cfg = fixtureConfig(); From 73289d46ae3c93d06b9add99d1c834d2e5733d9a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 30 Sep 2026 01:25:50 +0900 Subject: [PATCH 07/26] chore(release): open dev at 2.73.0 before releasing 2.72.0 (#6243) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- desktop/src-tauri/Cargo.lock | 2 +- desktop/src-tauri/Cargo.toml | 2 +- desktop/src-tauri/tauri.conf.json | 2 +- package.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index 8d650cfc89d..33e014aeeb0 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -2645,7 +2645,7 @@ dependencies = [ [[package]] name = "opencodex-desktop" -version = "2.72.0" +version = "2.73.0" dependencies = [ "base64 0.22.1", "dbus", diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index 0b995e022c6..274c198474d 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "opencodex-desktop" -version = "2.72.0" +version = "2.73.0" description = "OpenCodex desktop shell" authors = ["OpenCodex contributors"] license = "MIT" diff --git a/desktop/src-tauri/tauri.conf.json b/desktop/src-tauri/tauri.conf.json index c3058739b74..0242bc5d457 100644 --- a/desktop/src-tauri/tauri.conf.json +++ b/desktop/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "OpenCodex", - "version": "2.72.0", + "version": "2.73.0", "identifier": "com.opencodex.desktop", "build": { "frontendDist": "../ui", diff --git a/package.json b/package.json index ab92796acfa..9bf34d94c31 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@bitkyc08/opencodex", - "version": "2.72.0", + "version": "2.73.0", "description": "Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code", "type": "module", "main": "./bin/package-main.mjs", From 41cc5defd02380d5c563381a2db47bf75f8c962e Mon Sep 17 00:00:00 2001 From: JUN Date: Wed, 30 Sep 2026 03:10:46 +0900 Subject: [PATCH 08/26] fix(oauth): recheck Anthropic pause before auxiliary sends --- scripts/test-layout/layout.json | 1 + src/codex/catalog/gather-capture.ts | 2 + src/codex/catalog/provider-models.ts | 24 +++- src/lib/provider-outbound.ts | 13 ++ src/providers/quota.ts | 8 +- src/providers/quota/vendor-probes-oauth.ts | 9 +- structure/catalog.md | 2 +- structure/providers-and-adapters.md | 6 +- structure/transports/inventory.md | 2 +- .../anthropic-account-pause-outbound.test.ts | 127 ++++++++++++++++++ tests/fixtures/test-layout-expected.json | 1 + 11 files changed, 183 insertions(+), 12 deletions(-) create mode 100644 tests/adapters/anthropic/anthropic-account-pause-outbound.test.ts diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index c8ff7e50269..73d0450b20d 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -99,6 +99,7 @@ "ambiguous-resend-composition.test.ts": "lib", "ambiguous-resend-gate.test.ts": "lib", "anthropic-account-pool.test.ts": "adapters/anthropic", + "anthropic-account-pause-outbound.test.ts": "adapters/anthropic", "anthropic-account-pause.test.ts": "adapters/anthropic", "anthropic-model-routes.test.ts": "adapters/anthropic", "anthropic-agentrouter-language-framing.test.ts": "adapters/anthropic", diff --git a/src/codex/catalog/gather-capture.ts b/src/codex/catalog/gather-capture.ts index 7f67743cc0c..64eadad1c2e 100644 --- a/src/codex/catalog/gather-capture.ts +++ b/src/codex/catalog/gather-capture.ts @@ -120,6 +120,8 @@ export interface CatalogGatherProviderModelOutcome { export interface ModelsAuthResolution { readonly apiKey: string | undefined; readonly observed: boolean; + readonly oauthAccountId?: string; + readonly oauthGeneration?: string; readonly oauthApiBaseUrl?: string; readonly oauthProjectId?: string; } diff --git a/src/codex/catalog/provider-models.ts b/src/codex/catalog/provider-models.ts index f43b20904c0..a3db3bb898f 100644 --- a/src/codex/catalog/provider-models.ts +++ b/src/codex/catalog/provider-models.ts @@ -6,7 +6,7 @@ import { copyFileSync, existsSync, mkdirSync, readFileSync, realpathSync } from import { delimiter, dirname, join, resolve } from "node:path"; import { atomicWriteFile, expandUserPath, getConfigDir, websocketsEnabled } from "../../config"; import { resolveProviderApiKey } from "../../providers/key-store"; -import { getAccountSet } from "../../oauth/store"; +import { captureOAuthAccountSelection, credentialGeneration, getAccountCredentialWithStatus, getAccountSet } from "../../oauth/store"; import { readKiroAccountModels, kiroObservedContextWindow } from "../../providers/kiro-model-catalog"; import { CODEX_CONFIG_PATH, CODEX_MODELS_CACHE_PATH, DEFAULT_CATALOG_PATH, readRootTomlString, resolveCodexConfigPath } from "../paths"; import { @@ -74,6 +74,7 @@ import { import type { NormalizedComboConfig } from "../../combos/types"; import { ProviderOutboundPolicyError, + ProviderOutboundSendCancelledError, providerOutboundGet, providerOutboundPost, providerRedirectError, @@ -149,6 +150,8 @@ export function observedModelsAuthResolver( return { apiKey: observation.snapshot.accessToken, observed: true, + oauthAccountId: observation.snapshot.accountId, + oauthGeneration: observation.snapshot.generation, ...(observation.snapshot.apiBaseUrl ? { oauthApiBaseUrl: observation.snapshot.apiBaseUrl } : {}), ...(observation.snapshot.projectId ? { oauthProjectId: observation.snapshot.projectId } : {}), }; @@ -174,6 +177,8 @@ export async function fetchProviderModelsWithAuth( // generation, so a request started with the former account cannot later publish its result. const cacheGeneration = captureModelCacheGeneration(name); const isCurrentCacheGeneration = () => isModelCacheGenerationCurrent(name, cacheGeneration); + const anthropicSelection = name === "anthropic" && prov.authMode === "oauth" + ? captureOAuthAccountSelection(name) : null; if (prov.authMode === "forward") return observed([], "authoritative"); // ChatGPT backend has no /models const seedVertexDefault = prov.adapter === "google" && prov.googleMode === "vertex" @@ -256,6 +261,8 @@ export async function fetchProviderModelsWithAuth( .then(snapshot => ({ apiKey: snapshot.accessToken, observed: false, + oauthAccountId: snapshot.accountId, + oauthGeneration: snapshot.generation, ...(snapshot.apiBaseUrl ? { oauthApiBaseUrl: snapshot.apiBaseUrl } : {}), ...(snapshot.projectId ? { oauthProjectId: snapshot.projectId } : {}), })) @@ -263,6 +270,15 @@ export async function fetchProviderModelsWithAuth( : { apiKey: await resolveModelsAuthToken(name, prov), observed: false } : resolveAuth.resolve(name, prov)); const apiKey = auth.apiKey; + const maySendAnthropicDiscovery = () => { + if (name !== "anthropic" || prov.authMode !== "oauth") return true; + const selected = captureOAuthAccountSelection(name); + const row = auth.oauthAccountId ? getAccountCredentialWithStatus(name, auth.oauthAccountId) : null; + return !!anthropicSelection && !!selected && !!row && !row.paused && !row.needsReauth + && selected.accountId === anthropicSelection.accountId && selected.revision === anthropicSelection.revision + && row.credential.expires > Date.now() && auth.oauthAccountId === selected.accountId + && credentialGeneration(row.credential) === auth.oauthGeneration && row.credential.access === apiKey; + }; // A configured default is a real callable selector and must remain discoverable when a // compatible provider's live /models request fails (issue #308). Static providers already seed // their default selector above when no explicit model list exists. @@ -609,7 +625,8 @@ export async function fetchProviderModelsWithAuth( // proof is on the URL — not the provider name — because an OAuth/forward // name matches any baseUrl by design. Retargeted or renamed custom rows // fetch a different URL and keep the rejection. - const outboundDependencies = { isCanonicalUrl: isRegistryModelDiscoveryUrl }; + if (!maySendAnthropicDiscovery()) return observed(withConfiguredRetention(failedDiscoveryConfigured), "degraded"); + const outboundDependencies = { isCanonicalUrl: isRegistryModelDiscoveryUrl, beforeSend: maySendAnthropicDiscovery }; const res = request.method === "POST" ? await providerOutboundPost(name, prov, url, { headers, @@ -774,6 +791,9 @@ export async function fetchProviderModelsWithAuth( markProviderDiscoveryOk(name, liveModelCount); return observed(returned, "authoritative"); } catch (error) { + if (error instanceof ProviderOutboundSendCancelledError) { + return observed(withConfiguredRetention(failedDiscoveryConfigured), "degraded"); + } if (error instanceof ProviderOutboundPolicyError) { const { models, fallback, shouldLog } = failedDiscoveryFallback({ reason: "blocked" }); if (shouldLog) { diff --git a/src/lib/provider-outbound.ts b/src/lib/provider-outbound.ts index 995e9b4c8fb..87b752b71f0 100644 --- a/src/lib/provider-outbound.ts +++ b/src/lib/provider-outbound.ts @@ -28,12 +28,18 @@ export interface ProviderOutboundDependencies { * that forgets the seam fails closed, never open. */ isCanonicalUrl?: (name: string, url: string) => boolean; + /** Recheck caller-owned credential authority after DNS and immediately before transport. */ + beforeSend?: () => boolean; } export class ProviderOutboundPolicyError extends Error { override readonly name = "ProviderOutboundPolicyError"; } +export class ProviderOutboundSendCancelledError extends Error { + override readonly name = "ProviderOutboundSendCancelledError"; +} + function pickPinnedAddress(addresses: Array<{ address: string; family: number }>): { address: string; family: number } { return addresses.find(address => address.family === 4) ?? addresses[0]!; } @@ -157,6 +163,9 @@ async function providerOutboundRequest( // See PROVIDER_OUTBOUND_DEFAULT_USER_AGENT: this wrapper only carries proxy-originated // diagnostic traffic, so it identifies itself unless the caller already did. const init = withDefaultOutboundUserAgent(rawInit); + const assertSendAllowed = () => { + if (dependencies.beforeSend?.() === false) throw new ProviderOutboundSendCancelledError("provider credential changed before send"); + }; const postUrl = method === "POST" ? new URL(url) : undefined; if (postUrl?.protocol !== undefined && postUrl.protocol !== "https:") { throw new ProviderOutboundPolicyError("provider POST URL must use HTTPS"); @@ -195,6 +204,7 @@ async function providerOutboundRequest( }); if (destinationError) throw new ProviderOutboundPolicyError(destinationError); } + assertSendAllowed(); return provider.fetch(url, { ...init, method, redirect: "manual" }); } const parsed = postUrl ?? new URL(url); @@ -272,6 +282,7 @@ async function providerOutboundRequest( warnProxyDnsDegradationOnce(); // An explicit provider proxy stays pinned through the degradation too; re-inferring the // route from the environment here would quietly move the request to a different exit. + assertSendAllowed(); return configuredOutboundFetch(url, { ...init, method, redirect: "manual", ...(providerProxy ? { proxy: providerProxy } : {}), @@ -286,6 +297,7 @@ async function providerOutboundRequest( // An explicit provider proxy is always pinned, for the same reason and unconditionally: // the operator named the exit for this provider, so the environment must not re-decide it. const proxy = providerProxy ?? ((allowMihomoIpv6FakeIp && bindingProxy) ? bindingProxy : undefined); + assertSendAllowed(); return configuredOutboundFetch(url, { ...init, method, redirect: "manual", ...(proxy ? { proxy } : {}) }); } if (proxyApplies && resolved.privateNetwork) { @@ -300,6 +312,7 @@ async function providerOutboundRequest( context: "provider response", }; const pinned = pickPinnedAddress(resolved.addresses); + assertSendAllowed(); if (method === "POST") { return pinnedPost(url, pinned, (init as ProviderPostInit).body, init.signal ?? undefined, requestOptions); } diff --git a/src/providers/quota.ts b/src/providers/quota.ts index 5b7236f0b53..494c97543c1 100644 --- a/src/providers/quota.ts +++ b/src/providers/quota.ts @@ -1,6 +1,6 @@ import { listCodexAuthAccountsSnapshot } from "../codex/auth-api"; import { resolveEnvValue } from "../config"; -import { getAccountCredential, getAccountSet } from "../oauth/store"; +import { getAccountCredential, getAccountCredentialWithStatus, getAccountSet } from "../oauth/store"; import { apiKeyPoolEntryId } from "./api-keys"; import { captureConfigGeneration, sweepExpiredOnWrite } from "../lib/state-store-sweeper"; import { ACCOUNT_QUOTA_TTL_MS, CACHE_TTL_MS } from "./quota-wire"; @@ -403,7 +403,6 @@ async function fetchExplicitAccountQuota(provider: string, accountId: string, fo accountQuotaInflight.set(flightKey, flight); return flight; } - async function fetchExplicitCurrentQuota(provider: string, config: OcxProviderConfig, liveConfig: OcxConfig): Promise { const id = getAccountSet(provider)?.activeAccountId; if (!id) return null; @@ -425,6 +424,7 @@ async function fetchAccountQuota( if (explicitAccountReader(provider)) return fetchExplicitAccountQuota(provider, accountId, forceRefresh, providerConfig); if (provider === "anthropic" || provider === "kiro") hydrateAccountQuotaCache(); const key = accountCacheKey(provider, accountId); + const selectionRevision = provider === "anthropic" ? getAccountSet(provider)?.selectionRevision : undefined; const writerGeneration = captureConfigGeneration(); const kiroIdentity = provider === "kiro" ? kiroProbeIdentity(accountId) : undefined; const cachedCandidate = accountQuotaCache.get(key); @@ -439,7 +439,6 @@ async function fetchAccountQuota( return provider !== "kiro" || joined.identity === kiroIdentity ? joined : fetchAccountQuota(provider, accountId, true, providerConfig); } - const epoch = explicitAccountEpoch; const probe = (async (): Promise => { let diagnosticIdentity: string | undefined; @@ -469,6 +468,8 @@ async function fetchAccountQuota( quota = result.kind === "available" ? result.quota : null; if (result.kind === "unavailable") quotaFailure = result.failure; } else if (provider === "anthropic") { + const row = getAccountCredentialWithStatus(provider, accountId); + if (!row || row.paused || row.needsReauth || row.credential.access !== token || getAccountSet(provider)?.selectionRevision !== selectionRevision) return { ts: Date.now(), quota: null, unavailable: true }; quota = await fetchAnthropicUsageQuota(token); } else { return { ts: Date.now(), quota: null, unavailable: true }; @@ -524,7 +525,6 @@ async function fetchAccountQuota( accountQuotaInflight.set(key, probe); return probe; } - /** * Per-account quota rows for a provider's logged-in accounts. Probes run in parallel; a * single failing account never blocks the others. diff --git a/src/providers/quota/vendor-probes-oauth.ts b/src/providers/quota/vendor-probes-oauth.ts index 56ad530a7b8..134075f9820 100644 --- a/src/providers/quota/vendor-probes-oauth.ts +++ b/src/providers/quota/vendor-probes-oauth.ts @@ -1,7 +1,7 @@ import { effectiveCodexAuthAccountId, fetchMainAccountInfoSnapshot, listCodexAuthAccountsSnapshot } from "../../codex/auth-api"; import { MAIN_CODEX_ACCOUNT_ID } from "../../codex/main-account"; import { getValidAccessToken } from "../../oauth"; -import { getAccountCredential, getAccountSet } from "../../oauth/store"; +import { captureOAuthAccountSelection, getAccountCredential, getAccountCredentialWithStatus, getAccountSet } from "../../oauth/store"; import { hydrateKiroAccountState, persistKiroAccountState } from "../kiro-account-state-disk"; import { kiroProbeCurrent, kiroProbeIdentity } from "./kiro-account-probe"; import { fetchMuseKeyQuotaSnapshot } from "../muse-key-quota"; @@ -339,7 +339,8 @@ export async function fetchAnthropicUsageQuota(accessToken: string): Promise { // Capture the account we intend to probe before awaiting — a mid-flight active // switch must not seed the wrong account's cache with this response. - const probedAccountId = getAccountSet("anthropic")?.activeAccountId; + const selection = captureOAuthAccountSelection("anthropic"); + const probedAccountId = selection?.accountId; const probedAccountKey = probedAccountId ? accountCacheKey("anthropic", probedAccountId) : null; const writerGeneration = captureConfigGeneration(); let accessToken: string; @@ -348,6 +349,10 @@ export async function fetchAnthropicQuota(provider: string): Promise Decision record: [ADR-6013](decisions/ADR-6013-anthropic-account-pause.md) diff --git a/structure/transports/inventory.md b/structure/transports/inventory.md index dfdb4a86cd8..0b6a2503982 100644 --- a/structure/transports/inventory.md +++ b/structure/transports/inventory.md @@ -154,7 +154,7 @@ by transports that can preserve that request-local decision: | Request path | Per-provider route | Current contract | | --- | --- | --- | | Main routed inference through `providerFetch` in `src/server/responses/fetch-helpers.ts` | Honoured | Applied at the physical send in `sendWithConnectionPolicy`, so a request rebuilt against a different destination or a reselected provider transport resolves its route against the destination actually used. The built-in executor passes direct, HTTP(S)-proxy, and SOCKS5(H)-proxy choices through `configuredOutboundFetch` in `src/lib/proxy-env.ts`; an inherited route leaves the global decision unchanged. Native Chat in `src/server/chat-native.ts` and the Responses transport in `src/server/responses/request-transport.ts` bind their own sends. | -| Every caller of `providerOutboundGet` or `providerOutboundPost` in `src/lib/provider-outbound.ts` | Honoured | This includes provider discovery and model-catalog gathering in `src/codex/catalog/provider-models.ts`, management provider tests in `src/server/management/provider-routes.ts`, and the Ollama show probe in `src/providers/ollama-show.ts`. | +| Every caller of `providerOutboundGet` or `providerOutboundPost` in `src/lib/provider-outbound.ts` | Honoured | This includes provider discovery and model-catalog gathering in `src/codex/catalog/provider-models.ts`, management provider tests in `src/server/management/provider-routes.ts`, and the Ollama show probe in `src/providers/ollama-show.ts`. An optional caller authorization predicate is rechecked after DNS and immediately before each physical send, including injected executors and proxy fallback. | | API-key quota probes in `src/providers/quota/vendor-probes-key.ts` | Honoured | Each probe receives its provider config and sends through `configuredOutboundFetch` with the resolved route, so a quota reading and the inference it describes leave by the same exit. | | OAuth token exchange and refresh under `src/oauth/` | Not honoured | These reach fixed vendor endpoints from modules that hold no provider config, so no provider route is in scope at the call site. A provider pinned to its own proxy or to direct still refreshes credentials by the process-wide route. | | OAuth-backed quota probes in `src/providers/quota/vendor-probes-oauth.ts` and `src/providers/quota/devin.ts` | Not honoured | `fetchXaiQuota`, `fetchAnthropicQuota`, `fetchCursorQuota`, `fetchDevinQuota` and their neighbours receive a provider name and a token rather than a provider config. | diff --git a/tests/adapters/anthropic/anthropic-account-pause-outbound.test.ts b/tests/adapters/anthropic/anthropic-account-pause-outbound.test.ts new file mode 100644 index 00000000000..5c6986207fb --- /dev/null +++ b/tests/adapters/anthropic/anthropic-account-pause-outbound.test.ts @@ -0,0 +1,127 @@ +import { afterEach, beforeEach, expect, spyOn, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as oauth from "../../../src/oauth"; +import { captureProviderGather } from "../../../src/codex/catalog/gather-capture"; +import { fetchProviderModelsWithAuth } from "../../../src/codex/catalog/provider-models"; +import { ProviderOutboundSendCancelledError, providerOutboundGet } from "../../../src/lib/provider-outbound"; +import { clearAnthropicAccountPoolState } from "../../../src/oauth/anthropic-routing"; +import { credentialGeneration, getAccountCredential, getAccountSet, saveAccountCredential, saveCredential, setAccountPaused } from "../../../src/oauth/store"; +import { clearAccountQuotaCache, fetchProviderAccountQuotas } from "../../../src/providers/quota"; +import * as accountCache from "../../../src/providers/quota/account-cache"; +import { fetchAnthropicQuota } from "../../../src/providers/quota/vendor-probes-oauth"; +import { removeTreeWithRetry } from "../../helpers/remove-tree"; + +const previousHome = process.env.OPENCODEX_HOME; +const originalFetch = globalThis.fetch; +let home: string; +let accountId: string; +let sends: number; +beforeEach(async () => { + home = mkdtempSync(join(tmpdir(), "ocx-anthropic-pause-outbound-")); + process.env.OPENCODEX_HOME = home; + clearAnthropicAccountPoolState(); + clearAccountQuotaCache(); + await saveCredential("anthropic", { + access: "synthetic-access", refresh: "synthetic-refresh", + expires: Date.now() + 3_600_000, accountId: "outbound-pause", + }); + accountId = getAccountSet("anthropic")!.activeAccountId; + sends = 0; + globalThis.fetch = (async () => { sends++; return new Response("{}", { status: 200 }); }) as typeof fetch; +}); +afterEach(() => { + globalThis.fetch = originalFetch; + clearAnthropicAccountPoolState(); + clearAccountQuotaCache(); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + removeTreeWithRetry(home); +}); + +test("per-account quota does not send when pause commits while token resolution waits", async () => { + const started = Promise.withResolvers(); + const token = Promise.withResolvers(); + const read = spyOn(accountCache, "getTokenForAccountQuotaProbe").mockImplementation(async () => { + started.resolve(); + return token.promise; + }); + try { + const pending = fetchProviderAccountQuotas("anthropic", true); + await started.promise; + await setAccountPaused("anthropic", accountId, true); + token.resolve("synthetic-access"); + expect((await pending)[0]?.unavailable).toBe(true); + expect(sends).toBe(0); + } finally { read.mockRestore(); } +}); + +test("provider quota does not send when pause commits while token resolution waits", async () => { + const started = Promise.withResolvers(); + const token = Promise.withResolvers(); + const read = spyOn(oauth, "getValidAccessToken").mockImplementation(async () => { + started.resolve(); + return token.promise; + }); + try { + const pending = fetchAnthropicQuota("anthropic"); + await started.promise; + await setAccountPaused("anthropic", accountId, true); + token.resolve("synthetic-access"); + expect(await pending).toBeNull(); + expect(sends).toBe(0); + } finally { read.mockRestore(); } +}); + +test("model discovery rejects a bearer captured before pause", async () => { + const credential = getAccountCredential("anthropic", accountId)!; + const resolver = { kind: "observed" as const, resolve: () => ({ + apiKey: credential.access, observed: true, + oauthAccountId: accountId, oauthGeneration: credentialGeneration(credential), + }) }; + const captured = captureProviderGather("anthropic", { + adapter: "anthropic", baseUrl: "https://api.anthropic.com", authMode: "oauth", + liveModels: true, models: ["configured-model"], + }, resolver); + await setAccountPaused("anthropic", accountId, true); + const result = await fetchProviderModelsWithAuth(captured, 0, undefined, resolver); + expect(result.outcome.state).toBe("degraded"); + expect(sends).toBe(0); +}); + +test("model discovery rejects a replaced credential generation even when its access token matches", async () => { + const credential = getAccountCredential("anthropic", accountId)!; + const resolver = { kind: "observed" as const, resolve: () => ({ + apiKey: credential.access, observed: true, + oauthAccountId: accountId, oauthGeneration: credentialGeneration(credential), + }) }; + const captured = captureProviderGather("anthropic", { + adapter: "anthropic", baseUrl: "https://api.anthropic.com", authMode: "oauth", + liveModels: true, models: ["configured-model"], + }, resolver); + await saveAccountCredential("anthropic", accountId, { ...credential, refresh: "rotated-refresh" }); + const result = await fetchProviderModelsWithAuth(captured, 0, undefined, resolver); + expect(result.outcome.state).toBe("degraded"); + expect(sends).toBe(0); +}); + +test("model discovery transport cancels after DNS when pause commits before physical send", async () => { + const started = Promise.withResolvers(); + const releaseDns = Promise.withResolvers(); + const pending = providerOutboundGet("anthropic", { baseUrl: "https://api.anthropic.com" }, + "https://api.anthropic.com/v1/models", { headers: { Authorization: "Bearer synthetic-access" } }, { + resolveAddresses: async () => { + started.resolve(); + await releaseDns.promise; + return { hostname: "api.anthropic.com", addresses: [{ address: "93.184.216.34", family: 4 }], privateNetwork: false }; + }, + pinnedGet: async () => { sends++; return new Response("{}"); }, + beforeSend: () => getAccountSet("anthropic")?.accounts.find(row => row.id === accountId)?.paused !== true, + }); + await started.promise; + await setAccountPaused("anthropic", accountId, true); + releaseDns.resolve(); + await expect(pending).rejects.toBeInstanceOf(ProviderOutboundSendCancelledError); + expect(sends).toBe(0); +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 1f0daa84ca9..2be6d12e095 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -106,6 +106,7 @@ "ambiguous-resend-composition.test.ts": "lib", "ambiguous-resend-gate.test.ts": "lib", "anthropic-account-pool.test.ts": "adapters/anthropic", + "anthropic-account-pause-outbound.test.ts": "adapters/anthropic", "anthropic-account-pause.test.ts": "adapters/anthropic", "anthropic-model-routes.test.ts": "adapters/anthropic", "anthropic-agentrouter-language-framing.test.ts": "adapters/anthropic", From f966c9faad06b17005e3ddc8ed6dc2923cea58f9 Mon Sep 17 00:00:00 2001 From: zhouliuya Date: Wed, 30 Sep 2026 02:12:26 +0800 Subject: [PATCH 09/26] fix(cursor): redirect native tool denials through code-mode exec (#6230) --- .../src/content/docs/reference/adapters.md | 13 ++-- src/adapters/cursor/native-exec.ts | 38 ++++++++--- structure/providers/cursor.md | 9 +++ .../cursor/cursor-native-exec-policy.test.ts | 63 ++++++++++++++++++- 4 files changed, 107 insertions(+), 16 deletions(-) diff --git a/docs-site/src/content/docs/reference/adapters.md b/docs-site/src/content/docs/reference/adapters.md index 6742dde2bb1..c4c5c087645 100644 --- a/docs-site/src/content/docs/reference/adapters.md +++ b/docs-site/src/content/docs/reference/adapters.md @@ -526,11 +526,14 @@ compatibility pair: `agent.v1.AgentService/RunSSE` for server output and Foreground `shellArgs` and `shellStreamArgs` are an exception: both are rejected before spawn on every platform until kernel-backed descendant ownership is available. Use client shell tools; background-shell execution and other native operations retain their existing policy. -- The denial reply is a silent redirect whose wording follows the request catalog. A catalog that - carries `shell_command`/`exec_command` or a unified `exec` keeps the bridge wording; a catalog - that carries neither — an orchestrator client exposing only its own Responses tools, for example — - is redirected to the request's actual wire names, so the model is pointed at a tool that exists - rather than at an alias it cannot see. +- The denial reply is a silent redirect whose wording follows the request catalog. + In code mode — a freeform unified `exec` and no bare shell bridge — the redirect points inside `exec`, where + shell, file, search, and fetch are nested `tools.(...)` helpers of the JavaScript cell, + and never recommends the top-level shell bridge code mode does not expose. A flat catalog that + carries `shell_command`/`exec_command` or a non-freeform unified `exec` keeps the bridge wording; + a catalog that carries neither — an orchestrator client exposing only its own Responses tools, + for example — is redirected to the request's actual wire names, so the model is pointed at a + tool that exists rather than at an alias it cannot see. - A recognized Cursor data-policy gate is reported with its title, the action it requires, and the Cursor Dashboard review URL instead of a bare `failed_precondition: Error`. Recognition is limited to the known structured detail: unknown or malformed details keep the generic Connect error, no diff --git a/src/adapters/cursor/native-exec.ts b/src/adapters/cursor/native-exec.ts index 51817ee2a0b..f08e98a9313 100644 --- a/src/adapters/cursor/native-exec.ts +++ b/src/adapters/cursor/native-exec.ts @@ -54,7 +54,8 @@ import { import { clientBytes, execBytes, execStreamCloseBytes, execThrowBytes } from "./native-exec-common"; import type { McpToolDefinition } from "./gen/agent_pb"; import { OCX_RESPONSES_TOOL_PROVIDER } from "./tool-definitions"; -import { cursorRequestHasExecutionPath, cursorRequestHasShellAlias, cursorToolWireName } from "./tool-naming"; +import { CODEX_UNIFIED_EXEC_TOOL, cursorRequestHasExecutionPath, cursorRequestHasShellAlias, cursorRequestUsesCodeMode, cursorToolWireName } from "./tool-naming"; +import { CODE_MODE_RESULT_ECHO_SENTENCE } from "../exec-tool-result-normalize"; import type { OcxTool } from "../../types"; export type CursorNativeExecDeps = CursorNativeNetworkDeps & CursorNativeToolDeps; @@ -84,20 +85,23 @@ export interface CursorNativeExecContext extends CursorNativeExecDeps { const REDIRECT_HINT_MAX_TOOLS = 16; /** - * Redirect text for Cursor-native fs/shell/fetch attempts when the request catalog carries NO shell - * bridge or other execution-path tool (an orchestrator client that only exposes delegation tools, - * for example). The default refusal steers the model to `shell_command` / `exec_command`; when those - * are not in the catalog some models (kimi-k3 observed) conclude every tool is unavailable and give - * up instead of using the tools that ARE listed. Name the real catalog instead — the client tools - * plus any configured MCP tools advertised this turn — and stay neutral about what those tools can - * do, so a listed file/search/fetch tool is never contradicted. + * Catalog-aware redirect for denied Cursor-native fs/shell/fetch attempts. Code mode must point + * inside freeform `exec`, since the default refusal names top-level shell tools it does not expose. + * When no execution path exists, name the actual client and configured MCP tools instead and stay + * neutral about their capabilities, so a listed file/search/fetch tool is never contradicted. */ export function cursorNativeExecRedirectHint( - tools: readonly Pick[] | undefined, + tools: readonly Pick[] | undefined, mcpToolDefs: readonly Pick[] = [], ): string | undefined { const clientTools = tools ?? []; - if (cursorRequestHasShellAlias(clientTools) || cursorRequestHasExecutionPath(clientTools)) return undefined; + if (cursorRequestHasShellAlias(clientTools)) return undefined; + // Code mode (freeform unified `exec`, no bare shell bridge): the default bridge wording names + // top-level shell tools this catalog does not expose, so the model probes for tools that cannot + // exist. Redirect INSIDE `exec` instead — shell, file, search, and fetch are nested helpers of + // the code cell. The caller supplies the active-turn catalog, so no tool_choice re-filter here. + if (cursorRequestUsesCodeMode(clientTools)) return cursorCodeModeExecRedirectHint(); + if (cursorRequestHasExecutionPath(clientTools)) return undefined; // Client tools are advertised under OCX_RESPONSES_TOOL_PROVIDER, so the harness shows them as // `mcp__`; configured MCP servers are advertised under their own provider id. // A request with no client tools but configured MCP tools still gets those named; a request that @@ -118,6 +122,20 @@ export function cursorNativeExecRedirectHint( ); } +/** + * Code-mode half of the redirect above: the only execution surface is the freeform `exec` cell, + * so the denial names the nested helpers instead of the missing flat bridge. The result-echo + * sentence is the shared canonical wording tool-guidance emits for the same isolate. + */ +function cursorCodeModeExecRedirectHint(): string { + return ( + `Re-issue this operation NOW through the \`${CODEX_UNIFIED_EXEC_TOOL}\` tool: this turn uses Codex code mode, so \`${CODEX_UNIFIED_EXEC_TOOL}\` takes a JavaScript body and shell, file, search, and fetch are nested helpers called INSIDE that body as \`await tools.(...)\`, for example \`text(await tools.exec_command({cmd: "ls"}))\`. ` + + "Cursor-native Read/Glob/Grep/LS/Shell/Write/Fetch are not part of this request's catalog; do not retry them, and do not call `shell_command` or `exec_command` at the top level here — code mode exposes no bare shell bridge, only the nested helpers. Every other tool this turn lists remains callable at the top level as usual. " + + CODE_MODE_RESULT_ECHO_SENTENCE + " " + + `Do NOT narrate this redirect, do NOT comment on tool availability, and do NOT re-announce the task — just make the \`${CODEX_UNIFIED_EXEC_TOOL}\` call.` + ); +} + export function cursorUnsafeNativeLocalExecEnabled(input: Pick = {}): boolean { return input.unsafeAllowNativeLocalExec === true; } diff --git a/structure/providers/cursor.md b/structure/providers/cursor.md index 68eb3a00ba9..21e223e7091 100644 --- a/structure/providers/cursor.md +++ b/structure/providers/cursor.md @@ -39,6 +39,15 @@ that survives the transport budget: unified Desktop `exec` as well as the legacy unified `exec` keeps its own schema and is surfaced back to Codex as a client tool. It must never fall through to the separate native-local-exec dispatcher. +Denied native fs/shell/fetch frames carry a catalog-aware redirect (`cursorNativeExecRedirectHint`). +When the visible catalog is Codex code mode — a freeform unified `exec` and no bare shell bridge — +the redirect steers the model INSIDE `exec`: shell, file, search, and fetch are nested +`tools.(...)` helpers of the JavaScript cell with `text(...)` required for output, and the +wording never recommends the top-level `shell_command`/`exec_command` bridge code mode does not +expose; separately listed namespaced tools remain callable as usual. A flat catalog with a bare +bridge or a non-freeform unified `exec` keeps the default bridge wording, and a catalog with no +execution path at all keeps naming the request's actual client/MCP wire names. + In external Cursor turns using code mode or shell aliases, the bounded leading-commentary guard in `src/adapters/cursor/envelope-echo.ts` counts `Shell`, `네이티브 셸`, and `네이티브 쉘` as one `shell` identity, including spacing variants and names split across text deltas. Korean aliases require a Unicode-aware left token boundary so wording embedded in a larger word or identifier is not counted; punctuation and following Korean grammatical suffixes remain supported. Rejection still requires a failure claim plus either an explicit redirect or at least two distinct native-tool identities; repeated aliases alone do not count as multiple tools. > Decision record: [ADR-0048](../decisions/ADR-0048-cursor-native-exec.md) diff --git a/tests/providers/cursor/cursor-native-exec-policy.test.ts b/tests/providers/cursor/cursor-native-exec-policy.test.ts index e62221f06af..33aed16eec5 100644 --- a/tests/providers/cursor/cursor-native-exec-policy.test.ts +++ b/tests/providers/cursor/cursor-native-exec-policy.test.ts @@ -451,11 +451,72 @@ describe("Cursor native exec catalog-aware redirect hint", () => { ["an empty catalog", []], ["a bare exec_command bridge", [{ name: "exec_command" }]], ["a bare shell_command bridge next to client tools", [{ name: "task" }, { name: "shell_command" }]], - ["unified exec next to client tools", [{ name: "task" }, { name: "exec", freeform: true }]], + ["a non-freeform unified exec next to client tools", [{ name: "task" }, { name: "exec" }]], + ["freeform exec beside a bare shell bridge (flat catalog)", [{ name: "exec", freeform: true }, { name: "shell_command" }]], ])("keeps the default bridge wording for %s", (_name, tools) => { expect(cursorNativeExecRedirectHint(tools)).toBeUndefined(); }); + test("code mode redirects inside the freeform exec and never recommends a missing top-level shell bridge", () => { + const hint = cursorNativeExecRedirectHint([{ name: "exec", freeform: true }]) ?? ""; + expect(hint).toContain("`exec`"); + expect(hint).toContain("await tools.exec_command("); + expect(hint).toContain("text("); + expect(hint).toContain("Do NOT narrate"); + // The buggy default recommended the flat bridge by its harness display name; code mode has neither. + expect(hint).not.toContain("mcp_opencodex-responses_"); + // The bare bridge names may only appear in the "do not call at the top level" clause. + expect(hint).toContain("do not call `shell_command` or `exec_command` at the top level"); + // "No bare shell bridge" must not read as forbidding separately listed namespaced tools. + expect(hint).toContain("remains callable at the top level as usual"); + for (const pattern of SILENT_REDIRECT_FORBIDDEN) expect(hint).not.toMatch(pattern); + }); + + test("code-mode denial frames across fs, shell, and fetch carry the exec redirect and execute nothing", async () => { + const hint = cursorNativeExecRedirectHint([{ name: "exec", freeform: true }]); + expect(hint).toBeDefined(); + const dir = mkdtempSync(join(tmpdir(), "ocx-cursor-code-mode-hint-")); + const existing = join(dir, "grounding.txt"); + const content = "CODE-MODE-HINT-01 must not leak"; + writeFileSync(existing, content); + const newPath = join(dir, "must-not-exist.txt"); + let fetchCalled = false; + const deps = { + unsafeAllowNativeLocalExec: false, + nativeExecRedirectHint: hint, + fetch: async () => { + fetchCalled = true; + return new Response("SHOULD_NOT_FETCH"); + }, + }; + const frames = [ + execMessage({ case: "readArgs", value: create(ReadArgsSchema, { path: existing }) }), + execMessage({ case: "lsArgs", value: create(LsArgsSchema, { path: dir }) }), + execMessage({ case: "grepArgs", value: create(GrepArgsSchema, { pattern: "CODE-MODE", path: dir }) }), + execMessage({ case: "writeArgs", value: create(WriteArgsSchema, { path: newPath, fileText: "SHOULD_NOT_WRITE" }) }), + execMessage({ case: "deleteArgs", value: create(DeleteArgsSchema, { path: existing }) }), + execMessage({ case: "shellArgs", value: create(ShellArgsSchema, { command: "printf RAN_%s MARKER", workingDirectory: dir, hardTimeout: 2000 }) }), + execMessage({ case: "shellStreamArgs", value: create(ShellArgsSchema, { command: "printf RAN_%s MARKER", workingDirectory: dir }) }), + execMessage({ case: "backgroundShellSpawnArgs", value: create(BackgroundShellSpawnArgsSchema, { command: "printf RAN_%s MARKER", workingDirectory: dir }) }), + execMessage({ case: "writeShellStdinArgs", value: create(WriteShellStdinArgsSchema, { shellId: 999, chars: "SHOULD_NOT_WRITE" }) }), + execMessage({ case: "fetchArgs", value: create(FetchArgsSchema, { url: "https://metadata.invalid/latest" }) }), + ]; + for (const frame of frames) { + const text = stringifyReplies(await handleCursorNativeExec(frame, deps)); + expect(text).toContain("await tools.exec_command("); + expect(text).toContain("Do NOT narrate"); + expect(text).not.toContain("mcp_opencodex-responses_"); + expect(text).not.toContain(content); + // Denied shell frames echo the command text; only an executed command could produce the joined marker. + expect(text).not.toContain("RAN_MARKER"); + expect(text).not.toContain("SHOULD_NOT_WRITE"); + expect(text).not.toContain("SHOULD_NOT_FETCH"); + } + expect(fetchCalled).toBe(false); + expect(existsSync(existing)).toBe(true); + expect(existsSync(newPath)).toBe(false); + }); + test("lists namespaced tools by wire name and caps a long catalog", () => { const hint = cursorNativeExecRedirectHint([{ namespace: "mcp__docker", name: "ps" }, { name: "task" }]) ?? ""; expect(hint).toContain("`mcp__docker__ps`"); From d3d076c5f4360712f2e2b9c91460f63dd2f91b76 Mon Sep 17 00:00:00 2001 From: valerio Date: Wed, 30 Sep 2026 01:12:49 +0700 Subject: [PATCH 10/26] fix(catalog): leave routed compaction compatibility unknown (#6236) Use null instead of a synthetic opencodex comp_hash so native/routed switches do not imply incompatible history. Clear retained markers, preserve native metadata and token limits, and cover migration and aliases. --- .../content/docs/guides/sub-agent-surface.md | 17 ++ src/codex/catalog/build-entries.ts | 11 +- src/codex/catalog/derive-entry.ts | 7 +- src/codex/catalog/parsing.ts | 5 +- structure/catalog.md | 10 +- .../catalog-routed-comp-hash.test.ts | 170 +++++++++++++----- 6 files changed, 159 insertions(+), 61 deletions(-) diff --git a/docs-site/src/content/docs/guides/sub-agent-surface.md b/docs-site/src/content/docs/guides/sub-agent-surface.md index f4abe08f22d..fc79bb1fc9e 100644 --- a/docs-site/src/content/docs/guides/sub-agent-surface.md +++ b/docs-site/src/content/docs/guides/sub-agent-surface.md @@ -34,6 +34,23 @@ routed one arrives encrypted and fails. The dashboard asks before either, and li [Why v1 is the default](/guides/subagent-v1-default/). ::: +## Model switches and side-chat compaction + +Codex can compact inherited history before the first turn of a side chat or after a model switch. +One trigger compares the previous and destination models' `comp_hash` compatibility markers: +when both are present and differ, Codex compacts even if the history fits the destination's context. + +OpenCodex represents unknown compatibility for routed models as `comp_hash: null`, instead of +inventing an `"opencodex"` marker or copying one from a native template. Catalog rebuilds also clear +those old markers on OpenCodex rows retained during a provider discovery outage. Native models and +explicit Codex-forward aliases keep their upstream markers, so genuine native incompatibility +checks still apply. This behavior is independent of the v1/v2 sub-agent surface and the model +configured to perform compaction. + +Normal context and token-limit compaction still applies. This change removes the synthetic +hash mismatch; it does not select which visible messages a side chat inherits or restore original +messages from an already compacted context. Provider content support remains a separate constraint. + ## External task input Codex can deliver a task's initial input or follow-up in a result-shaped envelope diff --git a/src/codex/catalog/build-entries.ts b/src/codex/catalog/build-entries.ts index aef0f234054..4d35cd2c8c4 100644 --- a/src/codex/catalog/build-entries.ts +++ b/src/codex/catalog/build-entries.ts @@ -369,9 +369,8 @@ export function orderForModelPicker( * Every generated routed row — current full-slug form, the June–July 2026 * provider-name form, and legacy combo aliases — carries the stable * description prefix `Routed via opencodex → `; foreign rows from Cursor or - * user tooling do not. `owned_by` cannot serve as the signal (upstream - * ownership), and `comp_hash` defaults to "opencodex" for every normalized - * row. + * user tooling do not. `owned_by` describes upstream ownership, and `comp_hash` + * describes history compatibility; neither is an authorship signal. */ function isOcxAuthoredRoutedEntry(entry: RawEntry): boolean { if (isNativeAliasCatalogEntry(entry)) return true; @@ -782,11 +781,13 @@ export function mergeCatalogEntriesFromObservedState({ delete entry[SPAWN_PRIORITY_FIELD]; } const slug = String(entry.slug); - if (!isOcxAuthoredRoutedEntry(entry) || isNativeAliasCatalogEntry(entry)) continue; + if (!isOcxAuthoredRoutedEntry(entry)) continue; // The builder no longer copies a template's comp_hash onto routed rows (#5796), but a row // kept from disk may still carry one. Custom rows, Codex-forward aliases included, never // reach this loop: they are rebuilt from config. - entry.comp_hash = "opencodex"; + // Clear the former synthetic "opencodex" marker as well: it is not upstream evidence. + entry.comp_hash = null; + if (isNativeAliasCatalogEntry(entry)) continue; const featuredRank = featuredRankOf(slug); entry.priority = featuredRank !== undefined ? featuredRank * priorityStride diff --git a/src/codex/catalog/derive-entry.ts b/src/codex/catalog/derive-entry.ts index e1d85541704..98bd424e712 100644 --- a/src/codex/catalog/derive-entry.ts +++ b/src/codex/catalog/derive-entry.ts @@ -144,10 +144,9 @@ export function deriveEntry( delete e.context_window; delete e.max_context_window; delete e.auto_compact_token_limit; - // Nor its comp_hash (#5796). Codex compacts a thread whenever the recorded value - // changes, and the template is whichever native row a rebuild found first, so an - // inherited value moves with rebuild order. Left unset, normalization gives every - // routed row the same "opencodex" marker. + // Nor its comp_hash (#5796): template selection is not evidence of history + // compatibility. Normalization represents the unknown value as null, avoiding + // both rebuild-dependent hashes and a synthetic native/routed mismatch. delete e.comp_hash; } if (typeof e.base_instructions === "string") { diff --git a/src/codex/catalog/parsing.ts b/src/codex/catalog/parsing.ts index 233b882be46..a26b361550a 100644 --- a/src/codex/catalog/parsing.ts +++ b/src/codex/catalog/parsing.ts @@ -633,7 +633,10 @@ export function ensureStrictCatalogFields( entry.max_context_window = contextWindow; } if (typeof entry.effective_context_window_percent !== "number") entry.effective_context_window_percent = 95; - if (typeof entry.comp_hash !== "string") entry.comp_hash = "opencodex"; + // Unknown compatibility is not an incompatibility marker. Codex compares hashes only + // when both turns supply one; a synthetic value forces native/routed switches to compact + // even below the token limit. Keep authoritative strings and represent unknown as null. + if (typeof entry.comp_hash !== "string") entry.comp_hash = null; // Routed rows must not carry NATIVE eligibility metadata. `deriveEntry` deep-clones a // native template and deletes a fixed denylist, so these eligibility fields survive onto rows backed // by unrelated provider credentials — advertising ChatGPT plan eligibility for a model diff --git a/structure/catalog.md b/structure/catalog.md index 6bd10f2f486..9951e1c979f 100644 --- a/structure/catalog.md +++ b/structure/catalog.md @@ -60,11 +60,11 @@ provider-wide fallback. Exact model output limits precede the provider default o explicitly configured canonical `openai/gpt-daybreak-blue-latest` Codex-forward row from the pinned Sol capability metadata while preserving its selector and Daybreak wire identity; this never expands the bare/API-key model lists or rewrites the wire model to `gpt-5.6-sol`; -- clones a native template for routed `provider/model` entries without its `comp_hash`, and resets - that value on opencodex rows kept from disk while their provider's discovery is degraded, so these - rows carry the fixed `"opencodex"` marker instead of whichever native row a rebuild found first; - Codex compacts a thread whenever that value changes (#5796). Codex-forward aliases keep their - native value and rows written by other tools keep theirs; +- clones a native template for routed `provider/model` entries without its `comp_hash`, and clears + copied or synthetic hashes on opencodex rows kept during degraded discovery (#5796). Unknown + compatibility is `null`: Codex's hash-change trigger requires two non-null, unequal hashes, so + native/routed switches do not compact merely because of a synthetic marker. Native rows and + Codex-forward aliases keep upstream hashes; foreign rows keep valid hashes. Token limits are unchanged; - forces strict Codex catalog fields required by the current parser; - hides `disabledModels` without blocking direct routing (routed provider ids are excluded; account-qualified native ids hide only that selector row; BARE native slugs hide the bare row diff --git a/tests/codex-integration/catalog-routed-comp-hash.test.ts b/tests/codex-integration/catalog-routed-comp-hash.test.ts index d32a22bedeb..81c29751d65 100644 --- a/tests/codex-integration/catalog-routed-comp-hash.test.ts +++ b/tests/codex-integration/catalog-routed-comp-hash.test.ts @@ -3,67 +3,145 @@ import { buildCatalogEntries } from "../../src/codex/catalog"; import { buildCatalogEntriesFromObservedState, CANONICAL_NATIVE_CATALOG_CONTENT_POLICY, + deriveEntry, mergeCatalogEntriesFromObservedState, + mergeCatalogEntriesForSync, } from "../../src/codex/catalog/sync"; +import { ensureStrictCatalogFields } from "../../src/codex/catalog/parsing"; import type { RawEntry } from "../../src/codex/catalog/parsing"; -/** - * A routed row is cloned from whichever native row a rebuild picks as the template, and it - * used to keep that row's `comp_hash` (#5796). Codex compacts a thread when consecutive turns - * record different values, so a rebuild that picked another template compacted every active - * routed thread. - */ describe("catalog routed comp_hash (#5796)", () => { - const routedHash = (compHash: string | undefined) => buildCatalogEntries( - { - slug: "gpt-5.5", - display_name: "gpt-5.5", - description: "Native GPT model", - priority: 1, - visibility: "list", - base_instructions: "You are Codex, a coding agent based on GPT-5.", - ...(compHash === undefined ? {} : { comp_hash: compHash }), - }, - [], - [{ provider: "local", id: "qwen3-coder" }], - ).find(e => e.slug === "local/qwen3-coder")?.comp_hash; + const template = (compHash: string | null | undefined): RawEntry => ({ + slug: "gpt-5.5", + display_name: "gpt-5.5", + description: "Native GPT model", + priority: 1, + visibility: "list", + base_instructions: "You are Codex, a coding agent based on GPT-5.", + ...(compHash === undefined ? {} : { comp_hash: compHash }), + }); + + const routedEntry = (entries: RawEntry[], slug = "local/qwen3-coder") => + entries.find(entry => entry.slug === slug)!; + + const mergeOutage = (catalogModels: readonly RawEntry[], routedEntries: readonly RawEntry[]) => + mergeCatalogEntriesFromObservedState({ + catalogModels, + routedEntries, + baselineCatalogModels: [], + baseline: new Map(), + featured: [], + wsEnabled: false, + template: null, + disabledModels: new Set(), + selectedModelsByProvider: new Map(), + gatheredProviderNames: new Set(["local"]), + degradedProviderNames: new Set(["local"]), + legacyCustomModelSlugs: new Set(), + multiAgentMode: "default", + multiAgentV2Enabled: false, + exactComboSlugs: new Set(), + hasPhysicalComboProvider: false, + includeNativeOpenAi: true, + accountBoundEntries: [], + policy: { ...CANONICAL_NATIVE_CATALOG_CONTENT_POLICY, warningPolicy: "suppress" }, + }); + + test("generic routed rows have unknown hashes for every native template shape", () => { + for (const nativeHash of ["3000", "2911", null, undefined] as const) { + const row = routedEntry(buildCatalogEntries( + template(nativeHash), [], [{ provider: "local", id: "qwen3-coder" }], + )); + expect(row.comp_hash).toBeNull(); + expect(JSON.parse(JSON.stringify(row)).comp_hash).toBeNull(); + } + }); - test("routed rows keep one value whichever native row is the template", () => { - expect(routedHash("3000")).toBe("opencodex"); - expect(routedHash("2911")).toBe("opencodex"); - expect(routedHash(undefined)).toBe("opencodex"); + test("derivation strips native hashes while native and canonical Codex-forward rows keep theirs", () => { + const nativeTemplate = template("native-template-hash"); + const generic = deriveEntry(nativeTemplate, "local/model", "Routed model", 5, { + provider: "local", id: "model", contextWindow: 256_000, + maxInputTokens: 180_000, autoCompactTokenLimit: 150_000, + }); + expect(generic).toMatchObject({ + comp_hash: null, + context_window: 256_000, + max_context_window: 256_000, + auto_compact_token_limit: 150_000, + }); + + expect(ensureStrictCatalogFields({ slug: "gpt-5.5", comp_hash: "native-authoritative" }).comp_hash) + .toBe("native-authoritative"); + expect(deriveEntry(template("native-template-hash"), "openai/gpt-6-sol", "Forward", 5, { + provider: "openai", id: "gpt-6-sol", codexForwardNativeCapabilityAlias: true, + }).comp_hash).toBe("3000"); }); - test("a row kept from disk during a provider outage drops a copied value", () => { - const merge = (catalogModels: readonly RawEntry[], routedEntries: readonly RawEntry[], degraded: boolean) => - mergeCatalogEntriesFromObservedState({ - catalogModels, routedEntries, baselineCatalogModels: [], baseline: new Map(), featured: [], - wsEnabled: false, template: null, disabledModels: new Set(), selectedModelsByProvider: new Map(), - gatheredProviderNames: new Set(["local"]), - degradedProviderNames: new Set(degraded ? ["local"] : []), - legacyCustomModelSlugs: new Set(), multiAgentMode: "default", multiAgentV2Enabled: false, - exactComboSlugs: new Set(), hasPhysicalComboProvider: false, includeNativeOpenAi: true, - accountBoundEntries: [], - policy: { ...CANONICAL_NATIVE_CATALOG_CONTENT_POLICY, warningPolicy: "suppress" }, - }); + test("strict normalization makes absent or invalid hashes explicit null and preserves known values", () => { + for (const [value, expected] of [ + [undefined, null], [null, null], [42, null], ["", ""], ["upstream-hash", "upstream-hash"], + ] as const) { + const row: RawEntry = { slug: "provider/model", ...(value === undefined ? {} : { comp_hash: value }) }; + expect(ensureStrictCatalogFields(row, { isRouted: true }).comp_hash).toBe(expected); + expect(Object.hasOwn(row, "comp_hash")).toBe(true); + } + }); + + test("rebuild and JSON roundtrip keep the normalized routed row stable", () => { + const fresh = buildCatalogEntries(template("native-template-hash"), [], [ + { provider: "local", id: "qwen3-coder" }, + ]); + const first = mergeCatalogEntriesForSync([], fresh, new Map(), [], false); + const persisted = JSON.parse(JSON.stringify(first)) as RawEntry[]; + const second = mergeCatalogEntriesForSync(persisted, fresh, new Map(), [], false); + expect(routedEntry(first)).toEqual(routedEntry(second)); + expect(routedEntry(second).comp_hash).toBeNull(); + }); + + test.each(["3000", "opencodex"] as const)( + "outage migration clears the old %s routed hash without mutating input or foreign hashes", + oldHash => { const fresh = buildCatalogEntriesFromObservedState({ template: null, gptSlugs: [], goModels: [{ provider: "local", id: "qwen3-coder" }], featured: [], modelPickerOrder: [], wsEnabled: false, multiAgentMode: "default", exactComboSlugs: new Set(), accountSelectors: [], suppressedBareNativeSlugs: new Set(), disabledNativeAccountSlugs: new Set(), multiAgentV2Enabled: false, }); - // A catalog written before the fix, where the routed row still carries a template's value, - // plus a row another tool wrote under the same provider. That one is not ours to change. - const saved = merge([], fresh, false).map(entry => - entry.slug === "local/qwen3-coder" ? { ...entry, comp_hash: "3000" } : entry); + const saved = fresh.map(entry => entry.slug === "local/qwen3-coder" + ? { ...entry, comp_hash: oldHash } : entry); const foreign = { - ...saved.find(entry => entry.slug === "local/qwen3-coder")!, - slug: "local/imported", - description: "Imported model", + ...routedEntry(saved), slug: "local/imported", description: "Imported model", comp_hash: "foreign-valid-hash", }; - const kept = merge([...saved, foreign], [], true); - const hash = (slug: string) => kept.find(entry => entry.slug === slug)?.comp_hash; - expect(hash("local/qwen3-coder")).toBe("opencodex"); - expect(hash("local/imported")).toBe("3000"); + const input = structuredClone([...saved, foreign]); + const kept = mergeOutage(input, []); + expect(input).toEqual(structuredClone([...saved, foreign])); + expect(routedEntry(kept).comp_hash).toBeNull(); + expect(routedEntry(kept, "local/imported").comp_hash).toBe("foreign-valid-hash"); + }); + + test.each(["3000", "opencodex"] as const)( + "fresh and retained public native aliases normalize old %s hashes to null", oldHash => { + const alias = "gpt-5.6-sol"; + const built = buildCatalogEntriesFromObservedState({ + template: template("3000"), gptSlugs: [alias], + goModels: [{ provider: "combo", id: "nova-sol", alias, nativeAlias: true, owned_by: "combo" }], + featured: [], modelPickerOrder: [], wsEnabled: false, multiAgentMode: "default", + exactComboSlugs: new Set([alias]), accountSelectors: [], suppressedBareNativeSlugs: new Set(), + disabledNativeAccountSlugs: new Set(), multiAgentV2Enabled: false, + }); + const freshAlias = built.find(entry => entry.slug === alias)!; + expect(freshAlias.comp_hash).toBeNull(); + + const staleAlias = { ...freshAlias, comp_hash: oldHash }; + const kept = mergeCatalogEntriesFromObservedState({ + catalogModels: [staleAlias], routedEntries: [], baselineCatalogModels: [], baseline: new Map(), + featured: [], wsEnabled: false, template: null, disabledModels: new Set(), + selectedModelsByProvider: new Map(), gatheredProviderNames: new Set(), degradedProviderNames: new Set(), + legacyCustomModelSlugs: new Set(), multiAgentMode: "default", multiAgentV2Enabled: false, + exactComboSlugs: new Set([alias]), hasPhysicalComboProvider: false, includeNativeOpenAi: true, + suppressedBareNativeSlugs: new Set([alias]), accountBoundEntries: [], + policy: { ...CANONICAL_NATIVE_CATALOG_CONTENT_POLICY, warningPolicy: "suppress" }, + }).find(entry => entry.slug === alias)!; + expect(kept.comp_hash).toBeNull(); }); }); From f53f0c63af97ad36c6f9cf3b3e6f1cca65ba0c93 Mon Sep 17 00:00:00 2001 From: JUN Date: Wed, 30 Sep 2026 03:20:40 +0900 Subject: [PATCH 11/26] docs(devlog): record the 2.72.0 TokenLab release (#6250) * docs(devlog): plan the 2.72.0 TokenLab release * docs(devlog): record TokenLab payment and final-copy findings * docs(devlog): record the #6240 regression gate and merge * docs(devlog): record the 2.72.0 promotions * docs(devlog): record the 2.72.0 release results * docs(devlog): close the TokenLab sponsor and 2.72.0 release units * docs(devlog): keep the dev pre-move out of the 2.72.0 contents * docs(devlog): name the final #6240 head in the gate step --------- Co-authored-by: t --- .../260929_tokenlab_sponsor/000_roadmap.md | 0 .../260929_tokenlab_sponsor/010_merge_6221.md | 0 .../260929_tokenlab_sponsor/020_sponsor_pr.md | 0 devlog/_fin/260930_release_2_72_0/000_plan.md | 17 +++++++++++ .../260930_release_2_72_0/010_merge_6240.md | 14 +++++++++ .../011_wp2_execution.md | 14 +++++++++ .../_fin/260930_release_2_72_0/020_release.md | 25 ++++++++++++++++ .../021_wp3_execution.md | 29 +++++++++++++++++++ .../030_payment_and_email.md | 29 +++++++++++++++++++ .../_fin/260930_release_2_72_0/090_outcome.md | 23 +++++++++++++++ 10 files changed, 151 insertions(+) rename devlog/{_plan => _fin}/260929_tokenlab_sponsor/000_roadmap.md (100%) rename devlog/{_plan => _fin}/260929_tokenlab_sponsor/010_merge_6221.md (100%) rename devlog/{_plan => _fin}/260929_tokenlab_sponsor/020_sponsor_pr.md (100%) create mode 100644 devlog/_fin/260930_release_2_72_0/000_plan.md create mode 100644 devlog/_fin/260930_release_2_72_0/010_merge_6240.md create mode 100644 devlog/_fin/260930_release_2_72_0/011_wp2_execution.md create mode 100644 devlog/_fin/260930_release_2_72_0/020_release.md create mode 100644 devlog/_fin/260930_release_2_72_0/021_wp3_execution.md create mode 100644 devlog/_fin/260930_release_2_72_0/030_payment_and_email.md create mode 100644 devlog/_fin/260930_release_2_72_0/090_outcome.md diff --git a/devlog/_plan/260929_tokenlab_sponsor/000_roadmap.md b/devlog/_fin/260929_tokenlab_sponsor/000_roadmap.md similarity index 100% rename from devlog/_plan/260929_tokenlab_sponsor/000_roadmap.md rename to devlog/_fin/260929_tokenlab_sponsor/000_roadmap.md diff --git a/devlog/_plan/260929_tokenlab_sponsor/010_merge_6221.md b/devlog/_fin/260929_tokenlab_sponsor/010_merge_6221.md similarity index 100% rename from devlog/_plan/260929_tokenlab_sponsor/010_merge_6221.md rename to devlog/_fin/260929_tokenlab_sponsor/010_merge_6221.md diff --git a/devlog/_plan/260929_tokenlab_sponsor/020_sponsor_pr.md b/devlog/_fin/260929_tokenlab_sponsor/020_sponsor_pr.md similarity index 100% rename from devlog/_plan/260929_tokenlab_sponsor/020_sponsor_pr.md rename to devlog/_fin/260929_tokenlab_sponsor/020_sponsor_pr.md diff --git a/devlog/_fin/260930_release_2_72_0/000_plan.md b/devlog/_fin/260930_release_2_72_0/000_plan.md new file mode 100644 index 00000000000..149e4c57f19 --- /dev/null +++ b/devlog/_fin/260930_release_2_72_0/000_plan.md @@ -0,0 +1,17 @@ +# 2.72.0 — TokenLab release + +Owner request (2026-09-30): merge the TokenLab sponsor PR, verify for regressions, release, check +TokenLab's payment in the WORKS inbox through Aside, and have Aside email Vincent. + +Previous unit: `devlog/_fin/260929_tokenlab_sponsor/` merged #6221 (preset, `f6cddd7d69`) and opened +#6240 (sponsor placement + CLI pinning, head `21cddd35c9`, 32/32 PR checks green). Release shape is +2.71.0 (`devlog/_fin/260929_release_2_71_0/040_release.md`). + +| Doc | Work phase | Outcome | +|-----|-----------|---------| +| [010](./010_merge_6240.md) | wp2 | #6240 merged on full-lane CI (incl. windows 1–9) at its exact head | +| [020](./020_release.md) | wp3 | npm latest 2.72.0, preview 2.72.0-preview.20260930, GitHub releases, gitHead | +| [030](./030_payment_and_email.md) | wp4 | Payment/DocuSign status from WORKS; Vincent emailed by Aside exec; outcome recorded | + +Release content since v2.71.0: #6221 (TokenLab preset), #6240 (sponsor placement, CLI sponsor +pinning), devlog-only commits. diff --git a/devlog/_fin/260930_release_2_72_0/010_merge_6240.md b/devlog/_fin/260930_release_2_72_0/010_merge_6240.md new file mode 100644 index 00000000000..ec33545400b --- /dev/null +++ b/devlog/_fin/260930_release_2_72_0/010_merge_6240.md @@ -0,0 +1,14 @@ +# 010 — Regression gate and merge #6240 (wp2) + +1. Dispatch full-lane Cross-platform CI on the PR head: + `gh workflow run ci.yml -R lidge-jun/opencodex --ref codex/tokenlab-sponsor -f lane=all` + (the pull_request event skips windows 1–9 and macOS control; 2.71.0 used the same dispatch). + Required: every job success, windows 1/9–9/9 present, on the final head `2b9295fea6` (the run on `21cddd35c9` was cancelled when the final sponsor copy landed; see 030). +2. Local regression scope: the lanes' focused tests plus `test:changed`; the full local suite is + not runnable from a worktree under `~/.codex` (test home guard), so CI is the full-suite proof. +3. `scripts/ci/assert-mergeable-review.sh --maintainer-integration 6240 lidge-jun/opencodex`, a PR + comment recording owner authorization, exact head and run IDs: PR-event Cross-platform CI, + Service lifecycle (triggered by `desktop/**` and `package.json`), and the `lane=all` dispatch. +4. `gh pr merge 6240 --admin --squash --match-head-commit `. C is that exact squash commit, + fixed before anything else lands on `dev`; assert `git show C:package.json` reads 2.72.0. + The `260929_tokenlab_sponsor` plan docs on the branch land inside the squash. diff --git a/devlog/_fin/260930_release_2_72_0/011_wp2_execution.md b/devlog/_fin/260930_release_2_72_0/011_wp2_execution.md new file mode 100644 index 00000000000..6105dfe65d9 --- /dev/null +++ b/devlog/_fin/260930_release_2_72_0/011_wp2_execution.md @@ -0,0 +1,14 @@ +# 011 wp2 execution: regression gate and merge + +| Step | Evidence | +|---|---| +| Final copy | #6240 took TokenLab's final blurbs and referral link at `2b9295fea6` (030 findings); the `lane=all` run on `21cddd35c9` was cancelled | +| PR-event CI on `2b9295fea6` | Cross-platform CI 36591071773, Service lifecycle 36591071699, React Doctor 36591071756: success; 32 pass, 5 path-skipped | +| Full-lane gate | Cross-platform CI `lane=all` 36591083341: attempt 1 failed windows 7/9 (`cli-connect-readiness` installed-root probe, exit null at 18 s) and windows 8/9 (`main quota policy at native admission`, 32 s cases); neither loads a changed module (`init`, `provider-runtime` are lazy CLI imports). One rerun (attempt 2): both shards and `ci` success | +| Review | Codex P2 and CodeRabbit sponsor-first-run fixed; CodeRabbit wording suggestion declined (verbatim sponsor copy, adapter chip visible, Responses-first pending) | +| Policy | `assert-mergeable-review.sh --maintainer-integration 6240`: OK; decision comment 5894282491 | +| Merge | `gh pr merge 6240 --admin --squash --match-head-commit 2b9295fea6` → dev `1cd9d25517` = C; `package.json` 2.72.0, version-sources check 2.72.0 passes | +| Pre-move | #6243 (four version sources 2.72.0 → 2.73.0), `maintainer-sponsored` after review, merged → dev `73289d46ae` (2.73.0) | + +Local regression scope: focused sponsor/registry/README/GUI suites and `test:changed`; a full local run +is not possible from a worktree under `~/.codex` (test home guard), so CI above is the full-suite proof. diff --git a/devlog/_fin/260930_release_2_72_0/020_release.md b/devlog/_fin/260930_release_2_72_0/020_release.md new file mode 100644 index 00000000000..98da49e340a --- /dev/null +++ b/devlog/_fin/260930_release_2_72_0/020_release.md @@ -0,0 +1,25 @@ +# 020 — Release 2.72.0 (wp3) + +Same procedure as `devlog/_fin/260929_release_2_71_0/041_wp4_execution.md`, with C = the #6240 +squash commit from 010 (never a later `dev` tip, which would carry 2.73.0). + +1. Pre-move: `gh workflow run dev-version-bump.yml -R lidge-jun/opencodex --ref main + -f intended-version=2.72.0 -f mode=pre-move` → PR moving exactly the four version sources to + 2.73.0; merge `--admin --squash --match-head-commit` after its CI. +2. Preview: branch `codex/promote-preview-2.72.0` from C, `git merge -s ours origin/preview`, + `bun scripts/release-version-sources.ts sync 2.72.0-preview.20260930`, commit; diff vs C must be + exactly the four version sources, and `bun scripts/release-version-sources.ts` check mode passes. + PR to preview with the #6240 pr-assets screenshots, `gh pr merge --admin --merge --match-head-commit`. +3. Main: branch `codex/promote-main-2.72.0` from C, `git merge -s ours origin/main`, tree equals C. + (`git diff --quiet C HEAD`). PR to main with the screenshots, merged the same way. enforce-target + flags promotion PRs as wrong base by design; it is not required on preview/main. +4. Gate each promotion SHA: push-event Cross-platform CI and Service lifecycle `success`. +5. Dispatch preview then stable: + `gh workflow run release.yml --ref preview -f version=2.72.0-preview.20260930 -f tag=preview + -f dry-run=false -f expected-sha=`, then `--ref main -f version=2.72.0 -f tag=latest + -f dry-run=false -f expected-sha=
`. After an npm-acknowledged failure, resume with + `-f resume-after-npm-publish=true`; never republish. +6. Verify dist-tags, `npm view @bitkyc08/opencodex@2.72.0 gitHead` = main sha, `gh release view v2.72.0` + (not prerelease, 25 assets as v2.71.0), preview release prerelease, latest.json 2.72.0 signed. + +The installed proxy/app on this machine is not updated (same as 2.70.0/2.71.0). diff --git a/devlog/_fin/260930_release_2_72_0/021_wp3_execution.md b/devlog/_fin/260930_release_2_72_0/021_wp3_execution.md new file mode 100644 index 00000000000..a268eb8e9ef --- /dev/null +++ b/devlog/_fin/260930_release_2_72_0/021_wp3_execution.md @@ -0,0 +1,29 @@ +# 021 wp3 execution: promotion and release + +| Step | Evidence | +|---|---| +| Candidate | C = dev `1cd9d25517` (#6240 squash), version sources 2.72.0 | +| Pre-move | #6243 → dev `73289d46ae` (2.73.0), `maintainer-sponsored` after review | +| Preview promotion | `codex/promote-preview-2.72.0`: `-s ours` merge of origin/preview + sync to 2.72.0-preview.20260930 (`14ccfe1a1a`); diff vs C = four version sources; PR #6245 merged (merge commit) → preview `4f9e3f0afb` | +| Main promotion | `codex/promote-main-2.72.0`: `-s ours` merge of origin/main (`77cb00512f`), tree equals C; PR #6246 merged → main `5ab6d52b2a` | +| Push-event gates | preview: Cross-platform CI 36597831993, Service lifecycle 36597831950; main: Cross-platform CI 36597841262, Service lifecycle 36597841450 | + +Dispatches (after both gates of a SHA succeed), preview first: + +```sh +gh workflow run release.yml -R lidge-jun/opencodex --ref preview -f version=2.72.0-preview.20260930 -f tag=preview -f dry-run=false -f expected-sha=4f9e3f0afbbcf54a2b0421db8e962ec3d5682d5e +gh workflow run release.yml -R lidge-jun/opencodex --ref main -f version=2.72.0 -f tag=latest -f dry-run=false -f expected-sha=5ab6d52b2a4da722d398e4ab50a6c621ac3ce087 +``` + +## Results + +| Check | Evidence | +|---|---| +| Preview gates | Cross-platform CI 36597831993 success (push), Service lifecycle 36597831950 success | +| Main gates | Service lifecycle 36597841450 success; Cross-platform CI 36597841262 attempt 1 failed only `test 2/4` (batch 10/48 hit the 120 s process bound; the attribution sweep reported every file passing alone, "the timeout lives in multi-file process state"); one rerun, attempt 2 success | +| Preview release | release.yml 36602348988 success; npm `preview` = 2.72.0-preview.20260930, gitHead `4f9e3f0afb`, bins `ocx`/`opencodex` intact; GitHub release prerelease, 25 assets | +| Stable release | release.yml 36603799783 success; npm `latest` = 2.72.0 (published 17:45 UTC, visible ~10 min later, as with 2.71.0), gitHead `5ab6d52b2a`, bins intact; GitHub release v2.72.0 not prerelease, 25 assets; latest.json 2.72.0 signed for darwin-aarch64, darwin-x86_64, linux-x86_64, linux-x86_64-deb, windows-x86_64 | + +npm printed `"bin[...]" script name bin/ocx.mjs was invalid and removed` during both publishes; 2.70.0 and +2.71.0 printed the same, and the registry metadata keeps both bins (the `./` prefix is normalized). +The installed proxy and desktop app on this machine were not updated. diff --git a/devlog/_fin/260930_release_2_72_0/030_payment_and_email.md b/devlog/_fin/260930_release_2_72_0/030_payment_and_email.md new file mode 100644 index 00000000000..c76199ee79f --- /dev/null +++ b/devlog/_fin/260930_release_2_72_0/030_payment_and_email.md @@ -0,0 +1,29 @@ +# 030 — Payment check and sponsor email (wp4) + +1. Aside exec, read-only, WORKS inbox: TokenLab messages since 2026-09-29 18:00 KST (payment + confirmation, transaction hash, amount), DocuSign completion status. Where a transaction hash is + given, confirm it read-only on the public chain explorer against the recipient addresses in the + agreement (1,200 USDT, TRC-20 or ERC-20). +2. Only after npm `latest` reads 2.72.0 (the term starts at that release), Aside exec replies in the TokenLab thread from the maintainer mailbox, politely: payment received (only if confirmed), #6221/#6240 merged, released in + opencodex 2.72.0 (npm `@bitkyc08/opencodex`, release link), the 3-month term starts on that + release date per the agreement, README/picker placement live, the Responses-first proposal will be + evaluated separately. No attachments, no other recipients. +3. Record 090_outcome.md; move this unit and `260929_tokenlab_sponsor` to `devlog/_fin/` through a + docs PR to dev. + +Wallet addresses and transaction hashes stay out of the repository; the outcome records only that +payment was confirmed and when. + +## Findings (2026-09-30, before release) + +- WORKS inbox (Aside exec, read-only): Vincent wrote on 2026-09-29 23:08 KST that he signed the + agreement and paid 1,200 USDT on TRC-20, with a transaction hash, and sent final sponsor copy: a + longer English blurb, a Chinese blurb, and `https://tokenlab.sh/r/OPENCODEX` as the README and + picker link. A 23:13 message offers a USD 20 API-credit code for integration testing. +- On-chain (Tronscan, read-only): the hash is a confirmed, successful transfer on the official + USDT contract of exactly 1,200.000000 USDT to the agreement's TRC-20 address, at 2026-09-29 + 13:53 UTC; not flagged as risky. +- DocuSign: the only DocuSign mail in WORKS is the 21:08 sender-verification notice. No completion notice reached the maintainer mailbox; status notices go to the envelope sender's address, which this + check did not cover. Signature completion stays unverified here. +- Consequence for 010: #6240 takes the final copy and referral link (`2b9295fea6`) before the + regression gate; the earlier `lane=all` run on `21cddd35c9` was cancelled. diff --git a/devlog/_fin/260930_release_2_72_0/090_outcome.md b/devlog/_fin/260930_release_2_72_0/090_outcome.md new file mode 100644 index 00000000000..e8aedc57c73 --- /dev/null +++ b/devlog/_fin/260930_release_2_72_0/090_outcome.md @@ -0,0 +1,23 @@ +# 090 Outcome — 2.72.0 TokenLab release + +Closed 2026-09-30 (KST). + +| Criterion | Result | +|---|---| +| #6240 merged after full-lane CI on its exact head | Merged `1cd9d25517` from head `2b9295fea6`; lane=all 36591083341 success on attempt 2 (windows 7/9 and 8/9 reran once; neither loads a changed module) | +| npm and GitHub releases | `latest` 2.72.0 (gitHead `5ab6d52b2a`, main via #6246), `preview` 2.72.0-preview.20260930 (gitHead `4f9e3f0afb`, preview via #6245); release.yml 36603799783 / 36602348988; v2.72.0 has 25 assets and a signed latest.json | +| Payment and sponsor email | TokenLab's 1,200 USDT payment confirmed on-chain (2026-09-29 13:53 UTC). Reply sent from the maintainer mailbox through Aside exec at 2026-09-30 02:59 KST, confirming receipt, the 2.72.0 release, the placements and the term start | + +Release content since 2.71.0: #6221 (TokenLab preset, by @hedging8563), #6240 (sponsor placement, +CLI sponsor pinning, final sponsor copy and referral link). #6243 moved `dev` to 2.73.0 after the +candidate was pinned and is not part of 2.72.0. + +Per the agreement, the three-month sponsorship term starts with the 2.72.0 npm release +(2026-09-29 17:45 UTC, 2026-09-30 KST). + +Open items, outside this unit: +- DocuSign completion is not confirmed from the maintainer mailbox; TokenLab reports signing. Envelope + status goes to the sender account. +- TokenLab's Responses-first preset proposal (`X-TokenLab-Delivery-Policy`) needs its own PR and evidence. +- TokenLab offered a USD 20 API credit for integration testing; redeeming it is the maintainer's choice. +- The installed proxy and desktop app on this machine were not updated. From cbf5faaf333985312e4f5e46cf5e04cab9037c46 Mon Sep 17 00:00:00 2001 From: JUN Date: Wed, 30 Sep 2026 03:27:17 +0900 Subject: [PATCH 12/26] fix(bridge): preserve rate-limit retry advice for Codex (carry #6225) (#6248) Carries #6225 by @luvs01: typed Devin 429 errors keep the provider's longest retry delay first in the message Codex parses. Co-authored-by: luvs01 <27862058+luvs01@users.noreply.github.com> --- .../src/content/docs/reference/adapters.md | 14 +++ .../docs/reference/codex-retry-diagnostics.md | 96 +++++++++++++++++++ src/bridge/internal.ts | 12 +++ src/lib/errors.ts | 12 ++- src/lib/retry-delay.ts | 10 ++ structure/transports/responses-wire-shapes.md | 9 ++ structure/transports/responses.md | 4 +- tests/adapters/adapter-error-inline.test.ts | 81 +++++++++++++++- .../responses-grok-devin-preflight.test.ts | 30 +++--- tests/server/retry-delay-hardening.test.ts | 33 ++++++- 10 files changed, 280 insertions(+), 21 deletions(-) create mode 100644 docs-site/src/content/docs/reference/codex-retry-diagnostics.md diff --git a/docs-site/src/content/docs/reference/adapters.md b/docs-site/src/content/docs/reference/adapters.md index c4c5c087645..b2b8488b2df 100644 --- a/docs-site/src/content/docs/reference/adapters.md +++ b/docs-site/src/content/docs/reference/adapters.md @@ -613,6 +613,20 @@ configuration that names the old id is rewritten at startup. allowance on standalone turns surface the original 429 without an early retry. The final 429 preserves the stated delay as a cooldown hint. A `~` in its message marks a delay recovered from a secondhand trailer sentence rather than an exact header value. +- For Codex Responses streams, known typed rate-limit failures with a valid delay are normalized to + `rate_limit_exceeded` with `Please try again in Ns.` before the original redacted detail. + This lets Codex honor the stated delay and use its native reconnect notification without + adding a reasoning item to conversation history. Client retries are finite and controlled by + the client's `stream_max_retries`; this does not promise recovery after app shutdown or restart. + Leave `OPENCODEX_DEVIN_STATED_RESET_WAIT_MS` unset or `0` to let the client own the wait. + A positive proxy allowance keeps the existing proxy-owned wait; the client only learns of a + final refusal afterwards, and client retries can multiply the proxy's per-request attempts. + Combo target/account failover and Grok HTTP 429 handling retain their existing ordering. + The exact UI placement and text depend on the Codex version; this is not a custom countdown. + Message-only rate-limit errors use the same longest-delay-first formatting. Typed errors + without a usable delay retain their original code. Client retries create new HTTP requests; + they do not share one proxy request's send counter or cumulative wait allowance. This + compatibility mapping does not replace the controls of an explicitly enabled proxy wait. - Experimental unofficial bridge; not shown in the dashboard preset by default. See the [provider guide](/guides/providers/) for login instructions. diff --git a/docs-site/src/content/docs/reference/codex-retry-diagnostics.md b/docs-site/src/content/docs/reference/codex-retry-diagnostics.md new file mode 100644 index 00000000000..532ada6ddfd --- /dev/null +++ b/docs-site/src/content/docs/reference/codex-retry-diagnostics.md @@ -0,0 +1,96 @@ +--- +title: Codex retry diagnostics +description: Distinguish rate-limit advice, automatic retransmission, provider recovery, and Desktop notifications. +--- + +OpenCodex's [Devin adapter](/reference/adapters/#devin) preserves a usable retry +hint in the error contract Codex understands. That is not a guarantee that +Desktop displays a reconnect row, or that the provider accepts the next request. + +## Who waits + +With `OPENCODEX_DEVIN_STATED_RESET_WAIT_MS` unset or `0`, the Devin adapter does +not hold a refused request for the stated cooldown. Codex receives the final +failure and owns its bounded retry policy. A positive value opts into the +existing proxy-owned wait; it delays delivery of the final failure and can +compound with client retries. This compatibility mapping changes neither that +setting nor Codex's retry count. A process-only override need not survive restart. + +`rate_limit_exceeded` plus `Please try again in Ns.` is retry advice, not a +countdown event. Codex's own retry policy, cancellation and lifecycle still +apply. Do not add synthetic reasoning, tool, assistant or success items to make +waiting visible: those items can contaminate history or imply work happened. +Do not shorten the provider delay or provoke an extra request to advance the +client's retry counter. + +## Why the first reconnect row can be absent + +The public Codex `rust-v0.158.0-alpha.2.1` source gates the ordinary stream-retry +notification in +[`handle_response_stream_error`](https://github.com/openai/codex/blob/rust-v0.158.0-alpha.2.1/codex-rs/core/src/responses_retry.rs): + +```rust +let report_error = retry_count > 1 + || cfg!(debug_assertions) + || !sess.services.model_client.responses_websocket_enabled(); +``` + +In a release build, the first retry in this loop can therefore wait without +emitting `Reconnecting...` when the internal WebSocket-enabled predicate is +true. The selected delay is not part of that notification predicate, and the +sleep is outside the `if report_error` block. A long server-advised wait can be +silent too. This is a version-specific source observation, not proof that any +particular Desktop request took that branch. Do not infer the internal predicate +from a model name, an HTTP status, or the transport seen on one proxy hop. + +An enabled notification subscription and a renderer capable of drawing the row +only establish that the receiving path exists. They do not prove that the engine +emitted an event for the affected turn. Likewise, collapsed provider details can +explain missing detail text, but not a missing reconnect row by themselves. + +The corresponding app-server +[`ErrorNotification`](https://github.com/openai/codex/blob/rust-v0.158.0-alpha.2.1/codex-rs/app-server-protocol/schema/typescript/v2/ErrorNotification.ts) +contains `willRetry`, `threadId` and `turnId`. Observe emission, delivery to the +matching turn, and rendering separately. A proxy `response.failed` frame is not +itself that app-server notification. Changing this engine-side first-notification +policy requires a Codex change; an OpenCodex error-message rewrite cannot force +an event through a branch that does not emit it. + +## Read-only verification + +Keep the existing app, proxy, settings and test turn unchanged while collecting +evidence. Restarting, sending a new turn, or switching transports changes the +experiment. In an isolated follow-up, compare the same controlled failure under +both internal WebSocket states and inspect the first and subsequent retries; +do not call that an actual Desktop-rendering test unless the UI is observed. + +For the affected conversation and active turn, distinguish these outcomes: + +| Question | Evidence required | +| --- | --- | +| Was usable advice delivered? | The final failure's code, normalized delay and response-end time. | +| Did automatic retransmission occur? | A correlated next request after that failure, without a manual send. Measure from the failure response end to the next request start, not from the first request start. | +| Did the provider recover? | A successful response and continuation of the affected work, not merely another request or an unrelated successful turn. | +| Was a retry notification emitted and delivered? | The matching app-server error event and its retry flag, not subscription configuration alone. | +| Was the notice visible? | Observation of the affected Desktop turn; protocol or source-code checks alone are insufficient. | + +Approximate provider advice and scheduling overhead can produce a small timing +difference. If retransmission reaches a socket-close error and a later attempt +receives a new rate limit, record retransmission as working but provider recovery +as incomplete. The new refusal's delay is a new observation, not a timer inherited +from the previous refusal. An `inProgress` turn by itself proves neither recovery +nor correct rendering. Retry exhaustion, cancellation and app-restart recovery +must be assessed separately. + +Publish only the validation scope and aggregate durations/outcomes. Keep raw logs, +request bodies, credentials, account data, private paths, and thread/conversation/ +request/trace identifiers out of public PRs and diagnostic reports. + +## Regression-test scope + +`tests/server/retry-delay-hardening.test.ts` checks that 120-, 900-, 1,800-, +2,460- and 3,600-second advice survives formatting unchanged; a longer hint stays +first when a shorter one also exists; and an intervening untimed disconnect does +not make a later refusal inherit an old delay. These are pure parser/formatter +checks. They do not wait for an hour, exercise Codex's notification predicate, +verify an app restart, or prove recovery from a live provider failure. diff --git a/src/bridge/internal.ts b/src/bridge/internal.ts index cd126ca60aa..4b1543d1f13 100644 --- a/src/bridge/internal.ts +++ b/src/bridge/internal.ts @@ -15,6 +15,7 @@ import { type OcxErrorPayload, } from "../lib/errors"; import { redactSecretString } from "../lib/redact"; +import { formatRetryAfterAdvice } from "../lib/retry-delay"; import { usageDisplayTotalTokens } from "../usage/totals"; export function uuid(): string { @@ -144,6 +145,17 @@ export function adapterFailureFromEvent(event: Extract { + const refusal = (code = "resource_exhausted", message = "Devin cloud error resource_exhausted; retry after ~900s") => ({ + type: "error" as const, status: 429, errorType: "rate_limit_error", code, message, retryable: true, + }); + + test.each(["resource_exhausted", "rate_limit_exceeded", "slow_down"])("canonicalizes %s without changing the original event", code => { + const event = refusal(code); + const before = JSON.stringify(event); + const result = adapterFailureFromEvent(event); + expect(result).toMatchObject({ httpStatus: 429, error: { + type: "rate_limit_error", code: "rate_limit_exceeded", + message: `Please try again in 900s. ${event.message}`, + } }); + expect(JSON.stringify(event)).toBe(before); + }); + + test("uses the longest lower bound before any shorter client-readable hint", () => { + const event = refusal("resource_exhausted", "Please try again in 1s. Retry-After: 1h30m"); + expect(adapterFailureFromEvent(event).error.message).toStartWith("Please try again in 5400s."); + expect(adapterFailureFromEvent(refusal("rate_limit_exceeded", "Please try again in 1800s.")).error.message) + .toBe("Please try again in 1800s."); + }); + + test("does not invent a delay or rewrite unrelated explicit verdicts", () => { + for (const message of ["busy", "retry after ~0s", "retry after 2 months"]) { + expect(adapterFailureFromEvent(refusal("resource_exhausted", message)).error) + .toMatchObject({ code: "resource_exhausted", message }); + } + for (const code of ["insufficient_quota", "request_send_budget_exhausted", "invalid_argument", "vendor_custom"]) { + const event = refusal(code); + expect(adapterFailureFromEvent(event).error).toMatchObject({ code, message: event.message }); + } + const not429 = { ...refusal(), status: 503, errorType: "server_error" }; + expect(adapterFailureFromEvent(not429).error).toMatchObject({ code: "resource_exhausted", message: not429.message }); + }); + + test("typed and message-only errors share the same longest-first client advice", () => { + const message = "rate limit exceeded: Please try again in 1s. retry after ~900s"; + const typed = adapterFailureFromEvent(refusal("resource_exhausted", message)); + const untyped = adapterFailureFromMessage(message); + expect(typed).toEqual(untyped); + expect(typed.error.message).toBe(`Please try again in 900s. Provider detail: ${message}`); + expect(adapterFailureFromMessage(typed.error.message).error.message).toBe(typed.error.message); + }); + + test("redacts the retained provider detail before appending retry advice", () => { + const secret = "sk-" + "a".repeat(32); + const event = refusal("resource_exhausted", `retry after ~30s; api_key=${secret}`); + const result = adapterFailureFromEvent(event); + expect(result.error.message).toStartWith("Please try again in 30s."); + expect(result.error.message).not.toContain(secret); + }); + + test("SSE and buffered failures agree and do not synthesize reasoning or success", async () => { + const event = refusal(); + async function* source() { yield event; } + const frames = await collectSse(bridgeToResponsesSSE(source(), "devin/swe-2")); + const response = frames.find(frame => frame.event === "response.failed")!.data.response as Record; + const buffered = buildResponseJSON([event], "devin/swe-2"); + expect(response.error).toEqual(buffered.error); + expect(response.error).toMatchObject({ code: "rate_limit_exceeded", message: `Please try again in 900s. ${event.message}` }); + expect(frames.some(frame => frame.event?.includes("reasoning") || frame.event === "response.completed")).toBe(false); + expect(buffered.status).toBe("failed"); + }); + + test("the canonical failure remains pre-output and available to combo failover", async () => { + async function* source() { yield refusal(); } + const response = new Response(bridgeToResponsesSSE(source(), "devin/swe-2"), { + headers: { "content-type": "text/event-stream" }, + }); + const preflight = await preflightComboStreamResponse(response, { provider: "devin", model: "swe-2" }); + expect(preflight.kind).toBe("failed"); + expect(preflight.response.status).toBe(429); + expect((await preflight.response.json()).error.code).toBe("rate_limit_exceeded"); + }); +}); + async function collect(gen: AsyncGenerator): Promise { const out: AdapterEvent[] = []; for await (const e of gen) out.push(e); diff --git a/tests/responses/responses-grok-devin-preflight.test.ts b/tests/responses/responses-grok-devin-preflight.test.ts index f291ddfd131..34909f10501 100644 --- a/tests/responses/responses-grok-devin-preflight.test.ts +++ b/tests/responses/responses-grok-devin-preflight.test.ts @@ -20,6 +20,7 @@ const limit: AdapterEvent = { type: "error", status: 429, errorType: "rate_limit_error", code: "resource_exhausted", retryable: true, message: "Cognition chat failed (resource_exhausted); retry after ~60s", }; +const clientLimitMessage = `Please try again in 60s. ${limit.message}`; mock.module("../../src/server/adapter-resolve", () => ({ ...resolver, resolveAdapter(provider: OcxProviderConfig, cache?: "none" | "short" | "long") { if (provider.adapter !== "devin") return originalResolve(provider, cache); @@ -113,7 +114,7 @@ test.each([ expect(response.headers.get("content-type")).toContain("application/json"); expect(response.headers.get("retry-after")).toBe("60"); expect(await response.json()).toEqual({ error: { - message: limit.message, type: "rate_limit_error", code: "rate_limit_exceeded", + message: clientLimitMessage, type: "rate_limit_error", code: "rate_limit_exceeded", } }); expect(calls).toBe(1); }); @@ -268,10 +269,15 @@ test("a replay-unsafe heartbeat leaves the error in SSE", async () => { expect(calls).toBe(1); }); -test("other clients keep their existing SSE response", async () => { +test("Codex receives native retry advice in SSE without a synthetic reasoning item", async () => { const response = await run({ surface: "codex" }); expect(response.status).toBe(200); - expect(await response.text()).toContain("response.failed"); + const frames = (await response.text()).split("\n") + .filter(line => line.startsWith("data: {")).map(line => JSON.parse(line.slice(6))); + const failed = frames.find(frame => frame.type === "response.failed"); + expect(failed.response.error).toMatchObject({ code: "rate_limit_exceeded", message: clientLimitMessage }); + expect(frames.some(frame => frame.type.includes("reasoning") || frame.type === "response.completed")).toBe(false); + expect(calls).toBe(1); }); test.each([false, true])("Grok starts SSE after bounded Devin preflight (heartbeat=%s)", async heartbeat => { @@ -372,18 +378,18 @@ test("cancellation before the first event aborts the producer", async () => { expect(calls).toBe(1); }); -const bufferedExclusions: { name: string; surface?: "grok" | "codex"; source: AdapterEvent[] }[] = [ - { name: "other client", surface: "codex", source: [limit] }, - { name: "replay-unsafe activity", source: [{ type: "heartbeat", replayUnsafe: true }, limit] }, - { name: "local send budget", source: [{ ...limit, code: SEND_BUDGET_EXHAUSTED_CODE }] }, - { name: "non-429 failure", source: [{ ...limit, status: 503, errorType: "upstream_error" }] }, +const bufferedExclusions: { name: string; surface?: "grok" | "codex"; source: AdapterEvent[]; message: string }[] = [ + { name: "other client", surface: "codex", source: [limit], message: clientLimitMessage }, + { name: "replay-unsafe activity", source: [{ type: "heartbeat", replayUnsafe: true }, limit], message: clientLimitMessage }, + { name: "local send budget", source: [{ ...limit, code: SEND_BUDGET_EXHAUSTED_CODE }], message: limit.message }, + { name: "non-429 failure", source: [{ ...limit, status: 503, errorType: "upstream_error" }], message: limit.message }, ]; -test.each(bufferedExclusions)("buffered $name retains its existing JSON result", async ({ surface, source }) => { +test.each(bufferedExclusions)("buffered $name retains its existing JSON result", async ({ surface, source, message }) => { events = source; const response = await run({ stream: false, surface }); expect(response.status).toBe(200); expect(response.headers.get("retry-after")).toBeNull(); - expect(await response.json()).toMatchObject({ status: "failed", error: { message: limit.message } }); + expect(await response.json()).toMatchObject({ status: "failed", error: { message } }); }); test("buffered output before a 429 is retained exactly once", async () => { @@ -392,7 +398,7 @@ test("buffered output before a 429 is retained exactly once", async () => { expect(response.status).toBe(200); expect(await response.json()).toMatchObject({ status: "failed", output: [{ content: [{ type: "output_text", text: "answer" }] }], - error: { message: limit.message }, + error: { message: clientLimitMessage }, }); expect(calls).toBe(1); }); @@ -415,6 +421,6 @@ test.each([true, false])("OAuth replay preserves unsafe activity through heartbe const response = await run({ stream, oauthFailoverEnabled: true }); expect(response.status).toBe(200); if (stream) expect(await response.text()).toContain("response.failed"); - else expect(await response.json()).toMatchObject({ status: "failed", error: { message: limit.message } }); + else expect(await response.json()).toMatchObject({ status: "failed", error: { message: clientLimitMessage } }); expect(calls).toBe(2); }); diff --git a/tests/server/retry-delay-hardening.test.ts b/tests/server/retry-delay-hardening.test.ts index f0e5fa6db4a..08c65e19405 100644 --- a/tests/server/retry-delay-hardening.test.ts +++ b/tests/server/retry-delay-hardening.test.ts @@ -1,11 +1,12 @@ import { describe, expect, test } from "bun:test"; import { adapterFailureFromMessage, parseRetryAfterFromMessage } from "../../src/lib/errors"; +import { formatRetryAfterAdvice } from "../../src/lib/retry-delay"; describe("stated reset duration boundaries", () => { test("the error adapter retains a local parser binding after extraction", () => { const failure = adapterFailureFromMessage("Devin rate limit: reset in 5 minutes 30 seconds"); expect(failure.httpStatus).toBe(429); - expect(failure.error.message).toBe("Devin rate limit: reset in 5 minutes 30 seconds Please try again in 330s."); + expect(failure.error.message).toBe("Please try again in 330s. Devin rate limit: reset in 5 minutes 30 seconds"); expect(failure.error.code).toBe("rate_limit_exceeded"); }); @@ -47,3 +48,33 @@ describe("stated reset duration boundaries", () => { expect(parseRetryAfterFromMessage(message)).toBeUndefined(); }); }); + +// These are wire-advice checks, not elapsed-time or Desktop-rendering tests. +describe("Codex long-wait advice", () => { + test.each([120, 900, 1800, 2460, 3600])("preserves a %i-second hint without shortening it", seconds => { + const message = `Devin rate limit; retry after ~${seconds}s`; + const formatted = formatRetryAfterAdvice(message); + expect(formatted).toBe(`Please try again in ${seconds}s. ${message}`); + expect(parseRetryAfterFromMessage(formatted!)).toBe(seconds); + expect(formatRetryAfterAdvice(formatted!)).toBe(formatted); + }); + + test.each([2460, 3600])("puts the %i-second lower bound before shorter advice", seconds => { + const message = `Please try again in 1s. retry after ~${seconds}s`; + const formatted = formatRetryAfterAdvice(message); + expect(formatted).toBe(`Please try again in ${seconds}s. Provider detail: ${message}`); + expect(parseRetryAfterFromMessage(formatted!)).toBe(seconds); + expect(formatRetryAfterAdvice(formatted!)).toBe(formatted); + }); + + test("a later refusal uses its own delay; an intervening disconnect has no invented advice", () => { + const first = "Devin rate limit; retry after ~2460s"; + const disconnect = "upstream_server_error: socket closed"; + const later = "Devin rate limit; retry after ~120s"; + expect(formatRetryAfterAdvice(first)).toBe(`Please try again in 2460s. ${first}`); + expect(parseRetryAfterFromMessage(disconnect)).toBeUndefined(); + expect(formatRetryAfterAdvice(disconnect)).toBeUndefined(); + expect(formatRetryAfterAdvice(later)).toBe(`Please try again in 120s. ${later}`); + expect(formatRetryAfterAdvice("resource_exhausted")).toBeUndefined(); + }); +}); From ea7bd16abe061180b997e85e4367d1107f1b5c77 Mon Sep 17 00:00:00 2001 From: terin <100397903+sh940701@users.noreply.github.com> Date: Wed, 30 Sep 2026 03:28:59 +0900 Subject: [PATCH 13/26] fix(claude): end a passthrough stream on an upstream reset with an Anthropic error event (#6235) * fix(claude): end a passthrough stream on an upstream reset with an Anthropic error event tapAnthropicSseForLog relays the streamed body of the native Anthropic passthrough and of the managed native Messages lane. When the upstream socket reset after the 200 had gone out, the read rejection was handed to controller.error(): the client saw a connection reset (Bun 1.4.0) or a bare chunked EOF (Bun 1.3.x) with no protocol terminal, Bun printed the error stack to the service log, and the request log recorded the truncated turn as 200 / closeReason "terminal", which classifies as completed. A read failure now ends the body the way a stall already did: a blank-line boundary, an Anthropic `event: error` frame (`api_error`), and a clean close. The row matches the Responses relay's onReadError: status 502, terminalStatus "failed", transportPhase "mid_stream", a synthetic terminal source, the attempt marked streamAborted, and the redacted reason in upstreamError. messages-native forwards terminalStatus to its final log. A translator budget overflow is a local cap, not an upstream failure, and keeps erroring the stream so the non-streaming fold still answers 413. * fix(claude): keep cancels and finished turns out of the passthrough reset path Review follow-up. A read failure that is not an upstream failure keeps a non-failure outcome: - the cancel signal is already aborted (Bun can reject the read before it dispatches the abort listener): 499 client_cancel, as relay.ts checks; - the managed lane's own abort (shutdown, turn release): the tap now gets the upstream controller's signal, so it is a cancel too; - message_stop or an upstream error event was already seen, including one still buffered without its blank-line delimiter: 200, no extra frame. The reset path also cancels the upstream reader like every other exit, the test fixture clears its pending socket timers on stop, and the tests assert this proxy's message prefix instead of Bun's error text. * docs(structure): state the passthrough read-failure exceptions without a count * fix(claude): close the non-streaming fold's reset row like the streaming one Review follow-up. The managed lane's non-streaming fold answered a mid-stream reset with fail(502), whose default meta closed the row as closeReason "non_stream" without a terminalStatus, while the streaming lane's row for the same reset is terminal + failed. The fold now finishes the row with the tap's meta when the tap reported a failed terminal. An upstream error event (no reset) keeps its existing row. The structure doc also states that the logged status differs by frame: a stall or the byte cap keeps 200 with body_stall/body_overflow (incomplete), a reset is 502 (failed). * docs(structure): name the local body limits apart from upstream read failures * fix(claude): restore a cut-off terminal's delimiter and keep the fold's tap verdict CodeRabbit follow-up (outside-diff findings on 21a21d8ce): - When the read fails after message_stop arrived without its blank line, the tap now appends "\n\n" before closing. An SSE parser does not dispatch an event that EOF cuts off, so the client would otherwise miss the terminal the tap had counted. - The managed lane's non-streaming fold closes its row with the tap's meta for a stall or the byte cap as well as a reset, so body_stall / body_overflow survive instead of becoming non_stream. A plain upstream error event keeps the non_stream row. --- src/server/claude-messages.ts | 111 ++++++++--- src/server/messages-native.ts | 22 ++- structure/clients/claude-desktop.md | 4 + .../claude-native-passthrough.test.ts | 180 ++++++++++++++++++ .../messages-native.test.ts | 128 +++++++++++++ tests/helpers/truncated-sse-upstream.ts | 47 +++++ 6 files changed, 458 insertions(+), 34 deletions(-) create mode 100644 tests/helpers/truncated-sse-upstream.ts diff --git a/src/server/claude-messages.ts b/src/server/claude-messages.ts index 7f4d0dbd994..45eb39e9585 100644 --- a/src/server/claude-messages.ts +++ b/src/server/claude-messages.ts @@ -14,6 +14,7 @@ import { resolveAdmissionModelScope, } from "./admission-model-scope"; import { jsonUtf8Bytes } from "../lib/json-byte-size"; +import { redactSecretString } from "../lib/redact"; import { sseFieldValue } from "../lib/sse-decoder"; import { enforceAnthropicImageLimits, sniffImageDimensions } from "../adapters/anthropic-image-guard"; import { normalizeAnthropicImages } from "../adapters/anthropic-image-normalize"; @@ -295,46 +296,55 @@ export interface PassthroughBodyGuard { } type PassthroughCloseReason = "terminal" | "client_cancel" | "body_stall" | "body_overflow"; +type PassthroughFinalizeMeta = { closeReason: PassthroughCloseReason; terminalStatus?: "failed" }; /** * Tap an Anthropic-vocabulary SSE stream for the request log (usage + terminal), * bounding body occupancy: idle (silence-only, timed ONLY while a reader.read() is * pending so downstream backpressure never counts as upstream inactivity) and a - * cumulative byte cap. On stall/overflow it appends a protocol-compatible Anthropic - * `event: error` terminal frame after a blank-line boundary, closes, and cancels the - * upstream reader — never a total-wall-clock bound (slow-but-alive streams live). + * cumulative byte cap. On stall/overflow, or when the upstream read fails after the + * headers went out, it appends a protocol-compatible Anthropic `event: error` terminal + * frame after a blank-line boundary, closes, and cancels the upstream reader — never a + * total-wall-clock bound (slow-but-alive streams live). * Exported for deterministic unit tests. */ export function tapAnthropicSseForLog( upstream: ReadableStream, logCtx: RequestLogContext, - finalize: (status: number, meta: { closeReason: PassthroughCloseReason }) => void, + finalize: (status: number, meta: PassthroughFinalizeMeta) => void, guard?: PassthroughBodyGuard, ): ReadableStream { const decoder = new TextDecoder(); const encoder = new TextEncoder(); let buffer = ""; let usageAcc: Rec = {}; + // message_stop or an upstream error event: the turn's own terminal already went through. + let terminalSeen = false; + const inspectFrame = (frame: string) => { + const dataLine = frame + .split("\n") + .map(l => sseFieldValue(l, "data")) + .filter((v): v is string => v !== null) + .join(""); + if (!dataLine) return; + let data: unknown; + try { data = JSON.parse(dataLine); } catch { return; } + if (!isRec(data)) return; + if (data.type === "message_start" && isRec(data.message) && isRec(data.message.usage)) { + usageAcc = { ...usageAcc, ...data.message.usage }; + } else if (data.type === "message_delta" && isRec(data.usage)) { + usageAcc = { ...usageAcc, ...data.usage }; + } else if (data.type === "message_stop" || data.type === "error") { + terminalSeen = true; + } + }; const inspect = (chunk: Uint8Array) => { buffer += decoder.decode(chunk, { stream: true }); let sep: number; while ((sep = buffer.indexOf("\n\n")) !== -1) { const frame = buffer.slice(0, sep); buffer = buffer.slice(sep + 2); - const dataLine = frame - .split("\n") - .map(l => sseFieldValue(l, "data")) - .filter((v): v is string => v !== null) - .join(""); - if (!dataLine) continue; - let data: unknown; - try { data = JSON.parse(dataLine); } catch { continue; } - if (!isRec(data)) continue; - if (data.type === "message_start" && isRec(data.message) && isRec(data.message.usage)) { - usageAcc = { ...usageAcc, ...data.message.usage }; - } else if (data.type === "message_delta" && isRec(data.usage)) { - usageAcc = { ...usageAcc, ...data.usage }; - } + inspectFrame(frame); } }; const reader = upstream.getReader(); @@ -345,13 +355,7 @@ export function tapAnthropicSseForLog( const recordUsage = () => { logCtx.usage = anthropicUsageToOcx(Object.keys(usageAcc).length > 0 ? usageAcc : undefined); }; - const failBody = (closeReason: "body_stall" | "body_overflow", errType: string, message: string) => { - if (settled) return; - settled = true; - idle.cancel(); - detachAbort(); - recordUsage(); - finalize(200, { closeReason }); + const closeWithErrorFrame = (errType: string, message: string) => { const payload = JSON.stringify({ type: "error", error: { type: errType, message } }); try { // Leading blank line terminates any partial SSE block so the frame parses cleanly @@ -359,6 +363,15 @@ export function tapAnthropicSseForLog( tapController?.enqueue(encoder.encode(`\n\nevent: error\ndata: ${payload}\n\n`)); tapController?.close(); } catch { /* client already torn down */ } + }; + const failBody = (closeReason: "body_stall" | "body_overflow", errType: string, message: string) => { + if (settled) return; + settled = true; + idle.cancel(); + detachAbort(); + recordUsage(); + finalize(200, { closeReason }); + closeWithErrorFrame(errType, message); reader.cancel(new DOMException(message, closeReason === "body_stall" ? "TimeoutError" : "QuotaExceededError")).catch(() => {}); }; const idle = idleDeadline(guard?.stallMs ?? 0, () => { @@ -427,12 +440,56 @@ export function tapAnthropicSseForLog( controller.enqueue(value); } catch (err) { if (settled) return; + // Bun can settle a fetch body read before it dispatches the abort listeners + // (consumeForInspection in relay.ts): read the signal itself before calling this + // rejection an upstream failure. + if (guard?.reqSignal?.aborted) { + onClientAbort(); + return; + } settled = true; idle.cancel(); detachAbort(); + // A read error can follow the last SSE block before its blank-line delimiter. Count + // that block before deciding how the turn ended, as the Responses relay does. + const terminalBeforeTail = terminalSeen; + const tail = buffer + decoder.decode(); + buffer = ""; + if (tail) inspectFrame(tail); recordUsage(); - finalize(200, { closeReason: "terminal" }); - try { controller.error(err); } catch { /* torn down */ } + if (isTranslatorBudgetExceededError(err)) { + // A local cap, not an upstream failure: the non-streaming native Messages fold + // maps this error to a 413 itself, so it still errors the stream. + finalize(200, { closeReason: "terminal" }); + try { controller.error(err); } catch { /* torn down */ } + return; + } + if (terminalSeen) { + // Only the transport trailer was lost; the client already has the turn's terminal. + // The Responses relay likewise reports a read error only without a seen terminal. + finalize(200, { closeReason: "terminal" }); + try { + // An SSE parser drops an event that EOF cuts off before its blank line, so restore + // the delimiter when the terminal was only found in that unterminated tail. + if (!terminalBeforeTail) controller.enqueue(encoder.encode("\n\n")); + controller.close(); + } catch { /* torn down */ } + reader.cancel(err).catch(() => {}); + return; + } + // The upstream read failed after the 200 went out (a mid-stream socket reset). Log it + // the way the Responses relay does (onReadError): a truncated body is a failed turn, + // not a completed one. The client gets the same Anthropic error terminal as a stall, + // instead of a connection reset — or, on some Bun releases, a bare EOF that reads as + // a finished message. + const message = redactSecretString(`anthropic passthrough upstream stream failed: ${err instanceof Error ? err.message : String(err)}`); + logCtx.transportPhase = "mid_stream"; + logCtx.terminalSource = "synthetic"; + logCtx.upstreamError = message.slice(0, 500); + if (logCtx.activeAttempt) logCtx.activeAttempt.streamAborted = true; + finalize(502, { terminalStatus: "failed", closeReason: "terminal" }); + closeWithErrorFrame("api_error", message); + reader.cancel(err).catch(() => {}); } }, cancel(reason) { diff --git a/src/server/messages-native.ts b/src/server/messages-native.ts index 5a88dc50aa3..d03b6a33299 100644 --- a/src/server/messages-native.ts +++ b/src/server/messages-native.ts @@ -150,7 +150,7 @@ export interface HandleNativeMessagesOptions { callerAnthropicBeta?: string | null; } -type FinishLog = (status: number, message?: string, closeReason?: FinalRequestLogMeta["closeReason"]) => void; +type FinishLog = (status: number, message?: string, meta?: FinalRequestLogMeta) => void; /** Relay an upstream body unchanged, recording first output on its first non-empty chunk. */ function observeFirstChunk(body: ReadableStream, onFirst: () => void): ReadableStream { @@ -311,10 +311,10 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) attemptHandle.seal(logCtx.accountLogLabel); const { attempt } = attemptHandle; const finalLog = createFinalRequestLog(logIds, logCtx); - const finishLog: FinishLog = (status, message, closeReason = "non_stream") => { + const finishLog: FinishLog = (status, message, meta = { closeReason: "non_stream" }) => { if (finalLog.finished()) return; if (message) logCtx.upstreamError = redactSecretString(message).slice(0, 500); - finalLog.finish(status, { closeReason }); + finalLog.finish(status, meta); }; const bindUsage = (usage: OcxUsage | undefined) => { if (!usage) return; @@ -597,7 +597,9 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) const contentType = response.headers.get("content-type")?.toLowerCase() ?? ""; if (contentType.includes("text/event-stream") && response.body) { - const bodyGuard = resolvePassthroughBodyGuard(config, req.signal); + // `upstream` follows req.signal and is also what shutdown and turn release abort, so both + // the client leaving and this lane's own abort read as a cancel, not as a failed stream. + const bodyGuard = resolvePassthroughBodyGuard(config, upstream.signal); const observed = logIds ? observeFirstChunk(response.body, () => recordFirstOutput(logCtx, logIds.start)) : response.body; const renamed = activeRequest.oauthToolNames ? restoreOAuthToolNamesInSse(observed, activeRequest.oauthToolNames, translatorBudget) @@ -609,7 +611,7 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) try { cleanupAbort(); bindUsage(logCtx.usage); - finishLog(status, undefined, meta.closeReason); + finishLog(status, undefined, meta); if (meta.closeReason !== "terminal") upstream.abort(); } finally { releaseStreamTurn(); @@ -621,9 +623,10 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) }); } // A non-streaming caller whose upstream streamed anyway: fold the stream into one message. - const tapState: { closeReason?: FinalRequestLogMeta["closeReason"] } = {}; + const tapState: { closeReason?: FinalRequestLogMeta["closeReason"]; meta?: FinalRequestLogMeta } = {}; const tapped = tapAnthropicSseForLog(source, logCtx, (_status, meta) => { tapState.closeReason = meta.closeReason; + tapState.meta = meta; }, bodyGuard); try { const message = await collectAnthropicMessage(tapped, requestedModel, translatorBudget); @@ -634,7 +637,12 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) bindUsage(logCtx.usage); if (message.type === "error") { const error = isRec(message.error) ? message.error : {}; - return fail(502, typeof error.message === "string" ? error.message : "upstream stream failed", "api_error"); + const text = typeof error.message === "string" ? error.message : "upstream stream failed"; + // The tap's own verdict (a reset, a stall, the byte cap) closes this row with the same meta + // as the streaming lane's row; a plain upstream error event keeps the non_stream row. + const tapMeta = tapState.meta; + if (tapMeta && (tapMeta.terminalStatus || tapMeta.closeReason !== "terminal")) finishLog(502, text, tapMeta); + return fail(502, text, "api_error"); } finishLog(200); return Response.json(message); diff --git a/structure/clients/claude-desktop.md b/structure/clients/claude-desktop.md index a01466ab96b..e256089a2a8 100644 --- a/structure/clients/claude-desktop.md +++ b/structure/clients/claude-desktop.md @@ -462,4 +462,8 @@ The [compaction routing override](../transports/responses-failover.md#compaction Native Anthropic passthrough in `src/server/claude-messages.ts` forwards the caller's body except for tool-call ids: `sanitizePassthroughToolCallIds` runs the request-scoped allocator from `src/adapters/tool-call-id.ts` over every `*tool_use` id and `*tool_result` `tool_use_id`. Conforming ids are reserved first and stay byte-identical, a non-conforming or overlength id is rewritten to a conforming id of at most 64 characters with call/result pairing kept, and an empty id throws `AnthropicRequestError`, so the request fails with a local 400 before the upstream fetch. `tests/claude-integration/claude-native-passthrough.test.ts` covers rewriting, pairing, the empty id, the overlength id and collision with an existing valid id. +## Native passthrough stream terminals + +`tapAnthropicSseForLog` in `src/server/claude-messages.ts` relays the streamed body of both the native passthrough and the managed native Messages lane (`src/server/messages-native.ts`). The response headers are already sent, so a stall, a byte-cap overflow, or an upstream read failure ends the body with an Anthropic `event: error` frame after a blank-line boundary and a clean close: `timeout_error` for an idle stall, `api_error` for the byte cap, and `api_error` when an upstream read fails mid-stream (a socket reset). The mid-stream reset is logged like the Responses relay's read error: status 502, `terminalStatus: "failed"`, `closeReason: "terminal"`, `transportPhase: "mid_stream"`, a synthetic terminal source, the attempt marked `streamAborted`, the redacted reason in `upstreamError`, and the usage seen before the reset. The non-streaming fold in the managed lane closes its row with the tap's meta for a reset, a stall or the byte cap, so its row matches the streaming lane's. The request is not replayed. The logged status is not uniform across these frames: a stall and the byte cap keep status 200 with `closeReason` `body_stall` or `body_overflow`, which classify as `incomplete`, while a reset is a 502 that classifies as `failed`. Some read failures are not upstream failures. When the cancel signal is already aborted, the rejection is a `499` client cancel, because Bun can reject the read before it dispatches the abort listener. The managed lane passes its upstream controller's signal, so shutdown and turn release count as cancels too. When `message_stop` or an upstream `error` event has been seen, including one still in the buffer without its blank-line delimiter, the turn is complete: it logs 200 and closes with no error frame. A terminal found only in that unterminated tail gets its blank line restored, because an SSE parser drops an event that EOF cuts off. A translator budget overflow is a local cap, so it still errors the stream, and the non-streaming fold answers it with 413. `tests/claude-integration/claude-native-passthrough.test.ts` and `tests/claude-integration/messages-native.test.ts` cover both lanes against an upstream that resets after a partial or a complete body, plus both cancel paths. + Linked-machine data uses the [connection-bound relay contract](../remote-link.md#connection-bound-relay-authentication); client-local credentials and routing policy remain unchanged. diff --git a/tests/claude-integration/claude-native-passthrough.test.ts b/tests/claude-integration/claude-native-passthrough.test.ts index fe7b1ccb20f..c01425f744e 100644 --- a/tests/claude-integration/claude-native-passthrough.test.ts +++ b/tests/claude-integration/claude-native-passthrough.test.ts @@ -11,6 +11,10 @@ import { startServer } from "../../src/server"; import type { OcxConfig } from "../../src/types"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { startTruncatedSseUpstream } from "../helpers/truncated-sse-upstream"; +import { tapAnthropicSseForLog } from "../../src/server/claude-messages"; +import type { RequestLogContext } from "../../src/server/request-log"; +import { TranslatorBudgetExceededError } from "../../src/lib/translator-budget"; let testDir = ""; let previousHome: string | undefined; @@ -788,3 +792,179 @@ test("an overlength id is rewritten within 64 characters and a colliding valid i upstream.stop(true); } }); + +// --- Mid-stream upstream reset: the stream had started, then the upstream socket went away --- + +const PARTIAL_TURN_SSE = [ + `event: message_start\ndata: ${JSON.stringify({ type: "message_start", message: { id: "msg_up", type: "message", role: "assistant", content: [], model: "claude-fable-5", stop_reason: null, usage: { input_tokens: 12, output_tokens: 1 } } })}\n\n`, + `event: content_block_start\ndata: ${JSON.stringify({ type: "content_block_start", index: 0, content_block: { type: "text", text: "" } })}\n\n`, + `event: content_block_delta\ndata: ${JSON.stringify({ type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "half an ans" } })}\n\n`, +].join(""); + +test("a mid-stream upstream reset ends the native stream with an Anthropic error event and logs a failed turn", async () => { + const { clearRequestLogsForTests } = await import("../../src/server/request-log"); + clearRequestLogsForTests(); + const upstream = startTruncatedSseUpstream(PARTIAL_TURN_SSE); + saveConfig(cfg(`http://127.0.0.1:${upstream.port}`)); + const server = startServer(0); + try { + const res = await fetch(new URL("/v1/messages?beta=true", server.url), { + method: "POST", + headers: OAUTH_HEADERS, + body: JSON.stringify(claudeBody()), + }); + expect(res.status).toBe(200); + // The body ends cleanly with a protocol terminal the client can act on, instead of a + // connection reset (or, on some Bun releases, a bare chunked EOF) after "half an ans". + const text = await res.text(); + expect(text).toContain("half an ans"); + expect(text).toContain("\n\nevent: error\ndata: "); + const errorFrame = JSON.parse(text.slice(text.lastIndexOf("data: ") + 6).trim()) as { type: string; error: { type: string; message: string } }; + expect(errorFrame.type).toBe("error"); + expect(errorFrame.error.type).toBe("api_error"); + expect(errorFrame.error.message).toContain("anthropic passthrough upstream stream failed: "); + // The committed request is not replayed. + expect(upstream.requests()).toBe(1); + + const logs = logsFromApiBody<{ + status?: number; + terminalStatus?: string; + closeReason?: string; + transportPhase?: string; + terminalSource?: string; + failureCause?: string; + upstreamError?: string; + usage?: { inputTokens?: number }; + }>(await (await fetch(new URL("/api/logs?tail=1", server.url))).json()); + expect(logs).toHaveLength(1); + const row = logs[0]!; + // Same row the Responses relay writes for a mid-stream reset: a truncated 200 body is not a + // completed turn. + expect(row.status).toBe(502); + expect(row.terminalStatus).toBe("failed"); + expect(row.closeReason).toBe("terminal"); + expect(row.transportPhase).toBe("mid_stream"); + expect(row.terminalSource).toBe("synthetic"); + expect(row.failureCause).toBe("transport-ambiguous"); + expect(row.upstreamError).toContain("anthropic passthrough upstream stream failed: "); + // Usage seen before the reset is still recorded. + expect(row.usage?.inputTokens).toBe(12); + } finally { + await server.stop(true); + upstream.stop(); + } +}); + +test("a translator budget overflow still errors the tapped stream for callers that map it", async () => { + const overflow = new TranslatorBudgetExceededError("live_transient", 1024); + let sent = false; + const source = new ReadableStream({ + pull(controller) { + if (sent) { + controller.error(overflow); + return; + } + sent = true; + controller.enqueue(new TextEncoder().encode(PARTIAL_TURN_SSE)); + }, + }); + const calls: unknown[] = []; + const logCtx: RequestLogContext = { model: "claude-fable-5", provider: "anthropic-native" }; + const tapped = tapAnthropicSseForLog(source, logCtx, (status, meta) => calls.push({ status, ...meta }), { stallMs: 5_000, maxBytes: 0 }); + // The non-streaming native Messages fold turns this error into a 413; an error frame would + // have reached it as a generic 502 instead. + await expect(new Response(tapped).text()).rejects.toBe(overflow); + expect(calls).toEqual([{ status: 200, closeReason: "terminal" }]); + expect(logCtx.transportPhase).toBeUndefined(); +}); + +test("a read rejection that lands before the client abort listener still finalizes as a client cancel", async () => { + // Bun can settle a fetch body read before it dispatches the abort listeners (see + // consumeForInspection in src/server/relay.ts). Model that order: the signal is already + // aborted when the read rejects, and its listener has not run. + const signal = { aborted: false, reason: undefined as unknown, addEventListener() {}, removeEventListener() {} }; + let sent = false; + const source = new ReadableStream({ + pull(controller) { + if (sent) { + signal.aborted = true; + signal.reason = new DOMException("client went away", "AbortError"); + controller.error(signal.reason); + return; + } + sent = true; + controller.enqueue(new TextEncoder().encode(PARTIAL_TURN_SSE)); + }, + }); + const calls: unknown[] = []; + const logCtx: RequestLogContext = { model: "claude-fable-5", provider: "anthropic-native" }; + const tapped = tapAnthropicSseForLog(source, logCtx, (status, meta) => calls.push({ status, ...meta }), { + stallMs: 5_000, + maxBytes: 0, + reqSignal: signal as unknown as AbortSignal, + }); + const text = await new Response(tapped).text(); + expect(text).not.toContain("event: error"); + expect(calls).toEqual([{ status: 499, closeReason: "client_cancel" }]); + expect(logCtx.transportPhase).toBeUndefined(); + expect(logCtx.upstreamError).toBeUndefined(); +}); + +const COMPLETE_TURN_SSE = PARTIAL_TURN_SSE + [ + `event: content_block_stop\ndata: ${JSON.stringify({ type: "content_block_stop", index: 0 })}\n\n`, + `event: message_delta\ndata: ${JSON.stringify({ type: "message_delta", delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { output_tokens: 5 } })}\n\n`, + `event: message_stop\ndata: ${JSON.stringify({ type: "message_stop" })}\n\n`, +].join(""); + +test("a reset after message_stop is a finished turn: no error event and a completed row", async () => { + const { clearRequestLogsForTests } = await import("../../src/server/request-log"); + clearRequestLogsForTests(); + // Only the chunked-encoding trailer is lost; the turn itself arrived whole. + const upstream = startTruncatedSseUpstream(COMPLETE_TURN_SSE); + saveConfig(cfg(`http://127.0.0.1:${upstream.port}`)); + const server = startServer(0); + try { + const res = await fetch(new URL("/v1/messages?beta=true", server.url), { + method: "POST", + headers: OAUTH_HEADERS, + body: JSON.stringify(claudeBody()), + }); + const text = await res.text(); + expect(text.endsWith(`event: message_stop\ndata: ${JSON.stringify({ type: "message_stop" })}\n\n`)).toBe(true); + expect(text).not.toContain("event: error"); + const logs = logsFromApiBody<{ status?: number; closeReason?: string; transportPhase?: string; upstreamError?: string }>( + await (await fetch(new URL("/api/logs?tail=1", server.url))).json(), + ); + expect(logs).toHaveLength(1); + expect(logs[0]).toMatchObject({ status: 200, closeReason: "terminal" }); + expect(logs[0]!.transportPhase).toBeUndefined(); + expect(logs[0]!.upstreamError).toBeUndefined(); + } finally { + await server.stop(true); + upstream.stop(); + } +}); + +test("a terminal frame still in the buffer when the read fails counts as seen", async () => { + // The reset can land after message_stop but before its blank-line delimiter. + const withoutDelimiter = COMPLETE_TURN_SSE.slice(0, -2); + let sent = false; + const source = new ReadableStream({ + pull(controller) { + if (sent) { + controller.error(new Error("The socket connection was closed unexpectedly.")); + return; + } + sent = true; + controller.enqueue(new TextEncoder().encode(withoutDelimiter)); + }, + }); + const calls: unknown[] = []; + const logCtx: RequestLogContext = { model: "claude-fable-5", provider: "anthropic-native" }; + const tapped = tapAnthropicSseForLog(source, logCtx, (status, meta) => calls.push({ status, ...meta }), { stallMs: 5_000, maxBytes: 0 }); + const text = await new Response(tapped).text(); + // The delimiter is restored: an SSE parser drops an event that EOF cuts off before its blank line. + expect(text).toBe(`${withoutDelimiter}\n\n`); + expect(calls).toEqual([{ status: 200, closeReason: "terminal" }]); + expect(logCtx.usage).toEqual(expect.objectContaining({ inputTokens: 12, outputTokens: 5 })); +}); diff --git a/tests/claude-integration/messages-native.test.ts b/tests/claude-integration/messages-native.test.ts index 37b19deddaf..8deaabc5e68 100644 --- a/tests/claude-integration/messages-native.test.ts +++ b/tests/claude-integration/messages-native.test.ts @@ -14,8 +14,10 @@ import { clearKeyCooldowns } from "../../src/providers/key-failover"; import { estimateClaudeRequestTokens, handleClaudeCountTokens, handleClaudeMessages } from "../../src/server/claude-messages"; import { getRequestLogEntries } from "../../src/server/request-log"; import type { OcxConfig } from "../../src/types"; +import type { AdmissionLease } from "../../src/lib/admission"; import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { startTruncatedSseUpstream } from "../helpers/truncated-sse-upstream"; interface Seen { path: string; @@ -158,6 +160,36 @@ async function send(config: OcxConfig, body: Record, headers?: return { requestId, response, text }; } +/** Streams the first frames of a turn, then stays open until the request is aborted. */ +function hangingAfterPartialTurn(): Response { + return new Response(new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(sseText(SSE_FRAMES.slice(0, 3)))); + }, + }), { headers: { "content-type": "text/event-stream" } }); +} + +async function drain(reader: ReadableStreamDefaultReader): Promise { + const decoder = new TextDecoder(); + let text = ""; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) return text; + text += decoder.decode(value, { stream: true }); + } + } catch { + return text; + } +} + +async function settledRowFor(requestId: string) { + for (let i = 0; i < 100 && !getRequestLogEntries().some(entry => entry.requestId === requestId); i++) { + await Bun.sleep(10); + } + return rowFor(requestId); +} + describe("managed native Messages", () => { test("sends exactly the allowlisted source body with the provider key and no caller credential", async () => { const config = fixtureConfig(startUpstream()); @@ -254,6 +286,102 @@ describe("managed native Messages", () => { expect(callerForwardSeen[0]!.headers.get("x-api-key")).toBe("sk-ant-fixture-caller"); }); + test("a mid-stream upstream reset ends the relayed stream with an Anthropic error event and a failed row", async () => { + const truncated = startTruncatedSseUpstream(sseText(SSE_FRAMES.slice(0, 3))); + try { + const config = fixtureConfig(truncated.port); + const { requestId, response, text } = await send(config, { ...SOURCE_BODY, stream: true }); + expect(response.status).toBe(200); + expect(text).toContain("streamed"); + expect(text).toContain("\n\nevent: error\ndata: "); + expect(text).toContain('"type":"api_error"'); + expect(text).toContain("anthropic passthrough upstream stream failed: "); + expect(truncated.requests()).toBe(1); + const row = rowFor(requestId); + expect(row.status).toBe(502); + expect(row.terminalStatus).toBe("failed"); + expect(row.closeReason).toBe("terminal"); + expect(row.transportPhase).toBe("mid_stream"); + expect(row.terminalSource).toBe("synthetic"); + expect(row.failureCause).toBe("transport-ambiguous"); + expect(row.attempts?.at(-1)?.streamAborted).toBe(true); + expect(row.upstreamError).toContain("anthropic passthrough upstream stream failed: "); + expect(JSON.stringify(row)).not.toContain("fixture-key-alpha"); + } finally { + truncated.stop(); + } + }); + + test("a non-streaming caller whose upstream stream resets still gets an Anthropic 502", async () => { + const truncated = startTruncatedSseUpstream(sseText(SSE_FRAMES.slice(0, 3))); + try { + const config = fixtureConfig(truncated.port); + const { requestId, response, text } = await send(config, { ...SOURCE_BODY, stream: false }); + expect(response.status).toBe(502); + expect(JSON.parse(text)).toMatchObject({ type: "error", error: { type: "api_error" } }); + expect(text).toContain("anthropic passthrough upstream stream failed: "); + const row = rowFor(requestId); + // Same row as the streaming lane for the same reset. + expect(row.status).toBe(502); + expect(row.terminalStatus).toBe("failed"); + expect(row.closeReason).toBe("terminal"); + expect(row.transportPhase).toBe("mid_stream"); + expect(row.failureCause).toBe("transport-ambiguous"); + } finally { + truncated.stop(); + } + }); + + test("a non-streaming fold that hits the body byte cap keeps the tap's close reason", async () => { + const config = fixtureConfig(startUpstream(() => new Response(SSE_TEXT, { headers: { "content-type": "text/event-stream" } })), + { claudeCode: { bodyMaxBytes: 64 } as OcxConfig["claudeCode"] }); + const { requestId, response, text } = await send(config, { ...SOURCE_BODY, stream: false }); + expect(response.status).toBe(502); + expect(text).toContain("exceeded 64 bytes"); + const row = rowFor(requestId); + expect(row.status).toBe(502); + expect(row.closeReason).toBe("body_overflow"); + }); + + test("a client that disconnects mid-stream is logged as a cancel, not an upstream failure", async () => { + const config = fixtureConfig(startUpstream(hangingAfterPartialTurn)); + const client = new AbortController(); + const requestId = `pf08-${crypto.randomUUID()}`; + const response = await handleClaudeMessages(new Request("http://localhost/v1/messages", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ ...SOURCE_BODY, stream: true }), + signal: client.signal, + }), config, { model: "", provider: "" }, { requestId, start: Date.now() }); + const reader = response.body!.getReader(); + expect((await reader.read()).done).toBe(false); + client.abort(new DOMException("client went away", "AbortError")); + await drain(reader); + const row = await settledRowFor(requestId); + expect(row.status).toBe(499); + expect(row.closeReason).toBe("client_cancel"); + expect(row.transportPhase).toBeUndefined(); + }); + + test("the lane aborting its own upstream (shutdown, turn release) is logged as a cancel", async () => { + const config = fixtureConfig(startUpstream(hangingAfterPartialTurn)); + let turn: AbortController | undefined; + const lease = { bindAbortController(controller: AbortController) { turn = controller; }, release() {} }; + const requestId = `pf08-${crypto.randomUUID()}`; + const response = await handleClaudeMessages(messagesRequest({ ...SOURCE_BODY, stream: true }), config, + { model: "", provider: "" }, { requestId, start: Date.now(), turnAdmissionLease: lease as unknown as AdmissionLease }); + const reader = response.body!.getReader(); + expect((await reader.read()).done).toBe(false); + expect(turn).toBeDefined(); + turn!.abort(new Error("server shutdown")); + const text = await drain(reader); + expect(text).not.toContain("event: error"); + const row = await settledRowFor(requestId); + expect(row.status).toBe(499); + expect(row.closeReason).toBe("client_cancel"); + expect(row.upstreamError).toBeUndefined(); + }); + test("count_tokens counts the body the native lane sends", async () => { const config = fixtureConfig(startUpstream()); await send(config, { ...SOURCE_BODY, stream: false }); diff --git a/tests/helpers/truncated-sse-upstream.ts b/tests/helpers/truncated-sse-upstream.ts new file mode 100644 index 00000000000..d4ca914035b --- /dev/null +++ b/tests/helpers/truncated-sse-upstream.ts @@ -0,0 +1,47 @@ +/** + * A fake upstream that answers any request with HTTP 200 `text/event-stream`, writes the given SSE + * text as one chunk, and then closes the socket without the terminating zero-length chunk. + * + * That is a mid-stream reset as the proxy's `fetch` sees it: the response headers arrived, part of + * the body arrived, and the next `reader.read()` rejects with ECONNRESET ("The socket connection + * was closed unexpectedly"). `Bun.serve` cannot produce it deterministically — an errored response + * body is sent as a connection reset by one Bun release and as a clean chunked EOF by another — + * so the fixture speaks HTTP/1.1 over a raw socket instead. + */ +export function startTruncatedSseUpstream(sse: string): { port: number; requests: () => number; stop: () => void } { + const body = new TextEncoder().encode(sse); + const head = "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\n\r\n"; + const answered = new WeakSet(); + const timers = new Set>(); + let requests = 0; + const listener = Bun.listen({ + hostname: "127.0.0.1", + port: 0, + socket: { + data(socket) { + if (answered.has(socket)) return; + answered.add(socket); + requests += 1; + socket.write(`${head}${body.byteLength.toString(16)}\r\n`); + socket.write(body); + socket.write("\r\n"); + // Give the partial body time to reach the reader before the reset: ending the socket + // right after the write lets Bun 1.3 fail the fetch before the headers are read. + const timer = setTimeout(() => { + timers.delete(timer); + socket.end(); + }, 20); + timers.add(timer); + }, + }, + }); + return { + port: listener.port, + requests: () => requests, + stop: () => { + for (const timer of timers) clearTimeout(timer); + timers.clear(); + listener.stop(true); + }, + }; +} From 6da09f661c999edcb4a155480e4e0dde209b5438 Mon Sep 17 00:00:00 2001 From: JUN Date: Wed, 30 Sep 2026 03:45:57 +0900 Subject: [PATCH 14/26] fix(chat): keep Qwen3.8 reminders valid after first turn (#6249) Use the pinned Qwen3.8-27B chat template contract to preserve chronological developer reminders as user turns on the translated Chat wire. Keep other models and native OpenAI behavior unchanged. --- .../src/content/docs/reference/adapters.md | 6 ++ scripts/test-layout/layout.json | 1 + src/adapters/openai-chat/messages.ts | 5 +- structure/providers/chat-compat.md | 25 ++++--- .../openai-chat-qwen38-leading-system.test.ts | 74 +++++++++++++++++++ .../qwen38-27b-chat-template-contract.json | 7 ++ tests/fixtures/test-layout-expected.json | 1 + 7 files changed, 106 insertions(+), 13 deletions(-) create mode 100644 tests/adapters/openai/openai-chat-qwen38-leading-system.test.ts create mode 100644 tests/fixtures/qwen38-27b-chat-template-contract.json diff --git a/docs-site/src/content/docs/reference/adapters.md b/docs-site/src/content/docs/reference/adapters.md index b2b8488b2df..390a66e0b85 100644 --- a/docs-site/src/content/docs/reference/adapters.md +++ b/docs-site/src/content/docs/reference/adapters.md @@ -55,6 +55,12 @@ transport; it does not infer subscription attribution from the inbound protocol. tool message as the anchor. - **Rewrites Codex's GPT-5 identity prompt** to a model-agnostic intro so routed models don't claim to be OpenAI. +- For translated `Qwen3.8-27B` requests, a text-only developer reminder after the leading system + message stays in its conversation slot but is sent as `user`. The model's + [chat template](https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/chat_template.jinja) + rejects later `system` messages and does not accept `developer`, while later `user` messages + are valid. This preserves order but cannot preserve developer-role precedence. Other models + keep their configured developer-role behavior; native Chat passthrough is unchanged. - **Clamps `reasoning_effort`** to the model's advertised subset when an exact tier is unavailable; `xhigh` and `max` remain distinct labels unless a provider explicitly configures an alias. The adapter **omits it entirely** for ids in `provider.noReasoningModels`. diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index fd19e2040b0..3e8814ba98f 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1197,6 +1197,7 @@ "openai-chat-native-policy.test.ts": "adapters/openai", "openai-chat-parallel-stream.test.ts": "adapters/openai", "openai-chat-path-override.test.ts": "adapters/openai", + "openai-chat-qwen38-leading-system.test.ts": "adapters/openai", "openai-chat-reasoning-wire-policy.test.ts": "adapters/openai", "openai-chat-sanitization-review-regressions.test.ts": "adapters/openai", "openai-chat-serialized-tool-call-content.test.ts": "adapters/openai", diff --git a/src/adapters/openai-chat/messages.ts b/src/adapters/openai-chat/messages.ts index bce315a2a19..13ea4f64bc3 100644 --- a/src/adapters/openai-chat/messages.ts +++ b/src/adapters/openai-chat/messages.ts @@ -142,6 +142,9 @@ export function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProv }; const nativeOpenAI = isNativeOpenAIChatTarget(provider); + // Qwen3.8-27B's pinned chat template rejects non-leading system and every developer role; + // a later user is rendered in place. Keep this exception specific to that template family. + const qwen38LeadingSystemTemplate = !nativeOpenAI && /^(?:Qwen\/)?Qwen3\.8-27B$/i.test(parsed.modelId); // Which role a developer message carries, and why the unrecorded state folds, is stated once // in ./developer-role.ts and read from there by the native passthrough as well. Either way the // message keeps the slot it arrived in — only the role changes, never the position. @@ -191,7 +194,7 @@ export function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProv // message instead is rejected by some upstreams. Native OpenAI keeps its existing // empty-developer wire, which is a separate question from placement. if (!nativeOpenAI && text.length === 0) break; - chatMsg = { role: developerWireRole, content: text }; + chatMsg = { role: qwen38LeadingSystemTemplate ? "user" : developerWireRole, content: text }; } else if (typeof msg.content === "string") { chatMsg = { role: "user", content: msg.content }; } else if (!hasStructured) { diff --git a/structure/providers/chat-compat.md b/structure/providers/chat-compat.md index 9788c30b8e7..a469a3c8ef7 100644 --- a/structure/providers/chat-compat.md +++ b/structure/providers/chat-compat.md @@ -11,18 +11,13 @@ other providers whose models happen to share a name fragment. ## Chronological in-conversation instructions -`src/adapters/openai-chat/messages.ts` keeps a text-only timeline developer message in the -slot it arrived in, on every Chat destination and model. Appending a reminder therefore does -not hoist new text into the leading system prompt and rewrite the existing serialized message -prefix, and a mid-conversation instruction no longer moves ahead of the turns it was written -to follow. Pending tool results still precede deferred reminders. This was previously scoped -to the registry-recognized OpenCode Go destination and the exact model -`deepseek-v4.1-flash`, which made prompt-prefix stability read as a property of that one -destination. The base system prompt, vision conversion and native OpenAI developer roles -retain their existing behavior. This is independent of the Claude trailing-notice -stabilization option and does not guarantee upstream cache hits. Regression coverage is in -`tests/adapters/openai/openai-chat-system-order.test.ts` and -`tests/adapters/openai/openai-chat-developer-position.test.ts`. +`src/adapters/openai-chat/messages.ts` keeps text-only timeline developer messages in their +original slots on every Chat destination and model. A later reminder does not rewrite the +leading system prompt or move ahead of earlier turns; pending tool results still precede +deferred reminders. This behavior previously covered only OpenCode Go and +`deepseek-v4.1-flash`. The base system prompt, vision conversion, native OpenAI developer +roles, and Claude trailing-notice option are unchanged; cache hits are not guaranteed. +Tests: `tests/adapters/openai/openai-chat-system-order.test.ts` and `tests/adapters/openai/openai-chat-developer-position.test.ts`. The role that slot carries is a separate decision, and the setting that makes it is tri-state. `foldDeveloperRoleToSystem` unset sends `system`, `true` sends `system`, and `false` sends @@ -35,6 +30,12 @@ repository where no test can reach it. The role was previously decided by testin against `api.openai.com`, so every OpenAI-compatible gateway was assumed not to support a standard role until proven otherwise, and the instruction silently lost `developer` precedence. +The translated `Qwen3.8-27B` Chat route follows its [pinned template](https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/chat_template.jinja): a late `system` raises, `developer` is unsupported, and a late `user` renders in place. +Text-only developer items therefore keep their content and slot but use the `user` wire role, +losing developer precedence. Other models retain the mapping above; native Chat passthrough +is unchanged. The rule is recorded in `tests/fixtures/qwen38-27b-chat-template-contract.json` +and exercised by `tests/adapters/openai/openai-chat-qwen38-leading-system.test.ts`. + That mapping is not prose to be restated. `tests/ci-workflows/docs-developer-role-policy.test.ts` builds the sentence above from the role `src/adapters/openai-chat/messages.ts` serializes for each of the three states, and requires this document and diff --git a/tests/adapters/openai/openai-chat-qwen38-leading-system.test.ts b/tests/adapters/openai/openai-chat-qwen38-leading-system.test.ts new file mode 100644 index 00000000000..228bc50cca0 --- /dev/null +++ b/tests/adapters/openai/openai-chat-qwen38-leading-system.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { createOpenAIChatAdapter } from "../../../src/adapters/openai-chat"; +import type { OcxParsedRequest, OcxProviderConfig } from "../../../src/types"; +import { fixturePath } from "../../helpers/repo-root"; + +const contract = JSON.parse(readFileSync(fixturePath("qwen38-27b-chat-template-contract.json"), "utf8")) as { + acceptedRoles: string[]; + systemMustBeFirst: boolean; + lateSystemError: string; + unsupportedRoleError: string; +}; + +const provider: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://qwen.example.invalid/v1", + apiKey: "test", +}; + +function serialize(modelId: string, target = provider): Array<{ role: string; content: string }> { + const request = createOpenAIChatAdapter(target).buildRequest({ + modelId, + context: { + systemPrompt: ["Base instructions."], + messages: [ + { role: "user", content: "First turn.", timestamp: 0 }, + { role: "developer", content: "Answer in one sentence.", timestamp: 0 }, + { role: "user", content: "Second turn.", timestamp: 0 }, + ], + }, + stream: false, + options: {}, + } as OcxParsedRequest); + return (JSON.parse(request.body) as { messages: Array<{ role: string; content: string }> }).messages; +} + +// Qwen's pinned Jinja template raises on a non-leading system or an unrecognized developer role. +// A later user is rendered as its own turn. Keep this oracle separate from the adapter selector. +function assertTemplateAccepts(messages: Array<{ role: string }>): void { + messages.forEach((message, index) => { + if (contract.systemMustBeFirst && message.role === "system" && index > 0) { + throw new Error(contract.lateSystemError); + } + if (!contract.acceptedRoles.includes(message.role)) throw new Error(contract.unsupportedRoleError); + }); +} + +describe("Qwen3.8-27B leading-system template", () => { + test.each(["Qwen3.8-27B", "Qwen/Qwen3.8-27B"])("keeps a late reminder after the first user without an invalid system role: %s", modelId => { + const messages = serialize(modelId); + expect(messages).toEqual([ + { role: "system", content: "Base instructions." }, + { role: "user", content: "First turn." }, + { role: "user", content: "Answer in one sentence." }, + { role: "user", content: "Second turn." }, + ]); + expect(() => assertTemplateAccepts(messages)).not.toThrow(); + }); + + test("does not send a developer role even if a gateway declaration says it accepts one", () => { + const messages = serialize("Qwen3.8-27B", { ...provider, foldDeveloperRoleToSystem: false }); + expect(messages[2]).toEqual({ role: "user", content: "Answer in one sentence." }); + expect(() => assertTemplateAccepts(messages)).not.toThrow(); + }); + + test("other models keep their recorded role and chronological position", () => { + expect(serialize("other-model")[2]).toEqual({ role: "system", content: "Answer in one sentence." }); + expect(serialize("other-model", { ...provider, foldDeveloperRoleToSystem: false })[2]) + .toEqual({ role: "developer", content: "Answer in one sentence." }); + expect(serialize("Qwen3.8-27B-FP8")[2]).toEqual({ role: "system", content: "Answer in one sentence." }); + expect(serialize("Qwen3.8-27B", { ...provider, baseUrl: "https://api.openai.com/v1" })[2]) + .toEqual({ role: "system", content: "Answer in one sentence." }); + }); +}); diff --git a/tests/fixtures/qwen38-27b-chat-template-contract.json b/tests/fixtures/qwen38-27b-chat-template-contract.json new file mode 100644 index 00000000000..6ba94735a04 --- /dev/null +++ b/tests/fixtures/qwen38-27b-chat-template-contract.json @@ -0,0 +1,7 @@ +{ + "source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/chat_template.jinja", + "acceptedRoles": ["system", "user", "assistant", "tool"], + "systemMustBeFirst": true, + "lateSystemError": "System message must be at the beginning.", + "unsupportedRoleError": "Unexpected message role." +} diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 459506dab5b..b35190edf94 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1209,6 +1209,7 @@ "openai-chat-native-policy.test.ts": "adapters/openai", "openai-chat-parallel-stream.test.ts": "adapters/openai", "openai-chat-path-override.test.ts": "adapters/openai", + "openai-chat-qwen38-leading-system.test.ts": "adapters/openai", "openai-chat-reasoning-wire-policy.test.ts": "adapters/openai", "openai-chat-sanitization-review-regressions.test.ts": "adapters/openai", "openai-chat-serialized-tool-call-content.test.ts": "adapters/openai", From 8aa3c862ae89a4b19f912714fbe013afc1665127 Mon Sep 17 00:00:00 2001 From: JUN Date: Wed, 30 Sep 2026 03:50:27 +0900 Subject: [PATCH 15/26] docs(structure): split Anthropic account pool contract --- structure/INDEX.md | 5 ++- .../ADR-6013-anthropic-account-pause.md | 2 +- structure/gui-and-management-api.md | 2 +- structure/manifest.json | 10 +++++ structure/providers-and-adapters.md | 36 +---------------- structure/providers/anthropic-account-pool.md | 40 +++++++++++++++++++ 6 files changed, 57 insertions(+), 38 deletions(-) create mode 100644 structure/providers/anthropic-account-pool.md diff --git a/structure/INDEX.md b/structure/INDEX.md index a0dbbf3ea74..2f5e29efc0a 100644 --- a/structure/INDEX.md +++ b/structure/INDEX.md @@ -57,6 +57,7 @@ Per-vendor contracts and the adapter authority that constructs them. | Doc | Scope | | --- | --- | | [`providers-and-adapters.md`](providers-and-adapters.md) | Provider and adapter selection, the adapter inventory, live model discovery, and the hosted-search continuation bridge. | +| [`providers/anthropic-account-pool.md`](providers/anthropic-account-pool.md) | Anthropic OAuth account pause, model routes, and quota labels. | | [`providers/openai-tiers.md`](providers/openai-tiers.md) | Pool/Direct account modes, API-key separation, and the public provider and quota contract. | | [`providers/openai-accounts.md`](providers/openai-accounts.md) | Migration and restore, wire identity, store concurrency, pool ordering and exclusions, quota observations, and account-bound retention. | | [`providers/cursor.md`](providers/cursor.md) | Cursor native exec, parameterized models, checkpoints, and active-context usage. | @@ -134,10 +135,10 @@ A source area can be described by more than one doc, because these docs are orga | `src/lab/` | [`runtime.md`](runtime.md)
[`adapters/compatibility-lab.md`](adapters/compatibility-lab.md) | | `src/lib/` | [`overview.md`](overview.md)
[`runtime.md`](runtime.md)
[`transports/byte-accounting.md`](transports/byte-accounting.md)
[`transports/responses-wire-shapes.md`](transports/responses-wire-shapes.md)
[`transports/responses-failover.md`](transports/responses-failover.md)
[`transports/responses-spend.md`](transports/responses-spend.md)
[`transports/inventory.md`](transports/inventory.md)
[`gui-and-management-api.md`](gui-and-management-api.md)
[`dashboard-and-usage.md`](dashboard-and-usage.md)
[`clients/integrations.md`](clients/integrations.md)
[`ops/service-and-sidecars.md`](ops/service-and-sidecars.md)
[`ops/docs-and-release.md`](ops/docs-and-release.md) | | `src/link/` | [`remote-link.md`](remote-link.md) | -| `src/oauth/` | [`runtime.md`](runtime.md)
[`transports/inventory.md`](transports/inventory.md)
[`providers-and-adapters.md`](providers-and-adapters.md)
[`providers/xai-grok.md`](providers/xai-grok.md) | +| `src/oauth/` | [`runtime.md`](runtime.md)
[`transports/inventory.md`](transports/inventory.md)
[`providers-and-adapters.md`](providers-and-adapters.md)
[`providers/anthropic-account-pool.md`](providers/anthropic-account-pool.md)
[`providers/xai-grok.md`](providers/xai-grok.md) | | `src/plugins/` | [`ops/plugins.md`](ops/plugins.md) | | `src/protocols/` | [`data-planes/protocol-paths.md`](data-planes/protocol-paths.md) | -| `src/providers/` | [`runtime.md`](runtime.md)
[`subagents.md`](subagents.md)
[`transports/inventory.md`](transports/inventory.md)
[`providers-and-adapters.md`](providers-and-adapters.md)
[`providers/xai-grok.md`](providers/xai-grok.md) | +| `src/providers/` | [`runtime.md`](runtime.md)
[`subagents.md`](subagents.md)
[`transports/inventory.md`](transports/inventory.md)
[`providers-and-adapters.md`](providers-and-adapters.md)
[`providers/anthropic-account-pool.md`](providers/anthropic-account-pool.md)
[`providers/xai-grok.md`](providers/xai-grok.md) | | `src/quota/` | [`dashboard-and-usage.md`](dashboard-and-usage.md) | | `src/reasoning-effort.ts` | [`runtime.md`](runtime.md) | | `src/remote-control/` | [`remote-workspace.md`](remote-workspace.md) | diff --git a/structure/decisions/ADR-6013-anthropic-account-pause.md b/structure/decisions/ADR-6013-anthropic-account-pause.md index 5d5fb1f005c..ce0d904d457 100644 --- a/structure/decisions/ADR-6013-anthropic-account-pause.md +++ b/structure/decisions/ADR-6013-anthropic-account-pause.md @@ -1,6 +1,6 @@ # ADR-6013 — decision recorded under "Anthropic account pause" -- Contract owner: [Providers and adapters](../providers-and-adapters.md#anthropic-account-pause) +- Contract owner: [Anthropic account pool](../providers/anthropic-account-pool.md#anthropic-account-pause) ## Decision Log diff --git a/structure/gui-and-management-api.md b/structure/gui-and-management-api.md index b96f866d664..384fa6fa2cb 100644 --- a/structure/gui-and-management-api.md +++ b/structure/gui-and-management-api.md @@ -6,7 +6,7 @@ Anthropic account rows now expose the shared boolean `paused` DTO and use the ex disabled manual selection, and separate mutation-versus-refresh failure notices. CLI `ocx account pause|resume anthropic [--json]` uses that same endpoint. Only OAuth routes support this operation; API-key routes remain outside this switch. -Selection and persistence semantics: [Anthropic account pause](providers-and-adapters.md#anthropic-account-pause). +Selection and persistence semantics: [Anthropic account pause](providers/anthropic-account-pool.md#anthropic-account-pause). `src/server/management/oauth-account-routes.ts` exposes Anthropic `routes` through both unified `/api/pool/settings` and legacy `/api/oauth/accounts/pool`. Omitted rules survive other setting writes, `null` clears them, and other pool kinds reject supplied rules. The unified DTO declares `routes` supported only for Anthropic and reports null otherwise. Both Anthropic settings GETs validate saved rules before projection: malformed hand edits yield `routes: null` plus `routesError` without changing the stored value; valid and absent rules omit that diagnostic. Config and management responses retain route names; request logs use only the rule’s 1-based `route:#` position. `src/cli/account-extended.ts` reads, replaces and clears these rules with `ocx account routes anthropic`; the server validates content. diff --git a/structure/manifest.json b/structure/manifest.json index 0057a6464e4..d200b1c1e29 100644 --- a/structure/manifest.json +++ b/structure/manifest.json @@ -284,6 +284,16 @@ "src/web-search/" ] }, + { + "path": "providers/anthropic-account-pool.md", + "tier": 4, + "title": "Anthropic Account Pool", + "scope": "Anthropic OAuth account pause, model routes, and quota labels.", + "documents": [ + "src/oauth/", + "src/providers/" + ] + }, { "path": "providers/openai-tiers.md", "tier": 4, diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index 1983f764c1b..a937ed71c01 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -1,36 +1,7 @@ # Providers And Adapters -## Anthropic account pause - -Anthropic OAuth shares `ProviderAccount.paused` in the protected auth store with generic -OAuth, not a second list in provider config. `setAccountPaused` serializes pause/resume -with credential and selection writes, advances the selection revision, and only publishes -invalidation after persistence. Removing an account removes its pause; reauthentication -preserves it. The store moves active selection to an unpaused, non-reauth row if available. -Pause does not clear cooldowns, quota, or credentials and does not cancel an already-sent turn. - -`src/oauth/anthropic-routing.ts` excludes paused rows from quota, round-robin, fill-first, -manual, affinity, model-route and reactive 429 candidates, including when proactive pooling -is off. All-paused requests return 403 with resume guidance. Quorum is invalidated on pause -and resume; a sent account paused before its 429 may still recover on its sole unpaused -successor. Credential resolution, refresh-lock acquisition, selection commit and physical -dispatch recheck live eligibility after asynchronous waits. Responses and native Messages -preserve typed 401 authentication, 403 pause and 429 cooldown refusals after pacing, including pool-off recovery; -they do not report local rejection as 502. Only cooled usable survivors of a strict route -produce its scoped 429 and Retry-After, not a login error. An already-dispatched refresh -retains a successful rotated credential without unpausing; a late failure cannot mark the -paused row for reauthentication. Token Guardian and Anthropic quota probes recheck live -pause, selection and bearer ownership after token resolution and before each usage send; -a newly paused account makes no auxiliary request. Pool-off keeps a healthy active account; -pause/prior-429 recovery uses `only-eligible`, and logs name the committed account. - -> Decision record: [ADR-6013](decisions/ADR-6013-anthropic-account-pause.md) - -Regression coverage: `tests/adapters/anthropic/anthropic-account-pause.test.ts`, -`tests/adapters/anthropic/anthropic-model-routes.test.ts`, `tests/oauth/oauth-accounts-api.test.ts`, -`tests/cli/cli-account-pool-verbs.test.ts`, and `gui/tests/provider-quota-refresh-controls.test.tsx`. - -For Anthropic OAuth, `src/oauth/anthropic-routing.ts` applies the first matching `anthropicAccountPool.routes` rule to every eligible pick. The declared account order is stable while its candidates remain eligible; active, manual, affinity, quota and strategy preferences only choose inside that set. A healthy session affinity outside a model route is ignored for that request and retained for later unrouted or differently routed models; the routed commit does not overwrite it. An explicit fallback widens an empty route to the ordinary pool, and fill-first then advances in ordinary pool order from the active account. A missing eligible route fails locally without that fallback. The rules are operator allowlists, not provider entitlement evidence. Request logs use `route:#` for the 1-based rule position, not the operator name. +Anthropic account pause, model routes, and quota labels follow the +[Anthropic account-pool contract](providers/anthropic-account-pool.md). GitHub Copilot `modelContextTiers` is selected per upstream model. The Chat and Responses adapters set `contextTier` only when the canonical routed provider is `github-copilot` @@ -102,9 +73,6 @@ OrcaRouter key exchange uses the shared raw-byte reader before returning a durab 64 KiB response ceiling, single 30-second header/body deadline, and cancellation behavior follow the [bounded ingestion contract](transports/inventory.md#bounded-response-ingestion-and-orcarouter-login). -Anthropic model-scoped quota labels in `src/providers/quota/vendor-probes-oauth.ts` publish -only canonical Fable, Opus, or Sonnet labels after removing terminal controls; unknown upstream display names are omitted. - MiniMax and MiniMax CN Coding Plan quota in `src/providers/quota/vendor-probes-key.ts` uses the region-matched `/v1/api/openplatform/coding_plan/remains` endpoint. It publishes the `general` model's consumed 5-hour percentage and, when active, weekly percentage with their reset times; diff --git a/structure/providers/anthropic-account-pool.md b/structure/providers/anthropic-account-pool.md new file mode 100644 index 00000000000..6d6ebae120c --- /dev/null +++ b/structure/providers/anthropic-account-pool.md @@ -0,0 +1,40 @@ +# Anthropic Account Pool + +## Anthropic account pause + +Anthropic OAuth shares `ProviderAccount.paused` in the protected auth store with generic +OAuth, not a second list in provider config. `setAccountPaused` serializes pause/resume +with credential and selection writes, advances the selection revision, and only publishes +invalidation after persistence. Removing an account removes its pause; reauthentication +preserves it. The store moves active selection to an unpaused, non-reauth row if available. +Pause does not clear cooldowns, quota, or credentials and does not cancel an already-sent turn. + +`src/oauth/anthropic-routing.ts` excludes paused rows from quota, round-robin, fill-first, +manual, affinity, model-route and reactive 429 candidates, including when proactive pooling +is off. All-paused requests return 403 with resume guidance. Quorum is invalidated on pause +and resume; a sent account paused before its 429 may still recover on its sole unpaused +successor. Credential resolution, refresh-lock acquisition, selection commit and physical +dispatch recheck live eligibility after asynchronous waits. Responses and native Messages +preserve typed 401 authentication, 403 pause and 429 cooldown refusals after pacing, including pool-off recovery; +they do not report local rejection as 502. Only cooled usable survivors of a strict route +produce its scoped 429 and Retry-After, not a login error. An already-dispatched refresh +retains a successful rotated credential without unpausing; a late failure cannot mark the +paused row for reauthentication. Token Guardian and Anthropic quota probes recheck live +pause, selection and bearer ownership after token resolution and before each usage send; +a newly paused account makes no auxiliary request. Pool-off keeps a healthy active account; +pause/prior-429 recovery uses `only-eligible`, and logs name the committed account. + +> Decision record: [ADR-6013](../decisions/ADR-6013-anthropic-account-pause.md) + +Regression coverage: `tests/adapters/anthropic/anthropic-account-pause.test.ts`, +`tests/adapters/anthropic/anthropic-model-routes.test.ts`, `tests/oauth/oauth-accounts-api.test.ts`, +`tests/cli/cli-account-pool-verbs.test.ts`, and `gui/tests/provider-quota-refresh-controls.test.tsx`. + +## Model routes + +For Anthropic OAuth, `src/oauth/anthropic-routing.ts` applies the first matching `anthropicAccountPool.routes` rule to every eligible pick. The declared account order is stable while its candidates remain eligible; active, manual, affinity, quota and strategy preferences only choose inside that set. A healthy session affinity outside a model route is ignored for that request and retained for later unrouted or differently routed models; the routed commit does not overwrite it. An explicit fallback widens an empty route to the ordinary pool, and fill-first then advances in ordinary pool order from the active account. A missing eligible route fails locally without that fallback. The rules are operator allowlists, not provider entitlement evidence. Request logs use `route:#` for the 1-based rule position, not the operator name. + +## Quota labels + +Anthropic model-scoped quota labels in `src/providers/quota/vendor-probes-oauth.ts` publish +only canonical Fable, Opus, or Sonnet labels after removing terminal controls; unknown upstream display names are omitted. From 3092b555788eec19bedb14734b9a5ba54fc9977f Mon Sep 17 00:00:00 2001 From: ingwannu Date: Wed, 30 Sep 2026 04:00:58 +0900 Subject: [PATCH 16/26] fix(packaging): ship keyring native addon with desktop sidecar (#6161) * fix(packaging): ship keyring native addon * fix(packaging): make keyring bundle proof load-only * fix(packaging): resolve keyring addon from wrapper scope * fix(keyring): prefer the standalone-owned addon * fix(packaging): verify compiled keyring roots * fix(ci): cover packaged keyring prerequisites * fix(ci): keep keyring probe under file-size ratchet * style(cli): keep private probes inside size gate * fix(packaging): resolve symlinked keyring wrapper Resolve the wrapper entrypoint before locating its optional native addon, so virtual-store package layouts stage the installed addon. Cover nested and symlinked layouts. --------- Co-authored-by: Ingwannu Co-authored-by: JUN --- .github/workflows/ci.yml | 14 +- .github/workflows/release.yml | 33 +++- desktop/README.md | 5 +- desktop/scripts/prepare-sidecar.ts | 23 +-- desktop/scripts/verify-linux-sidecar.sh | 9 + desktop/scripts/verify-macos-runtime.sh | 36 +++- desktop/src-tauri/tauri.conf.json | 3 +- .../docs/getting-started/installation.md | 3 +- scripts/build-standalone.ts | 5 + scripts/keyring-smoke.ts | 5 +- scripts/standalone-keyring.ts | 30 +++ src/cli/index.ts | 2 +- src/codex/native-profile-store.ts | 3 +- src/lib/keyring-native.ts | 129 +++++++++++++ src/providers/api-key-resolve.ts | 6 +- ...DR-6139-packaged-native-keyring-binding.md | 12 ++ structure/desktop-shell.md | 27 +++ structure/ops/docs-and-release.md | 6 +- tests/ci-workflows/ci-scope-reduction.test.ts | 4 + tests/ci-workflows/keyring-smoke.test.ts | 179 +++++++++++++++++- .../linux-desktop-packaged-ci.test.ts | 5 + .../release-desktop-scripts.test.ts | 2 + tests/gui/gui-desktop-sidecar-script.test.ts | 2 + tests/gui/standalone-build-script.test.ts | 1 + 24 files changed, 512 insertions(+), 32 deletions(-) create mode 100644 scripts/standalone-keyring.ts create mode 100644 src/lib/keyring-native.ts create mode 100644 structure/decisions/ADR-6139-packaged-native-keyring-binding.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af09564cd08..9c9c6af8afd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -261,6 +261,10 @@ jobs: - 'src/service/**' - 'src/cli/index.ts' - 'src/lib/bun-runtime.ts' + - 'src/lib/standalone.ts' + - 'src/lib/keyring-native.ts' + - 'scripts/build-standalone.ts' + - 'scripts/standalone-keyring.ts' - 'package.json' - 'bun.lock' - '.github/workflows/ci.yml' @@ -276,6 +280,7 @@ jobs: - 'gui/**' - 'src/**' - 'scripts/build-standalone.ts' + - 'scripts/standalone-keyring.ts' - 'scripts/standalone-targets.ts' - 'package.json' - 'bun.lock' @@ -1400,10 +1405,11 @@ jobs: run: | set -euo pipefail triple="$(rustc -vV | sed -n 's/^host: //p')" - mkdir -p desktop/src-tauri/binaries desktop/src-tauri/resources/gui/dist + mkdir -p desktop/src-tauri/binaries desktop/src-tauri/resources/gui/dist desktop/src-tauri/resources/keyring : > "desktop/src-tauri/binaries/ocx-${triple}" chmod +x "desktop/src-tauri/binaries/ocx-${triple}" : > desktop/src-tauri/resources/gui/dist/.keep + : > desktop/src-tauri/resources/keyring/.keep - name: Check Rust formatting run: cargo fmt --manifest-path desktop/src-tauri/Cargo.toml --check @@ -1461,6 +1467,12 @@ jobs: cp -a "$DEB_BUNDLE/." "$BUNDLE_ROOT/deb/" chmod -R a-w "$BUNDLE_ROOT" + - name: Verify packaged Linux sidecar keyring + if: needs.changes.outputs.desktop == 'true' + env: + BUNDLE_ROOT: ${{ runner.temp }}/opencodex-linux-bundles + run: bash desktop/scripts/verify-linux-sidecar.sh "$BUNDLE_ROOT/appimage" + - name: Run Linux packaged-shell E2E if: needs.changes.outputs.desktop == 'true' env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8d76276829d..08a5e66cd05 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -126,18 +126,28 @@ jobs: include: - os: ubuntu-latest target: bun-linux-x64 + dependency_os: linux + dependency_cpu: x64 smoke: true - os: macos-latest target: bun-darwin-arm64 + dependency_os: darwin + dependency_cpu: arm64 smoke: true - os: macos-latest target: bun-darwin-x64 + dependency_os: darwin + dependency_cpu: x64 smoke: false - os: windows-latest target: bun-windows-x64 + dependency_os: win32 + dependency_cpu: x64 smoke: true - os: ubuntu-latest target: bun-linux-arm64 + dependency_os: linux + dependency_cpu: arm64 smoke: false runs-on: ${{ matrix.os }} timeout-minutes: 25 @@ -153,7 +163,7 @@ jobs: uses: ./.github/actions/setup-project-bun - name: Install dependencies - run: bun install --frozen-lockfile + run: bun install --frozen-lockfile --os=${{ matrix.dependency_os }} --cpu=${{ matrix.dependency_cpu }} - name: Build dashboard run: bun run build:gui @@ -200,9 +210,9 @@ jobs: set -euo pipefail cd "dist/standalone/$STANDALONE_TARGET" if [[ "$RUNNER_OS" == "Windows" ]]; then - powershell -NoProfile -Command 'Compress-Archive -Path ocx.exe,gui -DestinationPath ("../../ocx-{0}-{1}.zip" -f $env:RELEASE_VERSION,$env:STANDALONE_TARGET) -Force' + powershell -NoProfile -Command 'Compress-Archive -Path ocx.exe,gui,keyring -DestinationPath ("../../ocx-{0}-{1}.zip" -f $env:RELEASE_VERSION,$env:STANDALONE_TARGET) -Force' else - tar -czf "../../ocx-${RELEASE_VERSION}-${STANDALONE_TARGET}.tar.gz" ocx gui + tar -czf "../../ocx-${RELEASE_VERSION}-${STANDALONE_TARGET}.tar.gz" ocx gui keyring fi cd ../.. # The pre-publication verifier resolves every recorded checksum from @@ -232,16 +242,22 @@ jobs: include: - os: macos-latest target: universal-apple-darwin + dependency_os: darwin + dependency_cpu: "*" bundles: app,dmg sidecar-targets: macos artifact-suffixes: macos.dmg,macos.app.tar.gz - os: windows-latest target: x86_64-pc-windows-msvc + dependency_os: win32 + dependency_cpu: x64 bundles: msi sidecar-targets: x86_64-pc-windows-msvc artifact-suffixes: windows-x64.msi - os: ubuntu-22.04 target: x86_64-unknown-linux-gnu + dependency_os: linux + dependency_cpu: x64 bundles: appimage,deb sidecar-targets: x86_64-unknown-linux-gnu artifact-suffixes: linux-x86_64.AppImage,linux-amd64.deb @@ -275,7 +291,7 @@ jobs: run: bun scripts/release-version-sources.ts check "$RELEASE_VERSION" - name: Install project dependencies - run: bun install --frozen-lockfile + run: bun install --frozen-lockfile --os=${{ matrix.dependency_os }} --cpu=${{ matrix.dependency_cpu }} - name: Build dashboard run: bun run build:gui @@ -430,6 +446,15 @@ jobs: # diagnostics on the first attempt; Apple signing commands stay non-verbose. run: bunx tauri ${{ runner.os == 'Linux' && '--verbose' || '' }} build --ci --target ${{ matrix.target }} --bundles ${{ matrix.bundles }} --config "${{ runner.os == 'Windows' && format('{0}/opencodex-msi.json', runner.temp) || '{}' }}" + - name: Verify the packaged universal macOS runtime + if: runner.os == 'macOS' + run: | + set -euo pipefail + app=desktop/src-tauri/target/universal-apple-darwin/release/bundle/macos/OpenCodex.app + test -f "$app/Contents/Resources/keyring/keyring.darwin-arm64.node" + test -f "$app/Contents/Resources/keyring/keyring.darwin-x64.node" + bash desktop/scripts/verify-macos-runtime.sh "$app" + # Tauri patches a bundle-type marker into the application binary for each Linux format. # Keep each format in its own Cargo target so the deb cannot inherit the AppImage marker # and linuxdeploy cannot mutate the binary later consumed by the deb build. diff --git a/desktop/README.md b/desktop/README.md index 786bfbfc9e4..1ef61fa78b2 100644 --- a/desktop/README.md +++ b/desktop/README.md @@ -10,7 +10,10 @@ bunx tauri dev ``` The sidecar is generated from the repository's standalone binary build and is -not checked into git. +not checked into git. That build also stages the target-matching native keyring addon +under `keyring/`; Tauri copies it as a resource because Bun cannot load a `.node` addon +from the compiled executable's virtual filesystem. Universal macOS preparation requires +both Darwin optional packages (`bun install --frozen-lockfile --os=darwin --cpu=*`). The macOS tray panel is a SwiftUI/AppKit static library built from `app/Sources/NativeTray` by the Rust build script and linked into this process. diff --git a/desktop/scripts/prepare-sidecar.ts b/desktop/scripts/prepare-sidecar.ts index 2403e130933..20f7eec61f7 100644 --- a/desktop/scripts/prepare-sidecar.ts +++ b/desktop/scripts/prepare-sidecar.ts @@ -1,4 +1,4 @@ -import { copyFileSync, cpSync, existsSync, mkdirSync } from "node:fs"; +import { copyFileSync, cpSync, mkdirSync } from "node:fs"; import { join, resolve } from "node:path"; import { adHocSignSidecar, shouldAdHocSignSidecar } from "./sidecar-signing"; @@ -38,20 +38,20 @@ if (!triple || !targetByTriple[triple]) { const target = targetByTriple[triple]; const source = join(repoRoot, "dist", "standalone", target); const executable = join(source, target.startsWith("bun-windows-") ? "ocx.exe" : "ocx"); -if (!existsSync(executable)) { - const result = Bun.spawnSync([ - process.execPath, - "run", - "build:standalone", - "--target", - target, - ], { cwd: repoRoot, stdout: "inherit", stderr: "inherit" }); - if (result.exitCode !== 0) process.exit(result.exitCode); -} +// Preparation must consume this checkout, never a stale executable/addon pair left in dist. +const result = Bun.spawnSync([ + process.execPath, + "run", + "build:standalone", + "--target", + target, +], { cwd: repoRoot, stdout: "inherit", stderr: "inherit" }); +if (result.exitCode !== 0) process.exit(result.exitCode); const desktopRoot = resolve(import.meta.dir, ".."); const binaries = join(desktopRoot, "src-tauri", "binaries"); const resources = join(desktopRoot, "src-tauri", "resources", "gui", "dist"); +const keyringResources = join(desktopRoot, "src-tauri", "resources", "keyring"); mkdirSync(binaries, { recursive: true }); mkdirSync(resources, { recursive: true }); const destination = join(binaries, `ocx-${triple}${target.startsWith("bun-windows-") ? ".exe" : ""}`); @@ -60,5 +60,6 @@ if (shouldAdHocSignSidecar(process.platform, target)) { const signed = adHocSignSidecar(destination); if (signed !== 0) process.exit(signed); } +cpSync(join(source, "keyring"), keyringResources, { recursive: true }); cpSync(join(repoRoot, "gui", "dist"), resources, { recursive: true }); console.log(`Prepared ${destination}`); diff --git a/desktop/scripts/verify-linux-sidecar.sh b/desktop/scripts/verify-linux-sidecar.sh index 7695767ebe6..c502bb5c8ba 100644 --- a/desktop/scripts/verify-linux-sidecar.sh +++ b/desktop/scripts/verify-linux-sidecar.sh @@ -18,8 +18,17 @@ trap 'rm -rf "$scratch"' EXIT cd "$scratch" "${images[0]}" --appimage-extract > /dev/null sidecar="$scratch/squashfs-root/usr/bin/ocx" +keyring="$scratch/squashfs-root/usr/lib/OpenCodex/keyring/keyring.linux-x64-gnu.node" test ! -L "$sidecar" +test -f "$keyring" cmp "$original" "$sidecar" sha256sum "$original" "$sidecar" mkdir "$scratch/home" timeout 30s env OPENCODEX_HOME="$scratch/home" "$sidecar" --version +timeout 15s env HOME="$scratch/home" OPENCODEX_HOME="$scratch/home/opencodex" \ + "$sidecar" __keyring-load-check > "$scratch/keyring.json" +python3 - "$scratch/keyring.json" <<'PY' +import json, pathlib, sys +value = json.loads(pathlib.Path(sys.argv[1]).read_text()) +assert value == {"schema": "ocx-keyring-load/1", "available": True}, "Packaged keyring binding is unavailable" +PY diff --git a/desktop/scripts/verify-macos-runtime.sh b/desktop/scripts/verify-macos-runtime.sh index 1d9da80d634..43f1a31375c 100755 --- a/desktop/scripts/verify-macos-runtime.sh +++ b/desktop/scripts/verify-macos-runtime.sh @@ -1,12 +1,16 @@ #!/usr/bin/env bash set -euo pipefail -app="${1:?usage: verify-macos-runtime.sh /path/to/OpenCodex.app}" +app_input="${1:?usage: verify-macos-runtime.sh /path/to/OpenCodex.app}" +app="$(cd "$(dirname "$app_input")" && pwd)/$(basename "$app_input")" [[ "$(uname -s)" == Darwin ]] || { echo 'macOS bundle verification requires macOS' >&2; exit 1; } executable="$(/usr/libexec/PlistBuddy -c 'Print :CFBundleExecutable' "$app/Contents/Info.plist")" [[ -n "$executable" && "$executable" != */* ]] || { echo 'Invalid app executable name' >&2; exit 1; } scratch="$(mktemp -d "${TMPDIR:-/tmp}/opencodex-bundle-check.XXXXXX")" -trap 'rm -rf "$scratch"' EXIT +cleanup() { + rm -rf "$scratch" +} +trap cleanup EXIT codesign --verify --strict --deep "$app" verify_member() { @@ -39,4 +43,30 @@ value = json.loads(pathlib.Path(sys.argv[1]).read_text()) assert value.get("schema") == "ocx-resolve/1", "Unexpected resolve schema" assert value.get("liveness", {}).get("status") in ("live", "absent-proven"), "Unusable resolve result" PY -printf '%s\n' 'PASS: macOS signatures, exact entitlements, hardened runtime, Liquid Glass and bundled CLI resolve' + +# Reproduce the packaged-keyring boundary from an unrelated cwd. This is deliberately load-only: +# an ad-hoc CI identity can trigger a Keychain consent dialog, while issue #6139 is module resolution. +mkdir -p "$scratch/home" "$scratch/work" +python3 - "$app/Contents/MacOS/ocx" "$scratch/work" "$scratch/home" "$scratch/keyring.json" <<'PY' +import os, pathlib, subprocess, sys +ocx, work, home, output = sys.argv[1:] +env = os.environ.copy() +env.update(HOME=home, OPENCODEX_HOME=str(pathlib.Path(home) / ".opencodex")) +try: + with open(output, "wb") as stdout: + subprocess.run( + [ocx, "__keyring-load-check"], cwd=work, env=env, stdout=stdout, + stderr=subprocess.PIPE, check=True, timeout=15, + ) +except subprocess.TimeoutExpired as error: + raise SystemExit("Packaged keyring load probe timed out") from error +except subprocess.CalledProcessError as error: + sys.stderr.buffer.write((error.stderr or b"")[-4096:]) + raise SystemExit(f"Packaged keyring load probe exited {error.returncode}") from error +PY +python3 - "$scratch/keyring.json" <<'PY' +import json, pathlib, sys +value = json.loads(pathlib.Path(sys.argv[1]).read_text()) +assert value == {"schema": "ocx-keyring-load/1", "available": True}, "Packaged keyring binding is unavailable" +PY +printf '%s\n' 'PASS: macOS signatures, entitlements, hardened runtime, Liquid Glass, bundled CLI resolve and packaged keyring' diff --git a/desktop/src-tauri/tauri.conf.json b/desktop/src-tauri/tauri.conf.json index 0242bc5d457..69e4305acc8 100644 --- a/desktop/src-tauri/tauri.conf.json +++ b/desktop/src-tauri/tauri.conf.json @@ -22,7 +22,8 @@ "binaries/ocx" ], "resources": { - "resources/gui/dist": "gui/dist" + "resources/gui/dist": "gui/dist", + "resources/keyring": "keyring" }, "icon": [ "icons/icon.icns", diff --git a/docs-site/src/content/docs/getting-started/installation.md b/docs-site/src/content/docs/getting-started/installation.md index 9206ec9f30c..cd334936180 100644 --- a/docs-site/src/content/docs/getting-started/installation.md +++ b/docs-site/src/content/docs/getting-started/installation.md @@ -66,7 +66,8 @@ are not required. Download the archive for your platform, extract it, and run: ./ocx start ``` -The extracted `gui/dist` directory must stay beside the binary so `GET /` can serve the dashboard. +The extracted `gui/dist` and `keyring` directories must stay beside the binary. The first serves +the dashboard; the second carries the platform-native OS credential-store binding. ### Release channels diff --git a/scripts/build-standalone.ts b/scripts/build-standalone.ts index fdbed98e128..bf023f5609a 100644 --- a/scripts/build-standalone.ts +++ b/scripts/build-standalone.ts @@ -2,6 +2,7 @@ import { createHash } from "node:crypto"; import { cpSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join, resolve, basename } from "node:path"; import { isStandaloneTarget, standaloneExecutableName } from "./standalone-targets"; +import { stageStandaloneKeyringAddon } from "./standalone-keyring"; function hostTarget(): string { const platform = process.platform === "darwin" ? "darwin" : process.platform === "win32" ? "windows" : "linux"; @@ -77,6 +78,10 @@ try { } if (compileExitCode !== 0) process.exit(compileExitCode); +// N-API binaries cannot execute from Bun's virtual `$bunfs`. Keep the exact target addon outside +// the compiled executable so source/npm resolution and packaged resolution share one binding API. +stageStandaloneKeyringAddon(repoRoot, output, target); + // bun's ad-hoc linker signature does not always cover the embedded payload; // macOS kills the executable on launch (SIGKILL) unless it is re-signed. if (process.platform === "darwin") { diff --git a/scripts/keyring-smoke.ts b/scripts/keyring-smoke.ts index 1135887451c..cd3be07ad57 100644 --- a/scripts/keyring-smoke.ts +++ b/scripts/keyring-smoke.ts @@ -1,5 +1,6 @@ #!/usr/bin/env bun import { randomBytes, randomUUID, timingSafeEqual } from "node:crypto"; +import { loadKeyringBinding } from "../src/lib/keyring-native"; export interface KeyringSmokeEntry { setSecret(secret: Uint8Array, signal?: AbortSignal): Promise; @@ -15,8 +16,8 @@ export interface KeyringSmokeOptions { } async function createOsEntry(service: string, account: string): Promise { - const { AsyncEntry } = await import("@napi-rs/keyring"); - return new AsyncEntry(service, account); + const { AsyncEntry } = loadKeyringBinding(); + return new AsyncEntry(service, account) as unknown as KeyringSmokeEntry; } export async function runKeyringSmoke({ diff --git a/scripts/standalone-keyring.ts b/scripts/standalone-keyring.ts new file mode 100644 index 00000000000..a444d698ec8 --- /dev/null +++ b/scripts/standalone-keyring.ts @@ -0,0 +1,30 @@ +import { copyFileSync, existsSync, mkdirSync } from "node:fs"; +import { createRequire } from "node:module"; +import { basename, join } from "node:path"; +import { keyringAssetForStandaloneTarget } from "../src/lib/keyring-native"; + +/** Stage the platform N-API addon outside Bun's virtual filesystem beside a standalone binary. */ +export function stageStandaloneKeyringAddon(repoRoot: string, output: string, target: string): string { + const asset = keyringAssetForStandaloneTarget(target); + if (!asset) throw new Error(`No keyring native asset is declared for standalone target ${target}`); + // Resolve optional target packages from their declaring wrapper. This preserves the lockfile + // relationship without depending on Bun/npm/pnpm choosing a particular hoisting layout. + const projectRequire = createRequire(join(repoRoot, "package.json")); + const keyringRequire = createRequire(projectRequire.resolve("@napi-rs/keyring")); + let source: string; + try { + source = keyringRequire.resolve(asset.packageName); + } catch { + throw new Error( + `Missing ${asset.packageName}/${asset.filename}; install target optional dependencies before building ${target}`, + ); + } + if (basename(source) !== asset.filename || !existsSync(source)) { + throw new Error(`Resolved ${asset.packageName} to an unexpected native asset: ${source}`); + } + const keyringDir = join(output, "keyring"); + mkdirSync(keyringDir, { recursive: true }); + const destination = join(keyringDir, asset.filename); + copyFileSync(source, destination); + return destination; +} diff --git a/src/cli/index.ts b/src/cli/index.ts index 834b37ecbeb..f4870bbe7f9 100755 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -202,9 +202,9 @@ async function refreshOwnedRaycastCatalog( } initializeNodeLauncherContext(); - // The compiled executable is also the capture-only MCP server's launcher. // Handle this private entrypoint before CLI preflight or command dispatch. +if (process.argv[2] === "__keyring-load-check") { console.log(JSON.stringify((await import("../lib/keyring-native")).inspectKeyringBinding())); process.exit(0); } if (process.argv[2] === "__codebuddy-mcp") { const { runCodeBuddyMcpServer } = await import("../adapters/codebuddy/mcp-server"); await runCodeBuddyMcpServer(process.argv[3] ?? ""); diff --git a/src/codex/native-profile-store.ts b/src/codex/native-profile-store.ts index bbd1b5d82e4..7ab797761d6 100644 --- a/src/codex/native-profile-store.ts +++ b/src/codex/native-profile-store.ts @@ -20,6 +20,7 @@ import { } from "node:fs"; import { basename, dirname, join, resolve } from "node:path"; import { getConfigDir } from "../config"; +import { loadKeyringBinding } from "../lib/keyring-native"; import { extractAccountId } from "../oauth/chatgpt"; import { getCodexHome, readRootTomlString } from "./paths"; import { @@ -120,7 +121,7 @@ export class OsNativeProfileKeyProvider implements NativeProfileKeyProvider { private async entry(homeId: string): Promise { try { - const { AsyncEntry } = await import("@napi-rs/keyring"); + const { AsyncEntry } = loadKeyringBinding(); return new AsyncEntry(KEYRING_SERVICE, homeId) as unknown as NativeKeyringEntry; } catch { throw new NativeProfileError( diff --git a/src/lib/keyring-native.ts b/src/lib/keyring-native.ts new file mode 100644 index 00000000000..5ade1a003f0 --- /dev/null +++ b/src/lib/keyring-native.ts @@ -0,0 +1,129 @@ +import { existsSync } from "node:fs"; +import { createRequire } from "node:module"; +import { basename, dirname, join, resolve } from "node:path"; +import { isStandaloneBinary, standaloneRoot } from "./standalone"; + +export interface KeyringBinding { + Entry: new (service: string, account: string) => unknown; + AsyncEntry: new (service: string, account: string) => unknown; +} + +export interface KeyringNativeAsset { + packageName: string; + filename: string; +} + +/** Must match Tauri `productName`, which names Linux's usr/lib resource directory. */ +export const PACKAGED_DESKTOP_PRODUCT_NAME = "OpenCodex"; + +const ASSET_BY_TARGET: Readonly> = { + "bun-darwin-arm64": { + packageName: "@napi-rs/keyring-darwin-arm64", + filename: "keyring.darwin-arm64.node", + }, + "bun-darwin-x64": { + packageName: "@napi-rs/keyring-darwin-x64", + filename: "keyring.darwin-x64.node", + }, + "bun-windows-x64": { + packageName: "@napi-rs/keyring-win32-x64-msvc", + filename: "keyring.win32-x64-msvc.node", + }, + "bun-linux-x64": { + packageName: "@napi-rs/keyring-linux-x64-gnu", + filename: "keyring.linux-x64-gnu.node", + }, + "bun-linux-arm64": { + packageName: "@napi-rs/keyring-linux-arm64-gnu", + filename: "keyring.linux-arm64-gnu.node", + }, +}; + +/** Native addon that must accompany a compiled standalone target. */ +export function keyringAssetForStandaloneTarget(target: string): KeyringNativeAsset | undefined { + return ASSET_BY_TARGET[target]; +} + +function runtimeAsset(platform: NodeJS.Platform, arch: string): KeyringNativeAsset | undefined { + const target = platform === "darwin" + ? `bun-darwin-${arch}` + : platform === "win32" + ? `bun-windows-${arch}` + : platform === "linux" + ? `bun-linux-${arch}` + : ""; + return keyringAssetForStandaloneTarget(target); +} + +/** + * Deterministic packaged-addon paths. Never search cwd: a desktop command can be launched from an + * arbitrary directory, and loading a same-named native file from there would turn cwd into code. + */ +export function packagedKeyringCandidates({ + root = isStandaloneBinary() ? standaloneRoot() : undefined, + platform = process.platform, + arch = process.arch, +}: { + /** Test seam; production supplies the canonical compiled-executable directory. */ + root?: string; + platform?: NodeJS.Platform; + arch?: string; +} = {}): string[] { + // Source/npm installs must stay inside package resolution. Probing beside a shared Bun/Node + // executable would expand the native-code trust boundary and contradict the packaging contract. + if (root === undefined) return []; + const asset = runtimeAsset(platform, arch); + if (!asset) return []; + const executableDir = resolve(root); + const adjacent = join(executableDir, "keyring", asset.filename); + if (platform === "linux") { + const usrDir = dirname(executableDir); + // Tauri installs resources at usr/lib/ while its sidecar is usr/bin/ocx. + // Restrict that fallback to the exact bundle shape; ordinary standalone archives keep the + // executable-owned adjacent directory as their only candidate. + return basename(executableDir) === "bin" && basename(usrDir) === "usr" + ? [adjacent, join(usrDir, "lib", PACKAGED_DESKTOP_PRODUCT_NAME, "keyring", asset.filename)] + : [adjacent]; + } + if (platform !== "darwin") return [adjacent]; + return [ + // Prefer the executable-owned sibling. A standalone layout must not let an unrelated + // app-shaped ../Resources tree override the addon distributed with that executable. + adjacent, + // Tauri resources live in Contents/Resources while its external binary lives in Contents/MacOS. + join(executableDir, "..", "Resources", "keyring", asset.filename), + ]; +} + +const nodeRequire = createRequire(import.meta.url); + +/** + * Load the OS-keyring binding from the immutable packaged location, falling back to normal package + * resolution for source/npm installs. Bun cannot materialize a N-API binary from `$bunfs`, so a + * compiled executable must never depend on `@napi-rs/keyring` resolving inside its virtual tree. + */ +export function loadKeyringBinding({ + candidates = packagedKeyringCandidates(), + fileExists = existsSync, + load = (specifier: string): unknown => nodeRequire(specifier), +}: { + candidates?: string[]; + fileExists?: (path: string) => boolean; + load?: (specifier: string) => unknown; +} = {}): KeyringBinding { + for (const candidate of candidates) { + if (fileExists(candidate)) return load(candidate) as KeyringBinding; + } + return load("@napi-rs/keyring") as KeyringBinding; +} + +/** Load-only package probe: verifies constructors without reading or writing an OS credential. */ +export function inspectKeyringBinding( + load: () => KeyringBinding = loadKeyringBinding, +): { schema: "ocx-keyring-load/1"; available: true } { + const binding = load(); + if (typeof binding.Entry !== "function" || typeof binding.AsyncEntry !== "function") { + throw new Error("The keyring native binding does not export Entry and AsyncEntry constructors"); + } + return { schema: "ocx-keyring-load/1", available: true }; +} diff --git a/src/providers/api-key-resolve.ts b/src/providers/api-key-resolve.ts index 3b78418f98f..9bdfd3deb8c 100644 --- a/src/providers/api-key-resolve.ts +++ b/src/providers/api-key-resolve.ts @@ -16,8 +16,8 @@ * Policy: a reference that cannot be resolved fails closed (no key) and is warned once per * account; nothing ever rewrites plaintext into config or its backups. */ -import { createRequire } from "node:module"; import { resolveEnvValue } from "../config/proxy-env"; +import { loadKeyringBinding } from "../lib/keyring-native"; import type { OcxProviderConfig } from "../types"; export const KEYCHAIN_REFERENCE_PREFIX = "keychain:"; @@ -31,10 +31,8 @@ export interface ProviderKeychainEntry { export type ProviderKeychainEntryFactory = (service: string, account: string) => ProviderKeychainEntry; -const nodeRequire = createRequire(import.meta.url); - function defaultEntryFactory(service: string, account: string): ProviderKeychainEntry { - const { Entry } = nodeRequire("@napi-rs/keyring") as { Entry: new (s: string, a: string) => ProviderKeychainEntry }; + const { Entry } = loadKeyringBinding() as { Entry: new (s: string, a: string) => ProviderKeychainEntry }; return new Entry(service, account); } diff --git a/structure/decisions/ADR-6139-packaged-native-keyring-binding.md b/structure/decisions/ADR-6139-packaged-native-keyring-binding.md new file mode 100644 index 00000000000..cbda6556777 --- /dev/null +++ b/structure/decisions/ADR-6139-packaged-native-keyring-binding.md @@ -0,0 +1,12 @@ +# ADR-6139 — decision recorded under "Packaged native keyring binding" + +- Contract owner: [desktop-shell.md](../desktop-shell.md#packaged-native-keyring-binding) + +## Decision record + +- 목적과 의도: Keep provider and native-profile keychain operations available in compiled standalone and desktop `ocx` binaries, independent of the process working directory. +- 기존 구현 및 제약 조건: Source/npm installs resolve `@napi-rs/keyring` from `node_modules`, but Bun's compiled virtual filesystem cannot materialize its N-API `.node` file. The desktop ships one universal macOS sidecar, Tauri resources are outside that executable, and native bindings must match the executing architecture exactly. +- 검토한 주요 대안: Rely on dynamic package resolution from `$bunfs`; copy the JavaScript wrapper and its package tree; fetch an addon on first use; put an environment-selected arbitrary path into the loader; or stage pinned target addons as release resources and load one deterministic path. +- 선택한 방식: Stage the lockfile-selected platform addon beside every standalone build, copy it into the desktop resource set, carry both Darwin slices in a universal app, and let one shared loader choose a fixed platform/architecture filename from the canonical compiled-executable root before falling back to package resolution for non-compiled installs. +- 다른 대안 대신 이 방식을 선택한 이유: A native file must exist outside `$bunfs`; shipping only that file is smaller and easier to verify than recreating package resolution, needs no runtime download, and avoids making cwd or an ambient environment variable a native-code search path. +- 장점, 단점 및 영향: Packaged keychain paths work from arbitrary directories and fail at build time when an expected optional package is missing. Release archives grow by one addon per target and the universal macOS app by two. Linux Tauri bundles use their fixed `usr/bin` to `usr/lib/OpenCodex` resource layout; non-`usr/bin` standalone layouts do not enter that fallback. The packaged probe proves module loading without touching a credential or depending on an OS consent dialog. Adding another target now requires an explicit asset mapping and packaging test. diff --git a/structure/desktop-shell.md b/structure/desktop-shell.md index cea6e13d084..145d89639f7 100644 --- a/structure/desktop-shell.md +++ b/structure/desktop-shell.md @@ -366,6 +366,33 @@ Bun targets and prepares the external binary plus dashboard resources used by Tauri. Generated files under desktop/src-tauri/binaries/ and desktop/src-tauri/resources/ remain ignored. +### Packaged native keyring binding + +The compiled `ocx` sidecar cannot resolve or execute a N-API addon from Bun's virtual +`$bunfs`. `scripts/build-standalone.ts` therefore stages the exact target's pinned +`@napi-rs/keyring-*` binary under `keyring/`, and `desktop/scripts/prepare-sidecar.ts` +copies that directory into Tauri resources. A universal macOS bundle carries both Darwin +architectures. `src/lib/keyring-native.ts` selects only the platform/architecture filename +under `Contents/Resources/keyring` (or an adjacent standalone `keyring/` directory); it never +searches the launch working directory. Source and npm installs retain ordinary package +resolution and never probe beside the shared Bun or Node executable. Compiled installs derive +their asset root from the executable's canonical real path, so a symlinked launcher still finds +the addon shipped with the real binary. + +The macOS bundle verifier launches the signed sidecar from a disposable unrelated directory and +requires its bounded, load-only keyring probe to expose both native constructors. It does not read +or write an OS credential, which would make an ad-hoc CI identity depend on a consent dialog. +Release verification separately requires both Darwin architecture files inside the universal app. +Merely finding a `.node` file in the source checkout is not sufficient evidence. + +Linux desktop bundles place resources under `usr/lib/OpenCodex` while the sidecar lives under +`usr/bin`. The compiled loader recognizes only that exact bundle shape after the adjacent +standalone directory, and the extracted-AppImage verifier executes the same bounded load-only +probe in ordinary PR CI and release CI. This keeps source/npm runtimes and non-`usr/bin` +standalone layouts out of the Tauri resource fallback. + +> Decision record: [ADR-6139](decisions/ADR-6139-packaged-native-keyring-binding.md) + The management API companion presence check in `src/server/management/companion-routes.ts` accepts both `OpenCodexMenuBar/` (legacy Swift companion) and `OpenCodexDesktop/` user agents. diff --git a/structure/ops/docs-and-release.md b/structure/ops/docs-and-release.md index 168ad3c622f..da337060348 100644 --- a/structure/ops/docs-and-release.md +++ b/structure/ops/docs-and-release.md @@ -403,7 +403,11 @@ working tree and pins that wiring. The `package-standalone` job in `.github/workflows/release.yml` also builds Bun compiled `ocx` archives for Linux, macOS, and Windows, bundles `gui/dist`, smoke-tests `/healthz`, and -publishes SHA-256 sidecars for the attach job. +publishes SHA-256 sidecars for the attach job. Each archive also carries the target-matching +`@napi-rs/keyring` native addon under `keyring/`; the macOS release installs both optional Darwin +packages so its separate arm64 and x64 builds cannot silently reuse the hosted runner's +architecture. Desktop preparation copies those same pinned assets into Tauri resources. The loader +and packaged-app proof are owned by the [desktop keyring contract](../desktop-shell.md#packaged-native-keyring-binding). Opening a release starts with the `dev` pre-move. Dispatch `.github/workflows/dev-version-bump.yml` with the intended version, merge the pull request it opens, diff --git a/tests/ci-workflows/ci-scope-reduction.test.ts b/tests/ci-workflows/ci-scope-reduction.test.ts index 2023c551b58..ff8c09c1814 100644 --- a/tests/ci-workflows/ci-scope-reduction.test.ts +++ b/tests/ci-workflows/ci-scope-reduction.test.ts @@ -318,8 +318,12 @@ describe("the native path filter", () => { "bun.lock", "desktop/**", "package.json", + "scripts/build-standalone.ts", + "scripts/standalone-keyring.ts", "src/cli/index.ts", "src/lib/bun-runtime.ts", + "src/lib/keyring-native.ts", + "src/lib/standalone.ts", "src/service/**", ]); }); diff --git a/tests/ci-workflows/keyring-smoke.test.ts b/tests/ci-workflows/keyring-smoke.test.ts index 9cc9657bb8d..c853c3c0b46 100644 --- a/tests/ci-workflows/keyring-smoke.test.ts +++ b/tests/ci-workflows/keyring-smoke.test.ts @@ -1,5 +1,30 @@ -import { describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, test } from "bun:test"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, relative } from "node:path"; import { runKeyringSmoke, type KeyringSmokeEntry } from "../../scripts/keyring-smoke"; +import { stageStandaloneKeyringAddon } from "../../scripts/standalone-keyring"; +import { + keyringAssetForStandaloneTarget, + inspectKeyringBinding, + loadKeyringBinding, + PACKAGED_DESKTOP_PRODUCT_NAME, + packagedKeyringCandidates, + type KeyringBinding, +} from "../../src/lib/keyring-native"; +import { repoPath } from "../helpers/repo-root"; + +const roots: string[] = []; + +afterEach(() => { + while (roots.length > 0) rmSync(roots.pop()!, { recursive: true, force: true }); +}); + +function tempRoot(): string { + const root = mkdtempSync(join(tmpdir(), "ocx-keyring-package-")); + roots.push(root); + return root; +} class MemoryKeyringEntry implements KeyringSmokeEntry { secret: Buffer | null = null; @@ -132,3 +157,155 @@ describe("runKeyringSmoke", () => { expect(entry.deletes).toBe(1); }); }); + +describe("packaged keyring native binding", () => { + test("maps every standalone release target to an exact native package", () => { + expect(keyringAssetForStandaloneTarget("bun-darwin-arm64")).toEqual({ + packageName: "@napi-rs/keyring-darwin-arm64", + filename: "keyring.darwin-arm64.node", + }); + expect(keyringAssetForStandaloneTarget("bun-darwin-x64")?.filename).toBe("keyring.darwin-x64.node"); + expect(keyringAssetForStandaloneTarget("bun-windows-x64")?.filename).toBe("keyring.win32-x64-msvc.node"); + expect(keyringAssetForStandaloneTarget("bun-linux-x64")?.filename).toBe("keyring.linux-x64-gnu.node"); + expect(keyringAssetForStandaloneTarget("bun-linux-arm64")?.filename).toBe("keyring.linux-arm64-gnu.node"); + expect(keyringAssetForStandaloneTarget("bun-freebsd-x64")).toBeUndefined(); + }); + + test("source installs use package resolution instead of probing beside Bun", () => { + expect(packagedKeyringCandidates()).toEqual([]); + }); + + test("prefers the standalone-adjacent addon before a macOS app resource", () => { + expect(packagedKeyringCandidates({ + root: "/Applications/OpenCodex.app/Contents/MacOS", + platform: "darwin", + arch: "arm64", + })).toEqual([ + "/Applications/OpenCodex.app/Contents/MacOS/keyring/keyring.darwin-arm64.node", + "/Applications/OpenCodex.app/Contents/Resources/keyring/keyring.darwin-arm64.node", + ]); + }); + + test("finds the Linux Tauri resource only for an usr/bin sidecar layout", () => { + expect(packagedKeyringCandidates({ + root: "/tmp/.mount-OpenCodex/usr/bin", + platform: "linux", + arch: "x64", + })).toEqual([ + "/tmp/.mount-OpenCodex/usr/bin/keyring/keyring.linux-x64-gnu.node", + "/tmp/.mount-OpenCodex/usr/lib/OpenCodex/keyring/keyring.linux-x64-gnu.node", + ]); + expect(packagedKeyringCandidates({ root: "/opt/ocx", platform: "linux", arch: "x64" })) + .toEqual(["/opt/ocx/keyring/keyring.linux-x64-gnu.node"]); + }); + + test("loads only an existing deterministic packaged path and never consults cwd", () => { + const binding = { Entry: class {}, AsyncEntry: class {} } as unknown as KeyringBinding; + const calls: string[] = []; + const result = loadKeyringBinding({ + candidates: ["/signed/app/keyring.node", "./keyring.node"], + fileExists: path => path === "/signed/app/keyring.node", + load: specifier => { calls.push(specifier); return binding; }, + }); + expect(result).toBe(binding); + expect(calls).toEqual(["/signed/app/keyring.node"]); + }); + + test("falls back to package resolution for source and npm installs", () => { + const calls: string[] = []; + loadKeyringBinding({ + candidates: ["/missing/keyring.node"], + fileExists: () => false, + load: specifier => { calls.push(specifier); return { Entry: class {}, AsyncEntry: class {} }; }, + }); + expect(calls).toEqual(["@napi-rs/keyring"]); + }); + + test("the load-only probe verifies constructors without touching a credential", () => { + expect(inspectKeyringBinding(() => ({ Entry: class {}, AsyncEntry: class {} }))).toEqual({ + schema: "ocx-keyring-load/1", + available: true, + }); + expect(() => inspectKeyringBinding(() => ({ Entry: class {}, AsyncEntry: null } as unknown as KeyringBinding))) + .toThrow("does not export Entry and AsyncEntry"); + }); + + test("stages the selected addon under the standalone output", () => { + const root = tempRoot(); + const output = join(root, "dist", "standalone", "bun-darwin-arm64"); + const asset = keyringAssetForStandaloneTarget("bun-darwin-arm64")!; + const wrapper = join(root, "node_modules", "@napi-rs", "keyring"); + const packageRoot = join(wrapper, "node_modules", asset.packageName); + const source = join(packageRoot, asset.filename); + mkdirSync(join(source, ".."), { recursive: true }); + writeFileSync(join(wrapper, "package.json"), JSON.stringify({ name: "@napi-rs/keyring", main: "index.js" })); + writeFileSync(join(wrapper, "index.js"), "module.exports = {};"); + writeFileSync(join(packageRoot, "package.json"), JSON.stringify({ name: asset.packageName, main: asset.filename })); + writeFileSync(source, "native-addon"); + const destination = stageStandaloneKeyringAddon(root, output, "bun-darwin-arm64"); + expect(destination).toBe(join(output, "keyring", asset.filename)); + expect(existsSync(destination)).toBe(true); + expect(readFileSync(destination, "utf8")).toBe("native-addon"); + }); + + test("stages an addon beside a symlinked wrapper in a virtual store", () => { + const root = tempRoot(); + const output = join(root, "out"); + const asset = keyringAssetForStandaloneTarget("bun-darwin-arm64")!; + const scope = join(root, "node_modules", "@napi-rs"); + const store = join(root, "node_modules", ".pnpm"); + const parent = join(store, "keyring-wrapper", "node_modules"); + const wrapper = join(parent, "@napi-rs", "keyring"); + const addon = join(store, "keyring-addon", "node_modules", asset.packageName); + mkdirSync(scope, { recursive: true }); + mkdirSync(wrapper, { recursive: true }); + mkdirSync(addon, { recursive: true }); + writeFileSync(join(wrapper, "package.json"), JSON.stringify({ name: "@napi-rs/keyring", main: "index.js" })); + writeFileSync(join(wrapper, "index.js"), "module.exports = {};"); + writeFileSync(join(addon, "package.json"), JSON.stringify({ name: asset.packageName, main: asset.filename })); + writeFileSync(join(addon, asset.filename), "virtual-store-addon"); + symlinkSync(relative(scope, wrapper), join(scope, "keyring"), "dir"); + symlinkSync(relative(join(parent, "@napi-rs"), addon), join(parent, asset.packageName), "dir"); + + const destination = stageStandaloneKeyringAddon(root, output, "bun-darwin-arm64"); + expect(readFileSync(destination, "utf8")).toBe("virtual-store-addon"); + }); + + test("refuses a build whose target optional dependency was not installed", () => { + const root = tempRoot(); + const wrapper = join(root, "node_modules", "@napi-rs", "keyring"); + mkdirSync(wrapper, { recursive: true }); + writeFileSync(join(wrapper, "package.json"), JSON.stringify({ name: "@napi-rs/keyring", main: "index.js" })); + writeFileSync(join(wrapper, "index.js"), "module.exports = {};"); + expect(() => stageStandaloneKeyringAddon(root, join(root, "out"), "bun-darwin-x64")) + .toThrow("install target optional dependencies"); + }); + + test("desktop and release packaging retain the external addon and packaged-app proof", () => { + const config = JSON.parse(readFileSync(repoPath("desktop", "src-tauri", "tauri.conf.json"), "utf8")); + expect(config.productName).toBe(PACKAGED_DESKTOP_PRODUCT_NAME); + expect(config.bundle.resources["resources/keyring"]).toBe("keyring"); + const release = readFileSync(repoPath(".github", "workflows", "release.yml"), "utf8"); + expect(release).toContain("ocx.exe,gui,keyring"); + expect(release).toContain("ocx gui keyring"); + expect(release).toContain("--os=${{ matrix.dependency_os }} --cpu=${{ matrix.dependency_cpu }}"); + expect(release).toContain('dependency_cpu: "*"'); + expect(release).toContain("Verify the packaged universal macOS runtime"); + expect(release).toContain("keyring.darwin-arm64.node"); + expect(release).toContain("keyring.darwin-x64.node"); + const ci = readFileSync(repoPath(".github", "workflows", "ci.yml"), "utf8"); + expect(ci).toContain("src/lib/standalone.ts"); + expect(ci).toContain("Verify packaged Linux sidecar keyring"); + expect(ci).toContain('bash desktop/scripts/verify-linux-sidecar.sh "$BUNDLE_ROOT/appimage"'); + const verify = readFileSync(repoPath("desktop", "scripts", "verify-macos-runtime.sh"), "utf8"); + expect(verify).toContain('app="$(cd "$(dirname "$app_input")" && pwd)/$(basename "$app_input")"'); + expect(verify).toContain("cwd=work"); + expect(verify).toContain('"__keyring-load-check"'); + expect(verify).toContain('"schema": "ocx-keyring-load/1"'); + const verifyLinux = readFileSync(repoPath("desktop", "scripts", "verify-linux-sidecar.sh"), "utf8"); + expect(verifyLinux).toContain("usr/lib/OpenCodex/keyring/keyring.linux-x64-gnu.node"); + expect(verifyLinux).toContain("__keyring-load-check"); + const cli = readFileSync(repoPath("src", "cli", "index.ts"), "utf8"); + expect(cli).toContain('process.argv[2] === "__keyring-load-check"'); + }); +}); diff --git a/tests/ci-workflows/linux-desktop-packaged-ci.test.ts b/tests/ci-workflows/linux-desktop-packaged-ci.test.ts index abd1d14d1f5..10caad018ec 100644 --- a/tests/ci-workflows/linux-desktop-packaged-ci.test.ts +++ b/tests/ci-workflows/linux-desktop-packaged-ci.test.ts @@ -35,6 +35,7 @@ describe("Linux packaged desktop E2E in CI", () => { expect(shell?.if).toContain("needs.changes.outputs.desktop == 'true'"); const checkResources = shell?.steps?.find(step => step.name === "Prepare desktop check resources"); expect(checkResources?.run).toContain("binaries/ocx-"); + expect(checkResources?.run).toContain("resources/keyring"); expect(checkResources?.run).not.toContain("resources/sidecar/ocx"); const preserve = shell?.steps?.find(step => step.name === "Preserve the compiled Linux sidecar"); expect(preserve?.run).toContain("chmod +x desktop/scripts/appimage-patchelf.py"); @@ -48,6 +49,10 @@ describe("Linux packaged desktop E2E in CI", () => { expect(stage?.run).toContain("$APPIMAGE_BUNDLE/."); expect(stage?.run).toContain("$DEB_BUNDLE/."); expect(stage?.run).toContain('chmod -R a-w "$BUNDLE_ROOT"'); + const verifyKeyring = shell?.steps?.find(step => step.name === "Verify packaged Linux sidecar keyring"); + expect(verifyKeyring?.if).toBe("needs.changes.outputs.desktop == 'true'"); + expect(verifyKeyring?.env?.BUNDLE_ROOT).toContain("opencodex-linux-bundles"); + expect(verifyKeyring?.run).toBe('bash desktop/scripts/verify-linux-sidecar.sh "$BUNDLE_ROOT/appimage"'); const aggregate = workflow.jobs?.ci?.steps?.find(step => step.name === "Assert every job this event requested succeeded"); expect(aggregate?.env?.CHANGES_DESKTOP).toBe("${{ needs.changes.outputs.desktop }}"); diff --git a/tests/ci-workflows/release-desktop-scripts.test.ts b/tests/ci-workflows/release-desktop-scripts.test.ts index 0dd5865942f..db141b5e1c2 100644 --- a/tests/ci-workflows/release-desktop-scripts.test.ts +++ b/tests/ci-workflows/release-desktop-scripts.test.ts @@ -154,6 +154,8 @@ describe("desktop release scripts", () => { test("the Linux sidecar verifier takes the staged AppImage directory and keeps the local default", () => { const verifier = readFileSync(repoPath("desktop", "scripts", "verify-linux-sidecar.sh"), "utf8"); expect(verifier).toContain('bundle="${1:-$root/desktop/src-tauri/target/x86_64-unknown-linux-gnu/release/bundle/appimage}"'); + expect(verifier).toContain("usr/lib/OpenCodex/keyring/keyring.linux-x64-gnu.node"); + expect(verifier).toContain("__keyring-load-check"); const wrapper = readFileSync(repoPath("desktop", "scripts", "appimage-patchelf.py"), "utf8"); expect(wrapper).toContain('os.environ.get("CARGO_TARGET_DIR"'); expect(wrapper).toContain("APPDIR_SIDECAR_TAIL"); diff --git a/tests/gui/gui-desktop-sidecar-script.test.ts b/tests/gui/gui-desktop-sidecar-script.test.ts index 413e155f566..24ac6d3f818 100644 --- a/tests/gui/gui-desktop-sidecar-script.test.ts +++ b/tests/gui/gui-desktop-sidecar-script.test.ts @@ -14,5 +14,7 @@ test("desktop sidecar preparation maps supported Rust targets", () => { expect(script).toContain("build:standalone"); expect(script).toContain("binaries"); expect(script).toContain("resources"); + expect(script).toContain("keyringResources"); expect(script).toContain("ocx-${triple}"); + expect(script).not.toContain("if (!existsSync(executable))"); }); diff --git a/tests/gui/standalone-build-script.test.ts b/tests/gui/standalone-build-script.test.ts index 7dab243ae2e..4a04129920f 100644 --- a/tests/gui/standalone-build-script.test.ts +++ b/tests/gui/standalone-build-script.test.ts @@ -18,5 +18,6 @@ test("standalone build script exposes supported targets and packaging contract", expect(script).toContain("--compile"); expect(script).toContain("--outfile"); expect(script).toContain("gui/dist"); + expect(script).toContain("stageStandaloneKeyringAddon"); expect(script).toContain("SHA256SUMS"); }); From 0126cb43c12c328e3eafe9ee50101f257cc9dacd Mon Sep 17 00:00:00 2001 From: JUN Date: Wed, 30 Sep 2026 04:11:16 +0900 Subject: [PATCH 17/26] fix(oauth): let quota probes survive an active-account switch The auxiliary quota fences added for pause also aborted on any selection change, so a report probe in flight during an active-account switch no longer seeded the probed account. Pause, reauth and a replaced token still stop the send; the reading stays attributed to the account that was probed. --- src/providers/quota.ts | 3 +-- src/providers/quota/vendor-probes-oauth.ts | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/providers/quota.ts b/src/providers/quota.ts index 494c97543c1..07c8cf75a5d 100644 --- a/src/providers/quota.ts +++ b/src/providers/quota.ts @@ -424,7 +424,6 @@ async function fetchAccountQuota( if (explicitAccountReader(provider)) return fetchExplicitAccountQuota(provider, accountId, forceRefresh, providerConfig); if (provider === "anthropic" || provider === "kiro") hydrateAccountQuotaCache(); const key = accountCacheKey(provider, accountId); - const selectionRevision = provider === "anthropic" ? getAccountSet(provider)?.selectionRevision : undefined; const writerGeneration = captureConfigGeneration(); const kiroIdentity = provider === "kiro" ? kiroProbeIdentity(accountId) : undefined; const cachedCandidate = accountQuotaCache.get(key); @@ -469,7 +468,7 @@ async function fetchAccountQuota( if (result.kind === "unavailable") quotaFailure = result.failure; } else if (provider === "anthropic") { const row = getAccountCredentialWithStatus(provider, accountId); - if (!row || row.paused || row.needsReauth || row.credential.access !== token || getAccountSet(provider)?.selectionRevision !== selectionRevision) return { ts: Date.now(), quota: null, unavailable: true }; + if (!row || row.paused || row.needsReauth || row.credential.access !== token) return { ts: Date.now(), quota: null, unavailable: true }; quota = await fetchAnthropicUsageQuota(token); } else { return { ts: Date.now(), quota: null, unavailable: true }; diff --git a/src/providers/quota/vendor-probes-oauth.ts b/src/providers/quota/vendor-probes-oauth.ts index 134075f9820..2ee0d5472f8 100644 --- a/src/providers/quota/vendor-probes-oauth.ts +++ b/src/providers/quota/vendor-probes-oauth.ts @@ -350,9 +350,9 @@ export async function fetchAnthropicQuota(provider: string): Promise Date: Wed, 30 Sep 2026 04:12:00 +0900 Subject: [PATCH 18/26] fix(responses): strip internal summary:none marker on the wire (#6247) Carries #6232 by @cshyang: the internal reasoning summary "none" marker is stripped at final outbound serialization so native Responses upstreams do not reject it. Co-authored-by: cshyang --- scripts/test-layout/layout.json | 1 + scripts/test-layout/seeds.json | 3 +- src/adapters/openai-responses/passthrough.ts | 3 +- src/adapters/openai-responses/reasoning.ts | 15 +++++++ structure/transports/responses-wire-shapes.md | 2 +- tests/fixtures/test-layout-expected.json | 1 + .../openai-responses-summary-none.test.ts | 42 +++++++++++++++++++ 7 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 tests/responses/openai-responses-summary-none.test.ts diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 3e8814ba98f..78af7a75e64 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1214,6 +1214,7 @@ "openai-provider-option-tooling.test.ts": "adapters/openai", "openai-provider-option.test.ts": "adapters/openai", "openai-responses-passthrough.test.ts": "responses", + "openai-responses-summary-none.test.ts": "responses", "responses-forward-output-cap.test.ts": "responses", "opencode-cli.test.ts": "providers", "opencode-free-provider.test.ts": "providers", diff --git a/scripts/test-layout/seeds.json b/scripts/test-layout/seeds.json index 394627e716b..806bdabba99 100644 --- a/scripts/test-layout/seeds.json +++ b/scripts/test-layout/seeds.json @@ -55,12 +55,13 @@ "^(?:anthropic)-" ], "openai": [ - "^(?:openai)-" + "^openai-(?!responses-summary-none\\.test\\.ts$)" ] } }, "responses": { "match": [ + "^openai-responses-summary-none\\.test\\.ts$", "^(?:apply|chat|citation|continuation|eventstream|legacy|namespace|passthrough|responses|sse|thought|ws)-" ] }, diff --git a/src/adapters/openai-responses/passthrough.ts b/src/adapters/openai-responses/passthrough.ts index a3e5a85e1ca..d4533405dd0 100644 --- a/src/adapters/openai-responses/passthrough.ts +++ b/src/adapters/openai-responses/passthrough.ts @@ -34,7 +34,7 @@ import { import { createAdapterTierMetadata, } from "../../providers/fastwire"; -import { dropResponsesReasoningInputItems, mapRoutedResponsesReasoningEffort, normalizeConfiguredReasoningSummaryDelivery, sanitizeReasoningInputContent, stripDisabledReasoningSummaries, stripDisabledVerbosity, stripUnsupportedReasoningSummaryDelivery } from "./reasoning"; +import { dropResponsesReasoningInputItems, mapRoutedResponsesReasoningEffort, normalizeConfiguredReasoningSummaryDelivery, sanitizeReasoningInputContent, stripDisabledReasoningSummaries, stripDisabledVerbosity, stripNoneReasoningSummary, stripUnsupportedReasoningSummaryDelivery } from "./reasoning"; import { scrubOcxCompactionItems, stripCanonicalOnlyToolFields, stripCanonicalOnlyTopLevelFields, stripInternalChatMessageMetadataPassthrough, stripInvalidItemIds, stripItemIdsWhenUnstored, stripRejectedSamplingParams } from "./request-strips"; import { stripCanonicalForwardPromptCacheOptions, stripDeprecatedPromptCacheRetention } from "./prompt-cache"; import { isPlainObject } from "./internal"; @@ -352,6 +352,7 @@ export function createResponsesPassthroughAdapter(provider: OcxProviderConfig): outBody = repairOversizedReplayCallIds(outBody); } outBody = stripUnsupportedReasoningSummaryDelivery(outBody, parsed.modelId); + outBody = stripNoneReasoningSummary(outBody); // #4587: on a bridged provider, hand the destination back the search call and result the // proxy executed on its behalf, in place of the hosted cell the caller replays. Scoped to // its exact conversation and serving identity and recorded by the bridge itself, so a diff --git a/src/adapters/openai-responses/reasoning.ts b/src/adapters/openai-responses/reasoning.ts index 5f19edec946..5b79f9d7cec 100644 --- a/src/adapters/openai-responses/reasoning.ts +++ b/src/adapters/openai-responses/reasoning.ts @@ -156,6 +156,21 @@ export function sanitizeReasoningInputContent( return changed ? { ...raw, input } : body; } +/** + * `reasoning.summary: "none"` is an internal "hide the thinking summary" marker: Claude's + * `thinking.display: "omitted"` maps to it, and the parser reads it into + * `options.hideThinkingSummary`. It is not a Responses API value (valid: auto, concise, + * detailed), so drop it from the body that is serialized to the upstream. + */ +export function stripNoneReasoningSummary(body: unknown): unknown { + if (!isPlainObject(body) || !isPlainObject(body.reasoning) || body.reasoning.summary !== "none") return body; + const { summary: _summary, ...rest } = body.reasoning; + const next = { ...body }; + if (Object.keys(rest).length > 0) next.reasoning = rest; + else delete next.reasoning; + return next; +} + export function stripUnsupportedReasoningSummaryDelivery(body: unknown, modelId: string): unknown { if (catalogModelSupportsReasoningSummaries(modelId) !== false) return body; if (!isPlainObject(body) || !isPlainObject(body.stream_options)) return body; diff --git a/structure/transports/responses-wire-shapes.md b/structure/transports/responses-wire-shapes.md index c00e84de13e..31f2610de7d 100644 --- a/structure/transports/responses-wire-shapes.md +++ b/structure/transports/responses-wire-shapes.md @@ -569,7 +569,7 @@ An injected combo default supplies `summary: "auto"` only when no summary was sp summary choices remain intact. Raw display and hidden-envelope replay follow [reasoning display parity](../providers/chat-compat.md#reasoning-display-parity-hidethinkingsummary). Final-route normalization preserves visible raw reasoning when the parsed request has a validated -active effort and omits summary; explicit `summary: "none"` still hides it. +active effort and omits summary; explicit `summary: "none"` still hides it (passthrough strips that internal marker before the upstream send). The provider policy `hideRawReasoning` suppresses the raw `reasoning_raw_delta` channel only — openai-chat `reasoning_content`, kiro tags, and Gemini thought parts on routes that do not return thought summaries (direct and Vertex Gemini; a `cloud-code-assist` Gemini route emits its thought diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index b35190edf94..21b46b9efa1 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1226,6 +1226,7 @@ "openai-provider-option-tooling.test.ts": "adapters/openai", "openai-provider-option.test.ts": "adapters/openai", "openai-responses-passthrough.test.ts": "responses", + "openai-responses-summary-none.test.ts": "responses", "responses-forward-output-cap.test.ts": "responses", "opencode-cli.test.ts": "providers", "opencode-free-provider.test.ts": "providers", diff --git a/tests/responses/openai-responses-summary-none.test.ts b/tests/responses/openai-responses-summary-none.test.ts new file mode 100644 index 00000000000..f052c8fe07a --- /dev/null +++ b/tests/responses/openai-responses-summary-none.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, test } from "bun:test"; +import { createResponsesPassthroughAdapter } from "../../src/adapters/openai-responses"; +import { anthropicToResponsesBody } from "../../src/claude/inbound"; +import { parseRequest } from "../../src/responses/parser"; +import { withTestTranslatorBudget } from "../helpers/translator-budget"; + +const targets = [ + { adapter: "openai-responses", baseUrl: "https://chatgpt.com/backend-api/codex", authMode: "forward" as const }, + { adapter: "openai-responses", baseUrl: "https://api.openai.com/v1", authMode: "key" as const, apiKey: "sk-test" }, +]; + +function outboundReasoning(target: (typeof targets)[number], body: Record): Record | undefined { + const adapter = withTestTranslatorBudget(createResponsesPassthroughAdapter(target)); + const request = adapter.buildRequest(parseRequest(body), { headers: new Headers() }); + return (JSON.parse(request.body) as { reasoning?: Record }).reasoning; +} + +describe("Responses summary:none wire marker", () => { + test("Claude omitted thinking retains parser intent but removes the marker for both destinations", () => { + const claudeBody = anthropicToResponsesBody({ + model: "gpt-6-astra", + max_tokens: 10, + messages: [{ role: "user", content: "hi" }], + thinking: { type: "adaptive", display: "omitted" }, + output_config: { effort: "high" }, + }); + expect((claudeBody as { reasoning?: unknown }).reasoning).toEqual({ summary: "none", effort: "high" }); + expect(parseRequest(claudeBody).options.hideThinkingSummary).toBe(true); + + for (const target of targets) { + expect(outboundReasoning(target, claudeBody)).toEqual({ effort: "high" }); + expect(outboundReasoning(target, { model: "gpt-6-astra", input: "hi", reasoning: { summary: "none" } })).toBeUndefined(); + } + }); + + test.each(["auto", "concise", "detailed"])("preserves valid summary %s", summary => { + for (const target of targets) { + expect(outboundReasoning(target, { model: "gpt-6-astra", input: "hi", reasoning: { effort: "medium", summary } })) + .toEqual({ effort: "medium", summary }); + } + }); +}); From 99878c356997f013d3aa029e6d43c912c088e5ac Mon Sep 17 00:00:00 2001 From: JUN Date: Wed, 30 Sep 2026 04:33:34 +0900 Subject: [PATCH 19/26] fix(combos): a spent pool account must not cool the whole target (carry #6234) (#6255) Carries #6234 by @vadymhimself: a 429 attributed to a pooled account that was itself cooled no longer cools the whole combo target, so healthy sibling accounts stay usable; unknown-account 429s still cool the target. Co-authored-by: Vadym O --- docs-site/src/content/docs/guides/combos.md | 4 + scripts/test-layout/layout.json | 1 + src/combos/resolve.ts | 17 +- src/oauth/anthropic-routing.ts | 50 +++-- src/providers/label.ts | 21 +- src/server/responses/adapter-continuation.ts | 6 + src/server/responses/adapter-dispatch.ts | 6 + src/server/responses/core-combo.ts | 28 +++ structure/providers-and-adapters.md | 2 + structure/runtime.md | 2 +- structure/transports/inventory.md | 2 +- .../anthropic-combo-account-cooldown.test.ts | 184 ++++++++++++++++++ tests/fixtures/test-layout-expected.json | 1 + tests/routing/always-on-429-failover.test.ts | 18 +- .../server-combo-cooldown-recording.test.ts | 18 ++ tests/usage/usage-provider-label.test.ts | 9 +- 16 files changed, 341 insertions(+), 28 deletions(-) create mode 100644 tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts diff --git a/docs-site/src/content/docs/guides/combos.md b/docs-site/src/content/docs/guides/combos.md index 2049192ce2d..90fba7f99d6 100644 --- a/docs-site/src/content/docs/guides/combos.md +++ b/docs-site/src/content/docs/guides/combos.md @@ -354,6 +354,10 @@ five seconds. A valid immediate `Retry-After: 0` remains an immediate upstream directive rather than being replaced by a configured cooldown. +For an Anthropic OAuth or Codex pool, a 429 tied to one account that the pool has cooled does +not cool the whole combo target. Other accounts behind that target remain available. A 429 with +no identified, cooled pool account still cools the target, as do provider-wide failures. + ### Last-resort targets A brief cooldown on a preferred target otherwise routes straight to whatever diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 78af7a75e64..96c50b8f374 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -100,6 +100,7 @@ "ambiguous-resend-composition.test.ts": "lib", "ambiguous-resend-gate.test.ts": "lib", "anthropic-account-pool.test.ts": "adapters/anthropic", + "anthropic-combo-account-cooldown.test.ts": "adapters/anthropic", "anthropic-model-routes.test.ts": "adapters/anthropic", "anthropic-agentrouter-language-framing.test.ts": "adapters/anthropic", "anthropic-baseurl-override.test.ts": "adapters/anthropic", diff --git a/src/combos/resolve.ts b/src/combos/resolve.ts index bec48fde34d..1b2ad26370d 100644 --- a/src/combos/resolve.ts +++ b/src/combos/resolve.ts @@ -380,13 +380,28 @@ export function advanceComboAfterFailure( status?: number; code?: string | null; message?: string; + /** Account-qualified provider label of the account this failure came from, when a pool resolved one. */ + failedAccount?: string; } = {}, ): ComboPick | null { noteComboFailure(pick.comboId, pick.target, pick.writerGeneration); const combo = getCombo(config, pick.comboId); + // A 429 from a pooled provider names the ACCOUNT that is spent, not the target. The pool + // records that account's own cooldown and rotates past it, so cooling the target here blacks + // out the accounts that still answer: on 2026-09-28 one Anthropic account replied + // `Retry-After: 318747` (88.5h) and anthropic/claude-opus-5-5 left selection for four hours + // while four of five accounts still returned 200. + // + // Only a pool-resolved account qualifies, and an unidentified account still cools the target -- + // including the pool's OWN "all accounts are cooled" 429, which refuses locally before it picks + // one. That is what keeps this from trading a blackhole for a hammer: when nothing else is + // holding the target back, the target cooldown still does. + const accountScoped = options.cooldownScope === "target" + && options.status === 429 + && options.failedAccount !== undefined; // "none" records no cooldown at all: the failure described the request, not the target, so // the target must stay immediately selectable for the next (differently shaped) request. - if (options.cooldownScope !== "none") { + if (options.cooldownScope !== "none" && !accountScoped) { const cooldownTargets = options.cooldownScope === "provider" && combo ? combo.targets.filter(target => target.provider === pick.target.provider) : [pick.target]; diff --git a/src/oauth/anthropic-routing.ts b/src/oauth/anthropic-routing.ts index 36327c6d43d..60865a9d05a 100644 --- a/src/oauth/anthropic-routing.ts +++ b/src/oauth/anthropic-routing.ts @@ -736,13 +736,41 @@ export function rotateAnthropicAccountOn429( rateLimitHeaders?: AnthropicRateLimitHeaders | null, decision: AnthropicRouteDecision | null = null, ): string | null { + if (!recordAnthropicAccount429(config, failedAccountId, retryAfterHeader, now, rateLimitHeaders)) return null; + + // The pool's strategy is a PROACTIVE policy. When the pool is disabled, reactive + // presence-only recovery must not silently reactivate round-robin/fill-first merely + // because those dormant values remain in config. The quota picker is the neutral + // recovery policy already used by the default strategy. + const next = isAnthropicAccountPoolEnabled(config) + ? pickAlternateAnthropicAccount(config, failedAccountId, now, decision) + : pickLowestUsage(config, failedAccountId, now); + if (!next) { + console.warn(`[anthropic-pool] ${decision ? `route:#${decision.position} ` : ""}no eligible replacement; returning 429`); + return null; + } + + console.warn( + `[anthropic-pool] ${decision ? `route:#${decision.position} ` : ""}429 on ${formatAnthropicAccountOrdinal(failedAccountId)}; failing over to ${formatAnthropicAccountOrdinal(next)}`, + ); + return next; +} + +/** Record the account refusal even when this request has no remaining retry sends. */ +export function recordAnthropicAccount429( + config: OcxConfig, + failedAccountId: string, + retryAfterHeader: string | null | undefined, + now = Date.now(), + rateLimitHeaders?: AnthropicRateLimitHeaders | null, +): boolean { // Reactive 429 failover is NOT gated on the pool flag. That flag buys PROACTIVE routing -- // session affinity, quota-ranked new-session selection, autoSwitchThreshold, strategy -- all // of which move a HEALTHY request and stay opt-in. Rotating away from an account upstream has // just rate-limited is a different thing: it only ever runs after a refusal, and stranding a // 429 while a second logged-in account sits idle is a defect, not a configuration choice. // Presence is the activation rule, the same one an apiKeyPool of two keys already uses. - if (!isAnthropicAccountPoolEnabled(config) && !hasAnthropicFailoverQuorum(now)) return null; + if (!isAnthropicAccountPoolEnabled(config) && !hasAnthropicFailoverQuorum(now)) return false; // Retry-After first: it is the header written FOR this decision. The rejected window's // reset is the fallback, because a 429 that omits Retry-After still carries it -- and @@ -760,26 +788,10 @@ export function rotateAnthropicAccountOn429( sweepExpiredOnWrite(now); clearAnthropicSessionAffinityForAccount(failedAccountId); notePoolRotationFailure(POOL_KEY_ANTHROPIC, failedAccountId); - // A rotation means the roster in use just changed; do not answer the next activation question - // from a count read taken before the failure. + // The refused account changed the eligible roster even when no retry send remains. quorumCache = null; - // The pool's strategy is a PROACTIVE policy. When the pool is disabled, reactive - // presence-only recovery must not silently reactivate round-robin/fill-first merely - // because those dormant values remain in config. The quota picker is the neutral - // recovery policy already used by the default strategy. - const next = isAnthropicAccountPoolEnabled(config) - ? pickAlternateAnthropicAccount(config, failedAccountId, now, decision) - : pickLowestUsage(config, failedAccountId, now); - if (!next) { - console.warn(`[anthropic-pool] ${decision ? `route:#${decision.position} ` : ""}no eligible replacement; returning 429`); - return null; - } - - console.warn( - `[anthropic-pool] ${decision ? `route:#${decision.position} ` : ""}429 on ${formatAnthropicAccountOrdinal(failedAccountId)}; failing over to ${formatAnthropicAccountOrdinal(next)}`, - ); - return next; + return true; } export interface AnthropicSelectionRoutingOptions { diff --git a/src/providers/label.ts b/src/providers/label.ts index 10f1a09e705..f1faf5f0c53 100644 --- a/src/providers/label.ts +++ b/src/providers/label.ts @@ -1,4 +1,4 @@ -import { CODEX_ACCOUNT_LOG_LABEL_RE, KEY_ACCOUNT_LOG_LABEL_RE, apiKeyAccountLogLabel, oauthAccountLogLabel } from "../codex/account-label"; +import { ACCOUNT_LOG_LABEL_RE, CODEX_ACCOUNT_LOG_LABEL_RE, KEY_ACCOUNT_LOG_LABEL_RE, apiKeyAccountLogLabel, oauthAccountLogLabel } from "../codex/account-label"; import type { OcxProviderConfig } from "../types"; export function canonicalUsageProviderLabel(provider: string): string { @@ -25,6 +25,25 @@ export function stampApiKeyAccountLabel( } } +/** + * The account-qualified provider label a pool dispatch stamped for this request, or undefined. + * + * An account pool rewrites the logged provider to name the account that served the turn + * (`anthropic-p2e22d0`, `openai-main`); the configured provider name never does, so a label that + * differs from the configured one AND ends in an account label identifies exactly one account. + * + * Deliberately NOT `accountLogLabel`: that is stamped for a lone configured API key too, so it + * cannot tell a pooled account apart from a provider that has no pool to rotate within. + */ +export function poolAccountProviderLabel( + loggedProvider: string | undefined, + configuredProvider: string, +): string | undefined { + if (!loggedProvider?.startsWith(`${configuredProvider}-`)) return undefined; + return ACCOUNT_LOG_LABEL_RE.test(loggedProvider.slice(configuredProvider.length + 1)) + ? loggedProvider : undefined; +} + export function baseProviderLabel(provider: string): string { const canonical = canonicalUsageProviderLabel(provider); if (canonical !== provider) return canonical; diff --git a/src/server/responses/adapter-continuation.ts b/src/server/responses/adapter-continuation.ts index 7db1a602e16..d111cff8577 100644 --- a/src/server/responses/adapter-continuation.ts +++ b/src/server/responses/adapter-continuation.ts @@ -35,6 +35,7 @@ import { bindRouteReasoningReplayScope } from "./core-replay"; import { ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST, rotateAnthropicAccountOn429, + recordAnthropicAccount429, getAnthropicPoolAccessSnapshot, formatAnthropicProviderForLog, } from "../../oauth/anthropic-routing"; @@ -425,6 +426,11 @@ export function createAdapterContinuations( } } } + if (response.status === 429 && transportState.anthropicPoolAccountId + && transportState.anthropicPoolFailovers >= ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST) { + recordAnthropicAccount429(config, transportState.anthropicPoolAccountId, + response.headers.get("retry-after"), Date.now(), response.headers); + } // Generic OAuth rotation for the continuation loop. The streaming loop grew this arm with // #2568 and this one did not, so an xAI/Cursor/Kimi/Copilot/Antigravity/Nous continuation // 429 stayed terminal even with failover fully active -- the same class of divergence the diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index b836e619b2c..f0df55c1bd5 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -56,6 +56,7 @@ import { bindRouteReasoningReplayScope } from "./core-replay"; import { ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST, rotateAnthropicAccountOn429, + recordAnthropicAccount429, getAnthropicPoolAccessSnapshot, formatAnthropicProviderForLog, } from "../../oauth/anthropic-routing"; @@ -1000,6 +1001,11 @@ export async function prepareAdapterExchange( break; } } + if (upstreamResponse.status === 429 && transportState.anthropicPoolAccountId + && transportState.anthropicPoolFailovers >= ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST) { + recordAnthropicAccount429(config, transportState.anthropicPoolAccountId, + upstreamResponse.headers.get("retry-after"), Date.now(), upstreamResponse.headers); + } // Generic OAuth account failover (#2568) rotates reactively after a refusal when // two accounts are stored. Kiro additionally classifies bounded 400/403 refusals; // all other providers retain the original 429 loop below. diff --git a/src/server/responses/core-combo.ts b/src/server/responses/core-combo.ts index 1752e66d8db..622a5a026ee 100644 --- a/src/server/responses/core-combo.ts +++ b/src/server/responses/core-combo.ts @@ -46,6 +46,11 @@ import { import { hasUnreadableEncryptedAgentTask } from "./encrypted-payload"; import { routeConcreteModel, comboRouteDecisionTrace } from "../../router"; import { memoryModelRouteReason } from "./memory-models"; +import { poolAccountProviderLabel } from "../../providers/label"; +import { getAccountSet } from "../../oauth/store"; +import { formatAnthropicProviderForLog, getAnthropicAccountHealthSnapshot } from "../../oauth/anthropic-routing"; +import { codexAccountLogLabel } from "../../codex/account-label"; +import { codexQuotaScopeForModel, getCodexQuotaHealthSnapshot } from "../../codex/routing"; import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers"; import type { AgentTaskRecoveryFailureReason } from "./agent-task-recovery"; import { @@ -97,6 +102,22 @@ import { clientWireOf } from "../inference/client-wire"; */ export const COMBO_TARGET_BASE_SENDS = CODEX_TEXT_GUARDED_BUDGET_POLICY.baseSendAllowance; +function cooledPoolAccountLabel(config: OcxConfig, providerName: string, modelId: string, label: string | undefined): string | undefined { + if (!label) return undefined; + if (providerName === "anthropic") { + const matches = getAccountSet("anthropic")?.accounts.filter(account => + formatAnthropicProviderForLog("anthropic", account.id) === label) ?? []; + return matches.length === 1 && getAnthropicAccountHealthSnapshot(matches[0]!.id) ? label : undefined; + } + const provider = config.providers[providerName]; + if (!provider || !isCanonicalOpenAiForwardProvider(provider)) return undefined; + const matches = (config.codexAccounts ?? []).filter(account => + `${providerName}-${codexAccountLogLabel(account)}` === label); + return matches.length === 1 + && getCodexQuotaHealthSnapshot(matches[0]!.id, codexQuotaScopeForModel(modelId)) + ? label : undefined; +} + /** * A combo's execution policy is DECLARED by the combo, not inherited from the single-target @@ -977,6 +998,13 @@ export async function executeComboResponses( status: failure.response.status, code: failure.upstreamCode, message: failure.classificationText, + // The dispatch rewrote this to name the pool account that actually served the turn. + failedAccount: cooledPoolAccountLabel( + config, + pick.target.provider, + pick.target.model, + poolAccountProviderLabel(childLog.provider, pick.target.provider), + ), onCooldownRecorded: target => { failedTargetCooldownRecorded ||= targetKey(target) === failedTargetKey; }, diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index b35fe601b45..89efd30d9d7 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -2,6 +2,8 @@ For Anthropic OAuth, `src/oauth/anthropic-routing.ts` applies the first matching `anthropicAccountPool.routes` rule to every eligible pick. The declared account order is stable while its candidates remain eligible; active, manual, affinity, quota and strategy preferences only choose inside that set. A healthy session affinity outside a model route is ignored for that request and retained for later unrouted or differently routed models; the routed commit does not overwrite it. An explicit fallback widens an empty route to the ordinary pool, and fill-first then advances in ordinary pool order from the active account. A missing eligible route fails locally without that fallback. The rules are operator allowlists, not provider entitlement evidence. Request logs use `route:#` for the 1-based rule position, not the operator name. +An Anthropic 429 records the served account's cooldown even when the request has used its allowed retry sends. That final account remains excluded on the next request; combo target cooling is skipped only after the matching account cooldown is present. + GitHub Copilot `modelContextTiers` is selected per upstream model. The Chat and Responses adapters set `contextTier` only when the canonical routed provider is `github-copilot` and a tier is configured. Otherwise passthrough retains caller-supplied values. The server carries provider identity diff --git a/structure/runtime.md b/structure/runtime.md index 4d3f257e049..6e8036d85a4 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -504,7 +504,7 @@ This is also why the classifier cannot duplicate visible output. Native byte str Regression coverage: `tests/responses/responses-forward-prompt-envelope.test.ts`, `tests/routing/router-combo-failover-classification.test.ts`, `tests/routing/routing-policy-fallback.test.ts`, `tests/helpers/combo-context-overflow-cases.ts`, and `tests/server/server-combo-failover-e2e.test.ts`. -`src/combos/failover.ts` uses a 10-minute fallback for a spent account usage window (codes `usage_limit_exceeded`, `usage_limit_reached`, `1308`, or `usage limit reached` / `usage limit has been reached` / `token-plan quota has been exhausted` prose, including HTTP 502) and for provider-scoped credential or billing failure codes such as `invalid_api_key` and `insufficient_quota`. This duration does not change failure classification or cooldown scope; upstream retry/reset signals and configured durations retain precedence. It caps explicit upstream `Retry-After` target cooldowns at 24 hours while reset-derived, configured, and fallback cooldowns remain capped at 10 minutes. +`src/combos/failover.ts` uses a 10-minute fallback for a spent account usage window (codes `usage_limit_exceeded`, `usage_limit_reached`, `1308`, or `usage limit reached` / `usage limit has been reached` / `token-plan quota has been exhausted` prose, including HTTP 502) and for provider-scoped credential or billing failure codes such as `invalid_api_key` and `insufficient_quota`. This duration does not change failure classification or cooldown scope; upstream retry/reset signals and configured durations retain precedence. It caps explicit upstream `Retry-After` target cooldowns at 24 hours while reset-derived, configured, and fallback cooldowns remain capped at 10 minutes. `src/combos/resolve.ts` omits a target cooldown for a 429 only when the failed child identifies one account in an Anthropic OAuth or canonical Codex pool and that account has a live pool cooldown. `src/server/responses/core-combo.ts` checks the pool's health record before passing that attribution; unknown accounts, ambiguous labels, API-key and generic OAuth providers, and other failures keep their ordinary target cooldown. ## Combo default effort precedence diff --git a/structure/transports/inventory.md b/structure/transports/inventory.md index dfdb4a86cd8..18252c7916c 100644 --- a/structure/transports/inventory.md +++ b/structure/transports/inventory.md @@ -41,7 +41,7 @@ surface is listed here so a maintainer can find the owner without grepping: | Image/video generation loop | `src/images/loop.ts`, `src/images/plan.ts`, `src/images/fulfill.ts`, `src/images/xai-client.ts`, `src/images/xai-video-client.ts`, `src/images/artifacts.ts` | A provider-returned image URL is downloaded into a local artifact once, then served locally; warnings stay URL-free because provider CDN URLs may embed credentials. Artifact downloads go through the pinned-IP transport with a 10 s connect deadline (`DOWNLOAD_CONNECT_TIMEOUT_MS`) that bounds TCP/TLS setup on its own, in addition to the 60 s idle timer, and `pinnedHttpsGet` accepts a per-call `connectTimeoutMs`. | | GitHub Copilot | `src/providers/xai-transport.ts` (`resolveProviderTransport`), `src/providers/github-copilot-transport.ts` | `resolveProviderTransport` selects the Copilot transport when the routed provider name is `github-copilot`; the Copilot module then resolves its headers and base URL, and the registry seeds the provider row and model fallback. | | API-key pools | `src/providers/api-key-selection.ts`, `src/providers/key-failover.ts` | A configured `apiKeyPoolStrategy` plus a cooling committed key rotates before the first send (`selectProactiveApiKeyTransport`); a 429 still rotates after the send and records a cooldown. `provider.apiKey` keeps mirroring the active entry so routing stays single-key. The pick is inert without a strategy or while the committed key is healthy. | -| OAuth account failover | `src/oauth/generic-account-failover.ts`, `src/oauth/anthropic-routing.ts`, `src/server/responses/antigravity-validation-refusal.ts` | Reactive pre-output 429 recovery is presence-driven with 2+ eligible accounts. Primary Antigravity inference through the Google adapter main dispatch may send once on a live sibling after a replayed 401, a terminal credential-refresh failure that marks the account for reauthentication, or a normalized structured `VALIDATION_REQUIRED` 403, within the existing budget. The 401 and 403 sibling paths share one request guard; unrelated 403s and transient refresh failures do not rotate. Continuations, native passthrough and sidecars keep their existing behavior. Generic OAuth accounts can be operator-paused; paused accounts are excluded from selection, failover, and proactive Token Guardian refresh. If every account is paused, the request fails with 403 rather than as a login failure. Pool and `oauthAccountFailover` flags govern proactive routing, not the reactive retry: a disabled Anthropic pool recovers through quota ordering rather than its dormant strategy, a per-provider `enabled` beats the global default in either direction, and a non-positive fill-first threshold disables proactive usage-based rotation. Kiro's `kiroAutoSelection` projects the same candidate eligibility into routing and account-list status; unknown evidence stays eligible, while pause, reauth, suspension, cooldown, and confirmed exhaustion have closed reasons. An active singleton or a pool excluded only from automatic Kiro selection may still attempt its active account; an explicitly paused active account is never dispatched, and an all-paused pool fails with 403. A configured Kiro capacity cap can also time out. Kiro's optional process-local account lease is released at response completion or cancellation; other providers retain their admission path. | +| OAuth account failover | `src/oauth/generic-account-failover.ts`, `src/oauth/anthropic-routing.ts`, `src/server/responses/antigravity-validation-refusal.ts` | Reactive pre-output 429 recovery is presence-driven with 2+ eligible accounts. Anthropic records the refused account's cooldown even when the request has no retry sends left. Primary Antigravity inference through the Google adapter main dispatch may send once on a live sibling after a replayed 401, a terminal credential-refresh failure that marks the account for reauthentication, or a normalized structured `VALIDATION_REQUIRED` 403, within the existing budget. The 401 and 403 sibling paths share one request guard; unrelated 403s and transient refresh failures do not rotate. Continuations, native passthrough and sidecars keep their existing behavior. Generic OAuth accounts can be operator-paused; paused accounts are excluded from selection, failover, and proactive Token Guardian refresh. If every account is paused, the request fails with 403 rather than as a login failure. Pool and `oauthAccountFailover` flags govern proactive routing, not the reactive retry: a disabled Anthropic pool recovers through quota ordering rather than its dormant strategy, a per-provider `enabled` beats the global default in either direction, and a non-positive fill-first threshold disables proactive usage-based rotation. Kiro's `kiroAutoSelection` projects the same candidate eligibility into routing and account-list status; unknown evidence stays eligible, while pause, reauth, suspension, cooldown, and confirmed exhaustion have closed reasons. An active singleton or a pool excluded only from automatic Kiro selection may still attempt its active account; an explicitly paused active account is never dispatched, and an all-paused pool fails with 403. A configured Kiro capacity cap can also time out. Kiro's optional process-local account lease is released at response completion or cancellation; other providers retain their admission path. | | OAuth login callback (inbound) | `src/oauth/callback-server.ts` | Every response, including non-callback 404s, closes its connection so a pooled socket cannot deliver a later login to a retired flow on the same callback port. | | Alibaba regions | `src/providers/alibaba-region-backup.ts`, `src/providers/alibaba-region-migration.ts`, `src/providers/alibaba-region-startup.ts` | Region migration backs up before rewriting and is idempotent across restarts. | | Discovery and quota | `src/providers/model-discovery.ts`, `src/providers/quota.ts`, `src/providers/registry.ts` | Discovery rejects a response over 4 MiB or past 2,000 raw rows before caching it. Provider-scoped hints fill capabilities omitted by live rosters; OpenCode Go's `deepseek-v4.1-flash` keeps its 1,048,576-token context window. The fixed-key Opper preset uses the shared OpenAI Chat adapter at `https://api.opper.ai/v3/compat`, discovers models through its conventional authenticated `/models` path, preserves an older same-named custom destination, and falls back to bare pool ids while passing vendor-prefixed ids through unchanged. Codex quota DTOs suppress retired Spark evidence under the [OpenAI scope contract](../providers/openai-tiers.md#public-provider-contract), retaining ordinary custom windows. | diff --git a/tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts b/tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts new file mode 100644 index 00000000000..5dabba1e6d3 --- /dev/null +++ b/tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts @@ -0,0 +1,184 @@ +/** + * A pooled Anthropic provider must survive one spent account. + * + * 2026-09-28, 01:17-05:18 WITA: one account answered `Retry-After: 318747` (88.5 hours, its + * weekly window gone). The combo layer cooled `anthropic/` itself, so the gateway + * answered 503 with ZERO upstream sends for four hours while a direct probe showed four of five + * accounts returning 200. The pool had already cooled the one spent account and would have + * routed past it. + */ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { clearComboSelectionState } from "../../../src/combos/resolve"; +import { clearComboTargetCooldowns, isComboTargetInCooldown } from "../../../src/combos/failover"; +import { + clearAnthropicAccountPoolState, + forgetAnthropicFailoverQuorum, + getAnthropicAccountHealthSnapshot, +} from "../../../src/oauth/anthropic-routing"; +import { clearGenericFailoverHealth } from "../../../src/oauth/generic-account-failover"; +import { getAccountSet, saveCredential, setActiveAccount } from "../../../src/oauth/store"; +import { clearAccountQuotaCache, resetProviderQuotaReconcileStateForTests } from "../../../src/providers/quota"; +import { clearResponseStateForTests, flushResponseState } from "../../../src/responses/state"; +import { handleResponses } from "../../../src/server/responses"; +import type { OcxConfig, OcxProviderConfig } from "../../../src/types"; +import { acquireOwnedSpendHome } from "../../helpers/owned-spend-home"; +import { removeTreeWithRetry } from "../../helpers/remove-tree"; + +const MODEL = "claude-sonnet-4-5"; +const TARGET = { provider: "anthropic", model: MODEL }; +/** The outage's own header: 88.5 hours, far past every local cooldown ceiling. */ +const SPENT_WEEK_RETRY_AFTER = "318747"; + +const originalHome = process.env.OPENCODEX_HOME; +let home = ""; +let releaseSpendHome: (() => void) | undefined; +let sentTokens: string[] = []; + +function credential(index: number) { + return { + access: `synthetic-anthropic-access-${index}`, + refresh: `synthetic-anthropic-refresh-${index}`, + expires: Date.now() + 3_600_000, + accountId: `synthetic-account-${index}`, + }; +} + +async function seed(count: number): Promise { + for (let index = 0; index < count; index++) await saveCredential("anthropic", credential(index)); + const ids = getAccountSet("anthropic")!.accounts.map(account => account.id); + await setActiveAccount("anthropic", ids[0]!); + return ids; +} + +function spent(): Response { + return Response.json( + { type: "error", error: { type: "rate_limit_error", message: "synthetic weekly quota exhausted" } }, + { status: 429, headers: { "retry-after": SPENT_WEEK_RETRY_AFTER } }, + ); +} + +function answered(): Response { + return Response.json({ + id: "msg_synthetic", type: "message", role: "assistant", model: MODEL, + content: [{ type: "text", text: "The answer is complete." }], + stop_reason: "end_turn", usage: { input_tokens: 8, output_tokens: 6 }, + }); +} + +function configFor(reply: () => Response, poolEnabled = true): OcxConfig { + const transport = (async (_input, init) => { + sentTokens.push(String(new Headers(init?.headers).get("authorization"))); + return reply(); + }) as typeof fetch; + const provider: OcxProviderConfig & { fetch: typeof fetch } = { + adapter: "anthropic", baseUrl: "https://anthropic-combo.test", authMode: "oauth", + models: [MODEL], fetch: transport, + }; + return { + port: 0, defaultProvider: "anthropic", + anthropicAccountPool: { enabled: poolEnabled, strategy: "round-robin" }, + providers: { anthropic: provider }, + combos: { pooled: { strategy: "failover", targets: [TARGET] } }, + }; +} + +async function post(config: OcxConfig): Promise { + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "combo/pooled", input: "Answer briefly", stream: false }), + }), config, { model: "", provider: "" }); + await response.text(); + return response; +} + +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), "ocx-anthropic-combo-cooldown-")); + process.env.OPENCODEX_HOME = home; + sentTokens = []; + clearComboSelectionState(); + clearComboTargetCooldowns(); + clearAnthropicAccountPoolState(); + forgetAnthropicFailoverQuorum(); + clearGenericFailoverHealth(); + clearAccountQuotaCache(); + resetProviderQuotaReconcileStateForTests(); + clearResponseStateForTests(); + releaseSpendHome = acquireOwnedSpendHome(); +}); + +afterEach(async () => { + await flushResponseState(); + releaseSpendHome?.(); + releaseSpendHome = undefined; + clearComboSelectionState(); + clearComboTargetCooldowns(); + clearAccountQuotaCache(); + clearAnthropicAccountPoolState(); + forgetAnthropicFailoverQuorum(); + clearGenericFailoverHealth(); + resetProviderQuotaReconcileStateForTests(); + clearResponseStateForTests(); + if (originalHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = originalHome; + if (home) removeTreeWithRetry(home); +}); + +test("a spent pool account does not cool the combo target for the accounts that still answer", async () => { + const ids = await seed(5); + let refuse = true; + const config = configFor(() => refuse ? spent() : answered()); + + const refused = await post(config); + expect(refused.status).toBe(429); + // Only the accounts this request actually reached are cooled, and the pool -- not the combo + // layer -- is what cooled them. + const cooled = ids.filter(id => getAnthropicAccountHealthSnapshot(id) !== null); + expect(cooled.length).toBeGreaterThan(0); + expect(cooled.length).toBeLessThan(ids.length); + // The last refused account must be recorded even though the request spent its retry budget. + expect(cooled).toHaveLength(4); + expect(isComboTargetInCooldown("pooled", TARGET)).toBe(false); + + refuse = false; + sentTokens = []; + const served = await post(config); + expect(served.status).toBe(200); + // The retry left through an account the first request had not spent. + expect(sentTokens.length).toBeGreaterThan(0); + // Map each COOLED id back to its own token. `cooled` is a filtered subset of `ids`, so its + // positions are not account indexes: with accounts 0 and 2 cooled, indexing by position would + // check tokens 0 and 1 and let a retry through cooled account 2 unnoticed. + const cooledTokens = cooled.map(id => `Bearer ${credential(ids.indexOf(id)).access}`); + expect(cooledTokens).not.toContain(sentTokens[0]!); + // Stated the other way round, from the token actually sent: resolve it back to its account and + // assert that account is one the first request never cooled. + const servedIndex = ids.findIndex((_, index) => sentTokens[0] === `Bearer ${credential(index).access}`); + expect(servedIndex).toBeGreaterThanOrEqual(0); + expect(cooled).not.toContain(ids[servedIndex]!); +}, 20_000); + +test("every pool account spent still cools the combo target", async () => { + await seed(2); + const config = configFor(spent); + // Drive requests until the pool itself refuses locally: with no account resolved the failure + // names none, and an unidentified account must still cool the target rather than hammer. + for (let attempt = 0; attempt < 4 && !isComboTargetInCooldown("pooled", TARGET); attempt++) { + expect((await post(config)).status).toBe(429); + } + expect(isComboTargetInCooldown("pooled", TARGET)).toBe(true); + const sendsBefore = sentTokens.length; + // Cooled target, single-target combo: the next request is refused without an upstream send. + expect((await post(config)).status).toBe(503); + expect(sentTokens.length).toBe(sendsBefore); +}, 20_000); + +test("a lone OAuth account without pool cooldown still cools the combo target", async () => { + await seed(1); + const config = configFor(spent, false); + expect((await post(config)).status).toBe(429); + expect(isComboTargetInCooldown("pooled", TARGET)).toBe(true); + expect(sentTokens).toHaveLength(1); +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 21b46b9efa1..f2cb4ae134a 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -107,6 +107,7 @@ "ambiguous-resend-composition.test.ts": "lib", "ambiguous-resend-gate.test.ts": "lib", "anthropic-account-pool.test.ts": "adapters/anthropic", + "anthropic-combo-account-cooldown.test.ts": "adapters/anthropic", "anthropic-model-routes.test.ts": "adapters/anthropic", "anthropic-agentrouter-language-framing.test.ts": "adapters/anthropic", "anthropic-baseurl-override.test.ts": "adapters/anthropic", diff --git a/tests/routing/always-on-429-failover.test.ts b/tests/routing/always-on-429-failover.test.ts index 2068c7a58ac..af977d0feb0 100644 --- a/tests/routing/always-on-429-failover.test.ts +++ b/tests/routing/always-on-429-failover.test.ts @@ -164,14 +164,24 @@ describe("proactive Anthropic routing stays opt-in", () => { // fail -- every behavioural test seeds two accounts, which satisfies the quorum either way, // so they would keep passing while the feature was dead for the users who never opted in. // - // Pin the shape instead: the flag may still appear in the rotator, but only alongside the - // presence check, never as a gate of its own. + // Pin the activation gate in the recorder, which the rotator now calls before + // choosing a replacement. The rotator may use the flag separately to select its + // proactive strategy, but must not reject a pool-off request before recording. const source = await Bun.file("src/oauth/anthropic-routing.ts").text(); const start = source.indexOf("export function rotateAnthropicAccountOn429"); expect(start).toBeGreaterThan(-1); const body = source.slice(start, source.indexOf("\n}", start)); - const gate = body.split("\n").find(line => line.includes("isAnthropicAccountPoolEnabled")); - expect(gate, "the rotator no longer references the pool flag at all").toBeDefined(); + const recordCall = body.indexOf("if (!recordAnthropicAccount429("); + expect(recordCall, "the rotator no longer uses the recorder's quorum gate").toBeGreaterThan(-1); + expect(body.slice(0, recordCall), "the rotator added a pool-only gate before recording") + .not.toContain("isAnthropicAccountPoolEnabled"); + + const recordStart = source.indexOf("export function recordAnthropicAccount429"); + expect(recordStart).toBeGreaterThan(-1); + const recordBody = source.slice(recordStart, source.indexOf("\n}", recordStart)); + const gate = recordBody.split("\n").find(line => + line.trimStart().startsWith("if (") && line.includes("isAnthropicAccountPoolEnabled")); + expect(gate, "the recorder no longer checks the pool flag").toBeDefined(); expect(gate, "the pool flag became a gate of its own again").toContain("hasAnthropicFailoverQuorum"); }); }); diff --git a/tests/server/server-combo-cooldown-recording.test.ts b/tests/server/server-combo-cooldown-recording.test.ts index 64760a404e1..337244a62fa 100644 --- a/tests/server/server-combo-cooldown-recording.test.ts +++ b/tests/server/server-combo-cooldown-recording.test.ts @@ -114,6 +114,24 @@ test("advance reports only the current failure's committed cooldown", () => { expect(recorded).toEqual([]); expect(isComboTargetInCooldown("free", target)).toBe(true); }); +test("a pool account's 429 cools nothing at the combo layer, other failures still cool the target", () => { + const cfg = config(); + const account = { failedAccount: "anthropic-p2e22d0" }; + const cool = (options: Parameters[2]): boolean => { + clearComboTargetCooldowns(); + advanceComboAfterFailure(cfg, pickComboTarget(cfg, "free")!, { cooldownScope: "target", ...options }); + return isComboTargetInCooldown("free", target); + }; + // The outage's own header, attributed to one account of a pool. + expect(cool({ status: 429, retryAfter: "318747", ...account })).toBe(false); + // Unknown account: nothing else holds the target back, so the target cooldown still must. + expect(cool({ status: 429, retryAfter: "318747" })).toBe(true); + // A genuine target-level failure is not the account's fault and cools as it always did. + expect(cool({ status: 503, ...account })).toBe(true); + expect(cool({ status: 410, code: "model_not_found", ...account })).toBe(true); + // Provider-scoped evidence (a rejected credential) outranks the account attribution. + expect(cool({ status: 429, cooldownScope: "provider", ...account })).toBe(true); +}); test("a stale in-flight single-target request does not replay a reconciled-away target", async () => { let hits = 0; upstream = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch() { diff --git a/tests/usage/usage-provider-label.test.ts b/tests/usage/usage-provider-label.test.ts index 3627efbd727..018f1da880b 100644 --- a/tests/usage/usage-provider-label.test.ts +++ b/tests/usage/usage-provider-label.test.ts @@ -1,9 +1,16 @@ import { describe, expect, test } from "bun:test"; -import { baseProviderLabel } from "../../src/providers/label"; +import { baseProviderLabel, poolAccountProviderLabel } from "../../src/providers/label"; import { summarizeUsage } from "../../src/usage/summary"; import type { PersistedUsageEntry } from "../../src/usage/log"; const FIXED_NOW = Date.UTC(2026, 5, 28, 12, 0, 0); + +test("pool account labels must belong to the configured provider", () => { + expect(poolAccountProviderLabel("anthropic-pabc123", "anthropic")).toBe("anthropic-pabc123"); + expect(poolAccountProviderLabel("other-pabc123", "anthropic")).toBeUndefined(); + expect(poolAccountProviderLabel("anthropic", "anthropic")).toBeUndefined(); + expect(poolAccountProviderLabel("anthropic-kabc123", "anthropic")).toBeUndefined(); +}); function entry(overrides: Partial & { ts: number }): PersistedUsageEntry { const { ts, ...rest } = overrides; return { From 36a12fc9c789d9a4b078d0a483d1c7f2c2f35e7f Mon Sep 17 00:00:00 2001 From: JUN Date: Wed, 30 Sep 2026 04:34:26 +0900 Subject: [PATCH 20/26] fix(quota): suppress switched Anthropic provider report --- src/providers/quota/vendor-probes-oauth.ts | 7 ++++++- structure/providers/anthropic-account-pool.md | 2 +- tests/providers/provider-account-quota.test.ts | 5 ++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/providers/quota/vendor-probes-oauth.ts b/src/providers/quota/vendor-probes-oauth.ts index 2ee0d5472f8..c4c4d006be1 100644 --- a/src/providers/quota/vendor-probes-oauth.ts +++ b/src/providers/quota/vendor-probes-oauth.ts @@ -18,6 +18,7 @@ import { aggregateCodexPoolCapacity, CODEX_CAPACITY_MAX_QUOTA_AGE_MS, type Codex import { asRecord, normalizePercent, normalizeResetAt, readQuotaJson, REQUEST_TIMEOUT_MS, toFiniteNumber } from "../quota-wire"; import { providerCodexAccountMode } from "../registry"; import { + accountReportCurrent, TERMINAL_QUOTA_FAILURE, hasQuotaRows, providerLabel, @@ -363,7 +364,11 @@ export async function fetchAnthropicQuota(provider: string): Promise getAccountSet("anthropic")?.activeAccountId === probedAccountId); + } + return result; } /** diff --git a/structure/providers/anthropic-account-pool.md b/structure/providers/anthropic-account-pool.md index 6d6ebae120c..54564e418a0 100644 --- a/structure/providers/anthropic-account-pool.md +++ b/structure/providers/anthropic-account-pool.md @@ -21,7 +21,7 @@ produce its scoped 429 and Retry-After, not a login error. An already-dispatched retains a successful rotated credential without unpausing; a late failure cannot mark the paused row for reauthentication. Token Guardian and Anthropic quota probes recheck live pause, selection and bearer ownership after token resolution and before each usage send; -a newly paused account makes no auxiliary request. Pool-off keeps a healthy active account; +a newly paused account makes no auxiliary request. An account switch during a usage probe still seeds the probed account's quota cache but suppresses its stale provider report and reset observation. Pool-off keeps a healthy active account; pause/prior-429 recovery uses `only-eligible`, and logs name the committed account. > Decision record: [ADR-6013](../decisions/ADR-6013-anthropic-account-pause.md) diff --git a/tests/providers/provider-account-quota.test.ts b/tests/providers/provider-account-quota.test.ts index 25c6ae74bb1..e0f1dff3241 100644 --- a/tests/providers/provider-account-quota.test.ts +++ b/tests/providers/provider-account-quota.test.ts @@ -367,7 +367,10 @@ describe("fetchProviderAccountQuotas", () => { // Switch active mid-flight before Anthropic responds. await setActiveAccount("anthropic", second!.id); releaseUsage(); - await reportPromise; + const switchedReport = await reportPromise; + expect(switchedReport.reports).toEqual([]); + expect(getCachedProviderAccountQuota("anthropic", first!.id)?.fiveHourPercent).toBe(70); + expect(getCachedProviderAccountQuota("anthropic", second!.id)).toBeNull(); // First account still owns token-first — seed must land on first, not second. clearProviderQuotaCache(); From be218ba688fe2ca54ae61b63b996a03b67355804 Mon Sep 17 00:00:00 2001 From: JUN Date: Wed, 30 Sep 2026 04:54:07 +0900 Subject: [PATCH 21/26] fix(service): accept standalone Windows wrappers in ownership probe (carry #6238) (#6258) Carries #6238 by @lcBreathe (fixes #6237): the Windows standalone service wrapper is accepted by the ownership probe, and only a wrapper and registered task that match the standalone generator (quoted OCX_BUN assignment, generator markers and control flow, executable bound to the recorded install state, single wscript.exe Exec action with the exact launcher arguments) can claim ownership. Co-authored-by: lcBreathe <165003424+lcBreathe@users.noreply.github.com> --- src/service-manager-probe.ts | 121 ++++++- src/service/windows-taskxml.ts | 26 +- structure/ops/service-and-sidecars.md | 14 + ...ex-service-manager-probe-hardening.test.ts | 4 +- .../codex-service-manager-probe.test.ts | 304 +++++++++++++++++- 5 files changed, 443 insertions(+), 26 deletions(-) diff --git a/src/service-manager-probe.ts b/src/service-manager-probe.ts index d3d722bcfb5..418987e990c 100644 --- a/src/service-manager-probe.ts +++ b/src/service-manager-probe.ts @@ -28,6 +28,10 @@ import { } from "./lib/windows-elevation"; import { decodeWindowsTextBytes } from "./lib/windows-text"; import { WINSW_SERVICE_ID } from "./lib/winsw"; +import { BUN_RUNTIME_PATH_ENV, BUN_RUNTIME_SOURCE_ENV } from "./lib/bun-runtime"; +import { WINDOWS_WRAPPER_PROTOCOL_ENV, WINDOWS_WRAPPER_STAY_OUT_EXIT_CODE } from "./service/windows-wrapper-exit"; +import { buildWindowsServiceScript, windowsTaskActionMatches } from "./service/windows-taskxml"; +import { inspectServiceStateEvidence, serviceStatePathsForOpenCodexHome } from "./service/state"; /** Short: this runs inside admission, and a slow answer is the same as none. */ export const SERVICE_PROBE_TIMEOUT_MS = 2_000; @@ -192,6 +196,8 @@ export interface ProbeDeps { readonly windowsLocale?: string; /** Startup-local full-listing cache; targeted task queries always bypass it. */ readonly windowsTaskListingCache?: WindowsTaskListingCache; + /** Service-state evidence paths; production derives them from the effective config home. */ + readonly statePaths?: readonly string[]; } const LABEL = "com.opencodex.proxy"; @@ -559,9 +565,95 @@ function decodeBatchPathValue( .replaceAll(escapedPercent, "%"); } -/** Validate the generated wrapper before interpreting omitted optional homes. */ -function wrapperLooksGenerated(body: string): boolean { - return /:loop\s*[\s\S]*^"%OCX_BUN%" "%OCX_CLI%" start\b[^\r\n]*$/im.test(body); +/** One generated quoted assignment, rejecting unquoted and duplicate forms. */ +function generatedBatchSetValue(body: string, name: string): string | null { + const assignments = body.split(/\r?\n/).filter(line => new RegExp(`^\\s*@?set\\s+"?${name}=`, "i").test(line)); + return assignments.length === 1 ? batchSetValue(assignments[0]!, name) : null; +} + +/** Compare executable lines with the actual standalone generator's ordered script. */ +function matchesGeneratedStandaloneControlFlow(body: string, port: number): boolean { + const scriptLines = (script: string): string[] => { + const lines = script.replace(/\r\n/g, "\n").split("\n"); + if (lines.at(-1) === "") lines.pop(); + return lines; + }; + const lines = scriptLines(body); + const expected = scriptLines(buildWindowsServiceScript({ + bun: "C:\\OpenCodex\\ocx.exe", bunRuntimeSource: "standalone", cli: null, + }, port, [])); + const tokenBlock = 'if exist "%OCX_API_TOKEN_FILE%" ('; + const boundary = lines.indexOf(tokenBlock); + const expectedBoundary = expected.indexOf(tokenBlock); + if (boundary < 0 || expectedBoundary < 0) return false; + const allowed = [ + "OCX_SERVICE", WINDOWS_WRAPPER_PROTOCOL_ENV, BUN_RUNTIME_SOURCE_ENV, + BUN_RUNTIME_PATH_ENV, "PATH", "CODEX_HOME", "CODEX_SQLITE_HOME", + "OPENCODEX_HOME", "HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY", + "OCX_API_TOKEN_FILE", "OCX_SERVICE_LOG", "OCX_BUN", + ]; + const required = [ + "OCX_SERVICE", WINDOWS_WRAPPER_PROTOCOL_ENV, BUN_RUNTIME_SOURCE_ENV, + BUN_RUNTIME_PATH_ENV, "OCX_API_TOKEN_FILE", + "OCX_SERVICE_LOG", "OCX_BUN", + ]; + let previous = -1; + const seen = new Set(); + for (const line of lines.slice(0, boundary)) { + if (line === 'set "ERRORLEVEL="') continue; + if (!line.startsWith('set "')) continue; + const name = /^set "([A-Z_]+)=[^"\r\n]*"$/.exec(line)?.[1]; + const index = name ? allowed.indexOf(name) : -1; + if (index <= previous || !name) return false; + previous = index; + seen.add(name); + } + if (required.some(name => !seen.has(name))) return false; + const withoutPrefixSets = (scriptLines: string[], end: number): string[] => + scriptLines.filter((line, index) => index >= end || line === 'set "ERRORLEVEL="' || !line.startsWith('set "')); + const actualFlow = withoutPrefixSets(lines, boundary); + const generatedFlow = withoutPrefixSets(expected, expectedBoundary); + return actualFlow.length === generatedFlow.length + && actualFlow.every((line, index) => line === generatedFlow[index]); +} + +/** Validate the generated launch shape before interpreting omitted optional homes. */ +function wrapperLaunchShape(body: string): "source" | { standaloneBun: string } | null { + if (!/^:loop\s*$/im.test(body)) return null; + const launchLines = body.split(/\r?\n/).filter(line => /^\s*"%OCX_BUN%"/i.test(line)); + if (launchLines.length !== 1) return null; + const launch = launchLines[0]!.trim(); + const sourceLaunch = /^"%OCX_BUN%" "%OCX_CLI%" start --port ([0-9]{1,5}) >>"%OCX_SERVICE_LOG%" 2>&1$/i; + const standaloneLaunch = /^"%OCX_BUN%" start --port ([0-9]{1,5}) >>"%OCX_SERVICE_LOG%" 2>&1$/i; + const bun = generatedBatchSetValue(body, "OCX_BUN"); + if (!bun) return null; + const cliAssignments = body.split(/\r?\n/).filter(line => /^\s*@?set\s+"?OCX_CLI=/i.test(line)); + if (cliAssignments.length > 0) { + const port = sourceLaunch.exec(launch)?.[1]; + return cliAssignments.length === 1 && Boolean(generatedBatchSetValue(body, "OCX_CLI")) + && Boolean(port) && Number(port) >= 1 && Number(port) <= 65535 + ? "source" : null; + } + const port = standaloneLaunch.exec(launch)?.[1]; + if (!port || Number(port) < 1 || Number(port) > 65535 || !/^@echo off\s*$/im.test(body) + || !/^setlocal EnableExtensions DisableDelayedExpansion\s*$/im.test(body) + || !new RegExp(`^if "%ERRORLEVEL%"=="${WINDOWS_WRAPPER_STAY_OUT_EXIT_CODE}" goto stopped\\s*$`, "im").test(body) + || !/^:stopped\s*$/im.test(body)) return null; + if (!matchesGeneratedStandaloneControlFlow(body, Number(port))) return null; + for (const [name, value] of [ + ["OCX_SERVICE", "1"], + [WINDOWS_WRAPPER_PROTOCOL_ENV, "1"], + [BUN_RUNTIME_SOURCE_ENV, "standalone"], + ]) { + if (generatedBatchSetValue(body, name) !== value) return null; + } + const runtimePath = generatedBatchSetValue(body, BUN_RUNTIME_PATH_ENV); + if (!bun || !runtimePath || normalizeWindowsPath(decodeBatchPathValue(bun)) !== normalizeWindowsPath(decodeBatchPathValue(runtimePath))) return null; + for (const name of ["CODEX_HOME", "OPENCODEX_HOME"]) { + const assignments = body.split(/\r?\n/).filter(line => new RegExp(`^\\s*@?set\\s+"?${name}=`, "i").test(line)); + if (assignments.length > 0 && generatedBatchSetValue(body, name) === null) return null; + } + return { standaloneBun: decodeBatchPathValue(bun) }; } function normalizeWindowsPath(value: string): string { @@ -711,7 +803,7 @@ function probeWinswRegistration( function inspectWindows( deps: Required> - & Pick, + & Pick, ): ServiceManagerInstallation { const configDir = windowsConfigDirPath(deps); const taskXmlPath = join(configDir, "opencodex-service-task.xml"); @@ -802,6 +894,10 @@ function inspectWindows( const registeredWalk = walkWindowsChain(deps, registration.registeredXml, taskXmlPath); if (registeredWalk.kind !== "present") return registeredWalk; const registeredClaim = registeredWalk.claims[0]; + const registeredLauncher = /"([^"]+)"/.exec(windowsTaskArguments(registration.registeredXml) ?? "")?.[1]; + if (!registeredLauncher || !windowsTaskActionMatches(registration.registeredXml, registeredLauncher)) { + return unknown("the registered scheduled-task action does not match the generated launcher action"); + } if (!homesEqual(registeredClaim.homes, stagedClaim.homes)) { return unknown("the registered scheduled task names different homes than the staged task definition"); } @@ -837,7 +933,7 @@ function homesEqual( * generated service-asset directory. */ function walkWindowsChain( - deps: Required> & Pick, + deps: Required> & Pick, xml: string, definitionPath: string, ): ServiceManagerInstallation { @@ -885,9 +981,21 @@ function walkWindowsChain( return unknown(`the launcher wrapper could not be read: ${String(error)}`); } - if (!wrapperLooksGenerated(wrapperBody)) { + const launchShape = wrapperLaunchShape(wrapperBody); + if (launchShape === null) { return unknown(`the launcher wrapper does not look like a generated opencodex service wrapper: ${wrapperPath}`); } + if (typeof launchShape !== "string") { + const executable = launchShape.standaloneBun; + const evidence = inspectServiceStateEvidence(deps.statePaths ?? serviceStatePathsForOpenCodexHome(configDir)); + const valid = evidence.filter(e => e.kind === "valid"); + if (!win32Path.isAbsolute(executable) || win32Path.extname(executable).toLowerCase() !== ".exe" + || valid.length === 0 || evidence.some(e => e.kind === "invalid" || e.kind === "unreadable") + || valid.some(e => e.state.version !== 2 || e.state.backend !== "scheduler" || e.state.cliPath !== null + || !e.state.bunPath || normalizeWindowsPath(e.state.bunPath) !== normalizeWindowsPath(executable))) { + return unknown("the standalone service wrapper executable is not bound to recorded scheduler install state"); + } + } const rawCodexHome = batchSetValue(wrapperBody, "CODEX_HOME"); const rawOpencodexHome = batchSetValue(wrapperBody, "OPENCODEX_HOME"); @@ -992,6 +1100,7 @@ export function inspectServiceManagerInstallation(deps: ProbeDeps = {}): Service winswStatus: deps.winswStatus, windowsLocale: deps.windowsLocale, windowsTaskListingCache: deps.windowsTaskListingCache, + statePaths: deps.statePaths, }); } return unknown(`no service manager probe for platform ${platform}`); diff --git a/src/service/windows-taskxml.ts b/src/service/windows-taskxml.ts index 795d020a670..8884d7410ba 100644 --- a/src/service/windows-taskxml.ts +++ b/src/service/windows-taskxml.ts @@ -427,6 +427,25 @@ function taskXmlDecodedLossyValueEquals(xml: string, tag: string, expected: stri return taskXmlLossyValueEquals(taskXmlDecodeEntities(value), expected); } +/** Accept only the single Exec action emitted for this launcher's task. */ +export function windowsTaskActionMatches( + xml: string, + launcher: string, + wscript = windowsWscript(), + allowLossyPaths = false, +): boolean { + const scrubbed = taskXmlWithoutCommentsAndCdata(xml); + if (taskXmlElementCount(scrubbed, "Data") > 0 || taskXmlHasPrefixedTag(scrubbed, "Data") + || taskXmlHasPrefixedTag(scrubbed, "Actions") || taskXmlHasPrefixedTag(scrubbed, "Exec") + || taskXmlElementCount(scrubbed, "Actions") !== 1 || taskXmlElementCount(scrubbed, "Exec") !== 1) return false; + const actions = taskXmlSection(scrubbed, "Actions"); + const exec = /]*)?>([\s\S]*?)<\/Exec>/i.exec(actions); + if (!exec || actions.replace(exec[0], "").trim() !== "") return false; + const equals = allowLossyPaths ? taskXmlDecodedLossyValueEquals : taskXmlDecodedValueEquals; + return equals(exec[1]!, "Command", wscript) + && equals(exec[1]!, "Arguments", `/b /nologo "${launcher}"`); +} + export function taskXmlOptionalValueEquals(xml: string, tag: string, expected: string): boolean { // Check the prefixed form first: treating `false` as an // omission would turn an explicitly disabled task into a healthy one. @@ -520,7 +539,6 @@ function windowsTaskRegistrationBaseHealthy( const trigger = taskXmlSection(triggers, "LogonTrigger"); const principal = taskXmlSection(scrubbed, "Principal"); const settings = taskXmlSection(scrubbed, "Settings"); - const action = taskXmlSection(scrubbed, "Exec"); // A self-closing leaves an empty section, so look for the element // itself — scoped to so a decoy elsewhere cannot satisfy it. return taskXmlElementCount(triggers, "LogonTrigger") > 0 @@ -538,11 +556,7 @@ function windowsTaskRegistrationBaseHealthy( // cannot carry when the profile is named outside the code page (#3064). Only // unrepresentable characters are forgiven; every ASCII segment and every // separator is still matched literally. - && (allowLossyPaths - ? taskXmlDecodedLossyValueEquals(action, "Command", wscript) - && taskXmlDecodedLossyValueEquals(action, "Arguments", `/b /nologo "${launcher}"`) - : taskXmlDecodedValueEquals(action, "Command", wscript) - && taskXmlDecodedValueEquals(action, "Arguments", `/b /nologo "${launcher}"`)); + && windowsTaskActionMatches(xml, launcher, wscript, allowLossyPaths); } /** Validate the security/lifecycle-critical fields of the registered scheduler task. */ diff --git a/structure/ops/service-and-sidecars.md b/structure/ops/service-and-sidecars.md index 375e0ed055a..1e8deddd60f 100644 --- a/structure/ops/service-and-sidecars.md +++ b/structure/ops/service-and-sidecars.md @@ -76,6 +76,20 @@ two naming different homes, and on macOS a logged-out user can have the plist on domain to query. The probe returns what it saw and does not decide ownership; callers such as `src/integrations/native/ownership-preflight.ts` compare the homes. Every command it runs is read-only and time-bounded, so it is safe while the proxy runs under that same manager. +On Windows, the generated-wrapper check accepts package installs that invoke the source CLI. +A standalone wrapper that invokes `start` directly must carry the generated protocol and runtime +markers, one quoted `OCX_BUN` assignment, and no `OCX_CLI` assignment in either quoting form. +Its executable lines and control-flow order must match the standalone script emitted by +`src/service/windows-taskxml.ts`; added jumps, exits, calls, labels, or commands make the probe unknown. +When Task Scheduler reports a registered task, the probe also requires its action to contain exactly +one Exec with the generated `wscript.exe` command and exact `/b /nologo` launcher arguments. +A foreign command or additional action makes ownership unknown even if the wrapper and homes agree. +Its executable must be absolute, end in `.exe`, and agree with `bunPath` in every readable service +state record for the scheduler backend with `cliPath: null`. Missing, malformed, or contradictory +state leaves the probe unknown; it cannot authorize unattended native Codex writes. +The state records a lexical executable path, not an install-time file identity or digest. A +retargeted junction or replacement at the same path is therefore outside this probe's evidence; +resolving the path only at probe time cannot establish which file the installer recorded. ## Stable service launcher (launchd and systemd) diff --git a/tests/codex-integration/codex-service-manager-probe-hardening.test.ts b/tests/codex-integration/codex-service-manager-probe-hardening.test.ts index a7ece534269..b209341df50 100644 --- a/tests/codex-integration/codex-service-manager-probe-hardening.test.ts +++ b/tests/codex-integration/codex-service-manager-probe-hardening.test.ts @@ -11,6 +11,7 @@ import { } from "../../src/service-manager-probe"; import { inspectNativeCodexOwnership } from "../../src/integrations/native/ownership-preflight"; import { setTrustedWindowsSystemDirectoryResolverForTests } from "../../src/lib/windows-elevation"; +import { windowsWscript } from "../../src/service/windows-scheduler"; import { getDefaultConfig } from "../../src/config"; import { startServer } from "../../src/server"; import { removeTreeWithRetry } from "../helpers/remove-tree"; @@ -74,6 +75,7 @@ function schedulerXml(launcherPath: string): string { "", " ", " ", + ` ${windowsWscript()}`, ` /b /nologo "${escaped}"`, " ", " ", @@ -99,7 +101,7 @@ function writeSchedulerChain( 'set "OCX_BUN=C:\\bun\\bun.exe"', 'set "OCX_CLI=C:\\opencodex\\src\\cli\\index.ts"', ":loop", - '"%OCX_BUN%" "%OCX_CLI%" start --port 10100', + '"%OCX_BUN%" "%OCX_CLI%" start --port 10100 >>"%OCX_SERVICE_LOG%" 2>&1', ].join("\r\n")); writeFileSync(launcher, `shell.Run """${wrapper}""", 0, True\r\n`); if (options.writeTaskXml !== false) writeFileSync(taskXml, schedulerXml(launcher)); diff --git a/tests/codex-integration/codex-service-manager-probe.test.ts b/tests/codex-integration/codex-service-manager-probe.test.ts index 6c2281d5ea5..237f1c4dbac 100644 --- a/tests/codex-integration/codex-service-manager-probe.test.ts +++ b/tests/codex-integration/codex-service-manager-probe.test.ts @@ -8,7 +8,7 @@ * - mutation-test the fixture's argv instead of the argv production emits */ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { existsSync, mkdirSync, mkdtempSync, symlinkSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -17,6 +17,8 @@ import { type ProbeRunner, type RawProbeRunner, } from "../../src/service-manager-probe"; +import { buildWindowsServiceScript, buildWindowsTaskXml } from "../../src/service/windows-taskxml"; +import { windowsWscript } from "../../src/service/windows-scheduler"; import { inspectNativeCodexOwnership } from "../../src/integrations/native/ownership-preflight"; import { setTrustedWindowsSystemDirectoryResolverForTests } from "../../src/lib/windows-elevation"; import { removeTreeWithRetry } from "../helpers/remove-tree"; @@ -345,7 +347,7 @@ describe("the Windows chain walk", () => { "", " ", " ", - ` C:\\WINDOWS\\System32\\wscript.exe`, + ` ${windowsWscript()}`, ` /b /nologo "${launcherPath.replace(/&/g, "&").replace(/"/g, """)}"`, " ", " ", @@ -377,12 +379,33 @@ describe("the Windows chain walk", () => { 'set "OCX_CLI=C:\\opencodex\\src\\cli\\index.ts"', ":loop", '>>"%OCX_SERVICE_LOG%" echo start', - '"%OCX_BUN%" "%OCX_CLI%" start --port 10100', + '"%OCX_BUN%" "%OCX_CLI%" start --port 10100 >>"%OCX_SERVICE_LOG%" 2>&1', ); writeFileSync(path, lines.join("\r\n")); return path; } + function writeStandaloneState(bunPath = "C:\\OpenCodex\\ocx.exe"): string { + const path = join(home, ".opencodex", "service-state.json"); + writeFileSync(path, JSON.stringify({ + version: 2, codexHome: "C:\\Users\\ws\\.codex", + opencodexHome: "C:\\Users\\ws\\.opencodex", + backend: "scheduler", bunPath, cliPath: null, + })); + return path; + } + + function writeStandaloneWrapper(bunPath = "C:\\OpenCodex\\ocx.exe"): string { + const dir = join(home, ".opencodex"); + const wrapper = join(dir, "opencodex-service.cmd"); + mkdirSync(dir, { recursive: true }); + writeFileSync(wrapper, buildWindowsServiceScript({ + bun: bunPath, bunRuntimeSource: "standalone", cli: null, + }, 10100)); + writeWindowsTask(writeWindowsLauncher(wrapper)); + return wrapper; + } + test("the full chain is walked and the homes are extracted", () => { const wrapper = writeWindowsWrapper("C:\\Users\\ws\\.codex", "C:\\Users\\ws\\.opencodex"); const launcher = writeWindowsLauncher(wrapper); @@ -409,6 +432,264 @@ describe("the Windows chain walk", () => { expect(calls[0].args).toEqual(["/query", "/tn", "opencodex-proxy", "/xml"]); }); + test("a production-generated standalone wrapper is accepted", () => { + writeStandaloneWrapper(); + const statePath = writeStandaloneState(); + const { runRaw } = recorder(() => ({ status: 1, stderr: "ERROR: The system cannot find the file specified." })); + + const result = inspectServiceManagerInstallation({ + platform: "win32", + home, + runRaw, + winswStatus: () => "nonexistent", + statePaths: [statePath], + }); + expect(result.kind).toBe("present"); + if (result.kind !== "present") return; + expect(result.claims).toHaveLength(1); + expect(result.claims[0].backend).toBe("scheduler"); + }); + + test("a registered production-generated standalone action establishes native ownership", () => { + process.env.CODEX_HOME = "C:\\Users\\ws\\.codex"; + process.env.OPENCODEX_HOME = "C:\\Users\\ws\\.opencodex"; + const wrapper = writeStandaloneWrapper(); + const statePath = writeStandaloneState(); + const launcher = join(home, ".opencodex", "opencodex-service-launcher.vbs"); + const registeredXml = buildWindowsTaskXml(wrapper, launcher, undefined, "S-1-5-21-123"); + const { runRaw } = recorder(() => ({ status: 0, stdout: registeredXml })); + expect(inspectNativeCodexOwnership({ + platform: "win32", home, runRaw, winswStatus: () => "nonexistent", statePaths: [statePath], + configDir: join(home, ".opencodex"), + currentHomes: { codexHome: process.env.CODEX_HOME, opencodexHome: process.env.OPENCODEX_HOME }, + }).ownership).toBe("owned"); + }); + + test.each([ + ["foreign command", (xml: string) => xml.replace(/[^<]*<\/Command>/, "C:\\foreign\\proxy.exe")], + ["extra Exec action", (xml: string) => xml.replace("", "C:\\foreign\\proxy.exe")], + ])("a registered standalone task with %s cannot establish native ownership", (_, mutate) => { + process.env.CODEX_HOME = "C:\\Users\\ws\\.codex"; + process.env.OPENCODEX_HOME = "C:\\Users\\ws\\.opencodex"; + const wrapper = writeStandaloneWrapper(); + const statePath = writeStandaloneState(); + const launcher = join(home, ".opencodex", "opencodex-service-launcher.vbs"); + const registeredXml = mutate(buildWindowsTaskXml(wrapper, launcher, undefined, "S-1-5-21-123")); + const { runRaw } = recorder(() => ({ status: 0, stdout: registeredXml })); + const deps = { + platform: "win32" as const, home, runRaw, winswStatus: () => "nonexistent" as const, + statePaths: [statePath], configDir: join(home, ".opencodex"), + currentHomes: { codexHome: process.env.CODEX_HOME, opencodexHome: process.env.OPENCODEX_HOME }, + }; + expect(inspectServiceManagerInstallation(deps).kind).toBe("unknown"); + expect(inspectNativeCodexOwnership(deps).ownership).toBe("unknown"); + }); + + test.each([ + "goto stopped", + "exit /b 0", + "call C:\\foreign\\proxy.cmd", + ":foreign", + "echo foreign", + ])("a standalone wrapper with an inserted control line is unknown: %s", inserted => { + const wrapper = writeStandaloneWrapper(); + const statePath = writeStandaloneState(); + writeFileSync(wrapper, readFileSync(wrapper, "utf8").replace( + '"%OCX_BUN%" start --port 10100', + `${inserted}\r\n"%OCX_BUN%" start --port 10100`, + )); + const { runRaw } = recorder(() => ({ status: 1, stderr: "ERROR: The system cannot find the file specified." })); + expect(inspectServiceManagerInstallation({ + platform: "win32", home, runRaw, winswStatus: () => "nonexistent", statePaths: [statePath], + }).kind).toBe("unknown"); + }); + + test("an unreachable standalone launch cannot establish native ownership", () => { + process.env.CODEX_HOME = "C:\\Users\\ws\\.codex"; + process.env.OPENCODEX_HOME = "C:\\Users\\ws\\.opencodex"; + const wrapper = writeStandaloneWrapper(); + const statePath = writeStandaloneState(); + const { runRaw } = recorder(() => ({ status: 1, stderr: "ERROR: The system cannot find the file specified." })); + const deps = { + platform: "win32" as const, home, runRaw, winswStatus: () => "nonexistent" as const, + statePaths: [statePath], configDir: join(home, ".opencodex"), + currentHomes: { codexHome: process.env.CODEX_HOME, opencodexHome: process.env.OPENCODEX_HOME }, + }; + expect(inspectNativeCodexOwnership(deps).ownership).toBe("owned"); + writeFileSync(wrapper, readFileSync(wrapper, "utf8").replace( + '"%OCX_BUN%" start --port 10100', + 'goto stopped\r\n"%OCX_BUN%" start --port 10100', + )); + expect(inspectNativeCodexOwnership(deps).ownership).toBe("unknown"); + }); + + test("a production-generated source wrapper remains accepted", () => { + const dir = join(home, ".opencodex"); + const wrapper = join(dir, "opencodex-service.cmd"); + mkdirSync(dir, { recursive: true }); + writeFileSync(wrapper, buildWindowsServiceScript({ + bun: "C:\\OpenCodex\\bun.exe", bunRuntimeSource: "bundled", + cli: "C:\\OpenCodex\\src\\cli\\index.ts", + }, 10100)); + writeWindowsTask(writeWindowsLauncher(wrapper)); + const { runRaw } = recorder(() => ({ status: 1, stderr: "ERROR: The system cannot find the file specified." })); + expect(inspectServiceManagerInstallation({ + platform: "win32", home, runRaw, winswStatus: () => "nonexistent", + }).kind).toBe("present"); + }); + + test("a direct launch without an OCX_BUN assignment is unknown", () => { + const wrapper = writeStandaloneWrapper(); + const statePath = writeStandaloneState(); + writeFileSync(wrapper, readFileSync(wrapper, "utf8").replace(/^set "OCX_BUN=.*"\r?\n/im, "")); + const { runRaw } = recorder(() => ({ status: 1, stderr: "ERROR: The system cannot find the file specified." })); + expect(inspectServiceManagerInstallation({ + platform: "win32", home, runRaw, winswStatus: () => "nonexistent", statePaths: [statePath], + }).kind).toBe("unknown"); + }); + + test.each([ + ['set "OCX_WINDOWS_WRAPPER_PROTOCOL=1"\r\n', ""], + ['set "OCX_BUN_RUNTIME_SOURCE=standalone"', 'set "OCX_BUN_RUNTIME_SOURCE=bundled"'], + ['set "OCX_BUN=C:\\OpenCodex\\ocx.exe"', 'set OCX_BUN=C:\\OpenCodex\\ocx.exe'], + ])("a direct launch with a missing generator marker is unknown: %s", (original, replacement) => { + const wrapper = writeStandaloneWrapper(); + const statePath = writeStandaloneState(); + writeFileSync(wrapper, readFileSync(wrapper, "utf8").replace(original, replacement)); + const { runRaw } = recorder(() => ({ status: 1, stderr: "ERROR: The system cannot find the file specified." })); + expect(inspectServiceManagerInstallation({ + platform: "win32", home, runRaw, winswStatus: () => "nonexistent", statePaths: [statePath], + }).kind).toBe("unknown"); + }); + + test("a direct launch without independent install state is unknown", () => { + writeStandaloneWrapper(); + const { runRaw } = recorder(() => ({ status: 1, stderr: "ERROR: The system cannot find the file specified." })); + expect(inspectServiceManagerInstallation({ + platform: "win32", home, runRaw, winswStatus: () => "nonexistent", + statePaths: [join(home, ".opencodex", "service-state.json")], + }).kind).toBe("unknown"); + }); + + test.each(['set OCX_CLI=C:\\foreign\\index.ts', '@set "OCX_CLI=C:\\foreign\\index.ts"', 'set "OCX_CLI=C:\\foreign\\index.ts"\r\nset OCX_CLI=C:\\other\\index.ts'])( + "a direct launch with a mixed or unquoted CLI assignment is unknown: %s", assignment => { + const wrapper = writeStandaloneWrapper(); + const statePath = writeStandaloneState(); + writeFileSync(wrapper, readFileSync(wrapper, "utf8").replace(":loop", `${assignment}\r\n:loop`)); + const { runRaw } = recorder(() => ({ status: 1, stderr: "ERROR: The system cannot find the file specified." })); + expect(inspectServiceManagerInstallation({ + platform: "win32", home, runRaw, winswStatus: () => "nonexistent", statePaths: [statePath], + }).kind).toBe("unknown"); + }, + ); + + test("a direct launch with an extra shell command is unknown", () => { + const wrapper = writeStandaloneWrapper(); + const statePath = writeStandaloneState(); + writeFileSync(wrapper, readFileSync(wrapper, "utf8").replace( + '"%OCX_BUN%" start --port 10100 >>"%OCX_SERVICE_LOG%" 2>&1', + '"%OCX_BUN%" start --port 10100 & echo foreign', + )); + const { runRaw } = recorder(() => ({ status: 1, stderr: "ERROR: The system cannot find the file specified." })); + expect(inspectServiceManagerInstallation({ + platform: "win32", home, runRaw, winswStatus: () => "nonexistent", statePaths: [statePath], + }).kind).toBe("unknown"); + }); + + test("a foreign direct-launch executable stays unknown despite valid scheduler state", () => { + writeStandaloneWrapper("C:\\foreign\\proxy.exe"); + const statePath = writeStandaloneState(); + const { runRaw } = recorder(() => ({ status: 1, stderr: "ERROR: The system cannot find the file specified." })); + const deps = { + platform: "win32" as const, home, runRaw, winswStatus: () => "nonexistent" as const, + statePaths: [statePath], configDir: join(home, ".opencodex"), + }; + expect(inspectServiceManagerInstallation(deps).kind).toBe("unknown"); + expect(inspectNativeCodexOwnership({ + ...deps, currentHomes: { codexHome: "C:\\Users\\ws\\.codex", opencodexHome: "C:\\Users\\ws\\.opencodex" }, + }).ownership).toBe("unknown"); + }); + + test("a source wrapper that drops its CLI argument is unknown", () => { + const wrapper = writeWindowsWrapper("C:\\a\\.codex", "C:\\a\\.opencodex"); + writeFileSync(wrapper, readFileSync(wrapper, "utf8").replace( + '"%OCX_BUN%" "%OCX_CLI%" start', + '"%OCX_BUN%" start', + )); + const launcher = writeWindowsLauncher(wrapper); + writeWindowsTask(launcher); + const { runRaw } = recorder(() => ({ status: 1, stderr: "ERROR: The system cannot find the file specified." })); + + expect(inspectServiceManagerInstallation({ + platform: "win32", + home, + runRaw, + winswStatus: () => "nonexistent", + }).kind).toBe("unknown"); + }); + + test("a source wrapper with an extra standalone launch is unknown", () => { + const wrapper = writeWindowsWrapper("C:\\a\\.codex", "C:\\a\\.opencodex"); + writeFileSync(wrapper, readFileSync(wrapper, "utf8").replace( + '"%OCX_BUN%" "%OCX_CLI%" start --port 10100 >>"%OCX_SERVICE_LOG%" 2>&1', + '"%OCX_BUN%" "%OCX_CLI%" start --port 10100\r\n"%OCX_BUN%" start --port 10100', + )); + const launcher = writeWindowsLauncher(wrapper); + writeWindowsTask(launcher); + const { runRaw } = recorder(() => ({ status: 1, stderr: "ERROR: The system cannot find the file specified." })); + + expect(inspectServiceManagerInstallation({ + platform: "win32", home, runRaw, winswStatus: () => "nonexistent", + }).kind).toBe("unknown"); + }); + + test("a source wrapper with an appended shell command is unknown", () => { + const wrapper = writeWindowsWrapper("C:\\a\\.codex", "C:\\a\\.opencodex"); + writeFileSync(wrapper, readFileSync(wrapper, "utf8").replace( + '"%OCX_BUN%" "%OCX_CLI%" start --port 10100 >>"%OCX_SERVICE_LOG%" 2>&1', + '"%OCX_BUN%" "%OCX_CLI%" start --port 10100 >>"%OCX_SERVICE_LOG%" 2>&1 & echo foreign', + )); + const launcher = writeWindowsLauncher(wrapper); + writeWindowsTask(launcher); + const { runRaw } = recorder(() => ({ status: 1, stderr: "ERROR: The system cannot find the file specified." })); + + expect(inspectServiceManagerInstallation({ + platform: "win32", home, runRaw, winswStatus: () => "nonexistent", + }).kind).toBe("unknown"); + }); + + test("a source wrapper without an OCX_BUN assignment is unknown", () => { + const wrapper = writeWindowsWrapper("C:\\a\\.codex", "C:\\a\\.opencodex"); + writeFileSync(wrapper, readFileSync(wrapper, "utf8").replace(/^set "OCX_BUN=.*"\r?\n/im, "")); + const launcher = writeWindowsLauncher(wrapper); + writeWindowsTask(launcher); + const { runRaw } = recorder(() => ({ status: 1, stderr: "ERROR: The system cannot find the file specified." })); + + expect(inspectServiceManagerInstallation({ + platform: "win32", home, runRaw, winswStatus: () => "nonexistent", + }).kind).toBe("unknown"); + }); + + test("a standalone wrapper with an extra source launch is unknown", () => { + const dir = join(home, ".opencodex"); + const wrapper = join(dir, "opencodex-service.cmd"); + mkdirSync(dir, { recursive: true }); + writeFileSync(wrapper, buildWindowsServiceScript({ + bun: "C:\\OpenCodex\\ocx.exe", bunRuntimeSource: "standalone", cli: null, + }, 10100).replace( + '"%OCX_BUN%" start --port 10100', + '"%OCX_BUN%" "%OCX_CLI%" start --port 10100\r\n"%OCX_BUN%" start --port 10100', + )); + const statePath = writeStandaloneState(); + const launcher = writeWindowsLauncher(wrapper); + writeWindowsTask(launcher); + const { runRaw } = recorder(() => ({ status: 1, stderr: "ERROR: The system cannot find the file specified." })); + + expect(inspectServiceManagerInstallation({ + platform: "win32", home, runRaw, winswStatus: () => "nonexistent", statePaths: [statePath], + }).kind).toBe("unknown"); + }); + test("a registered task whose chain disagrees with the staged definition is unknown", () => { const wrapper = writeWindowsWrapper("C:\\Users\\ws\\.codex", "C:\\Users\\ws\\.opencodex"); const launcher = writeWindowsLauncher(wrapper); @@ -423,14 +704,11 @@ describe("the Windows chain walk", () => { 'set "OCX_BUN=C:\\bun\\bun.exe"', 'set "OCX_CLI=C:\\opencodex\\src\\cli\\index.ts"', ":loop", - '"%OCX_BUN%" "%OCX_CLI%" start --port 10100', + '"%OCX_BUN%" "%OCX_CLI%" start --port 10100 >>"%OCX_SERVICE_LOG%" 2>&1', ].join("\r\n")); const foreignLauncher = join(home, ".opencodex", "foreign-launcher.vbs"); writeFileSync(foreignLauncher, `shell.Run """${foreignWrapper}""", 0, True\r\n`); - const registeredXml = [ - '', - `/b /nologo "${foreignLauncher}"`, - ].join("\n"); + const registeredXml = windowsTaskXmlFor(foreignLauncher); const { runRaw } = recorder(() => ({ status: 0, stdout: registeredXml })); const result = inspectServiceManagerInstallation({ platform: "win32", home, runRaw, winswStatus: () => "nonexistent" }); @@ -451,7 +729,7 @@ describe("the Windows chain walk", () => { 'set "OCX_BUN=C:\\bun\\bun.exe"', 'set "OCX_CLI=C:\\opencodex\\src\\cli\\index.ts"', ":loop", - '"%OCX_BUN%" "%OCX_CLI%" start --port 10100', + '"%OCX_BUN%" "%OCX_CLI%" start --port 10100 >>"%OCX_SERVICE_LOG%" 2>&1', ].join("\r\n")); const launcher = writeWindowsLauncher(wrapper); writeWindowsTask(launcher); @@ -478,7 +756,7 @@ describe("the Windows chain walk", () => { 'set "OCX_BUN=C:\\bun\\bun.exe"', 'set "OCX_CLI=C:\\opencodex\\src\\cli\\index.ts"', ":loop", - '"%OCX_BUN%" "%OCX_CLI%" start --port 10100', + '"%OCX_BUN%" "%OCX_CLI%" start --port 10100 >>"%OCX_SERVICE_LOG%" 2>&1', ].join("\r\n")); const launcher = writeWindowsLauncher(wrapper); writeWindowsTask(launcher); @@ -530,7 +808,7 @@ describe("the Windows chain walk", () => { 'set "OCX_BUN=C:\\bun\\bun.exe"', 'set "OCX_CLI=C:\\opencodex\\src\\cli\\index.ts"', ":loop", - '"%OCX_BUN%" "%OCX_CLI%" start --port 10100', + '"%OCX_BUN%" "%OCX_CLI%" start --port 10100 >>"%OCX_SERVICE_LOG%" 2>&1', ].join("\r\n")); const launcher = join(custom, "opencodex-service-launcher.vbs"); writeFileSync(launcher, `shell.Run """${wrapper}""", 0, True\r\n`); @@ -1009,7 +1287,7 @@ describe("ownership refuses what it cannot prove", () => { 'set "OCX_BUN=C:\\bun\\bun.exe"', 'set "OCX_CLI=C:\\opencodex\\src\\cli\\index.ts"', ":loop", - '"%OCX_BUN%" "%OCX_CLI%" start --port 10100', + '"%OCX_BUN%" "%OCX_CLI%" start --port 10100 >>"%OCX_SERVICE_LOG%" 2>&1', ].join("\r\n")); const launcher = join(opencodexHome, "opencodex-service-launcher.vbs"); writeFileSync(launcher, `shell.Run """${wrapper}""", 0, True\r\n`); @@ -1045,7 +1323,7 @@ describe("ownership refuses what it cannot prove", () => { 'set "OCX_BUN=C:\\bun\\bun.exe"', 'set "OCX_CLI=C:\\opencodex\\src\\cli\\index.ts"', ":loop", - '"%OCX_BUN%" "%OCX_CLI%" start --port 10100', + '"%OCX_BUN%" "%OCX_CLI%" start --port 10100 >>"%OCX_SERVICE_LOG%" 2>&1', ].join("\r\n")); const launcher = join(opencodexHome, "opencodex-service-launcher.vbs"); writeFileSync(launcher, `shell.Run """${wrapper}""", 0, True\r\n`); From eafa6bcf0812a9a240c9686947bf5639c989359d Mon Sep 17 00:00:00 2001 From: ingwannu Date: Wed, 30 Sep 2026 05:36:06 +0900 Subject: [PATCH 22/26] feat(oauth): add per-account Anthropic usage thresholds (#6207) * feat(oauth): add per-account Anthropic usage thresholds * fix(oauth): preserve manual Anthropic threshold intent * test(cli): update Anthropic auto-switch rejection * fix(oauth): fence Anthropic threshold policy ownership * fix(oauth): return committed Anthropic threshold policy * fix(gui): reconcile Anthropic threshold state safely * fix(gui): satisfy threshold lifecycle lint --------- Co-authored-by: Ingwannu Co-authored-by: JUN --- .../fr/reference/cli/providers-accounts.md | 6 +- .../docs/fr/reference/management-api.md | 10 + .../ja/reference/cli/providers-accounts.md | 6 +- .../docs/ja/reference/management-api.md | 10 + .../ko/reference/cli/providers-accounts.md | 6 +- .../docs/ko/reference/management-api.md | 10 + .../docs/reference/cli/providers-accounts.md | 6 +- .../content/docs/reference/management-api.md | 10 + .../ru/reference/cli/providers-accounts.md | 6 +- .../docs/ru/reference/management-api.md | 10 + .../tr/reference/cli/providers-accounts.md | 6 +- .../docs/tr/reference/management-api.md | 10 + .../zh-cn/reference/cli/providers-accounts.md | 6 +- .../docs/zh-cn/reference/management-api.md | 10 + .../zh-tw/reference/cli/providers-accounts.md | 6 +- .../docs/zh-tw/reference/management-api.md | 10 + .../components/AccountAutoSwitchControl.tsx | 4 +- .../AnthropicAccountPoolSettings.tsx | 45 ++- .../provider-workspace/ProviderAuthPanel.tsx | 18 +- .../components/provider-workspace/types.ts | 5 + gui/src/hooks/useProviderAccountPools.ts | 79 ++++- gui/src/i18n/de.ts | 1 + gui/src/i18n/en.ts | 1 + gui/src/i18n/fr.ts | 1 + gui/src/i18n/ja.ts | 1 + gui/src/i18n/ko.ts | 1 + gui/src/i18n/ru.ts | 1 + gui/src/i18n/tr.ts | 1 + gui/src/i18n/vi.ts | 1 + gui/src/i18n/zh-TW.ts | 1 + gui/src/i18n/zh.ts | 1 + gui/src/pages/Providers.tsx | 4 +- .../anthropic-pool-quota-window.test.tsx | 41 ++- .../provider-account-pause-refresh.test.tsx | 87 ++++- .../provider-quota-refresh-controls.test.tsx | 17 + scripts/test-layout/layout.json | 2 + .../ocx/references/01_management_surface.md | 4 + src/cli/account-anthropic-threshold.ts | 41 +++ src/cli/account-api.ts | 2 + src/cli/account-extended.ts | 3 + src/cli/account.ts | 1 + src/cli/capabilities.ts | 6 +- src/lib/account-selection-events.ts | 28 ++ src/oauth/anthropic-account-threshold.ts | 13 + src/oauth/anthropic-routing.ts | 43 ++- src/oauth/store.ts | 44 ++- src/oauth/types.ts | 2 + .../management/anthropic-account-threshold.ts | 27 ++ src/server/management/oauth-account-routes.ts | 6 + src/server/management/route-registry.ts | 1 + src/server/responses/request-transport.ts | 10 +- structure/INDEX.md | 3 +- .../ADR-6014-anthropic-account-threshold.md | 49 +++ structure/gui-and-management-api.md | 12 + structure/manifest.json | 7 + structure/providers-and-adapters.md | 1 + .../providers/anthropic-account-thresholds.md | 31 ++ .../anthropic-account-threshold.test.ts | 312 ++++++++++++++++++ .../anthropic/anthropic-model-routes.test.ts | 23 +- tests/cli/cli-account.test.ts | 8 +- .../cli-anthropic-account-threshold.test.ts | 58 ++++ tests/fixtures/test-layout-expected.json | 2 + 62 files changed, 1136 insertions(+), 41 deletions(-) create mode 100644 src/cli/account-anthropic-threshold.ts create mode 100644 src/oauth/anthropic-account-threshold.ts create mode 100644 src/server/management/anthropic-account-threshold.ts create mode 100644 structure/decisions/ADR-6014-anthropic-account-threshold.md create mode 100644 structure/providers/anthropic-account-thresholds.md create mode 100644 tests/adapters/anthropic/anthropic-account-threshold.test.ts create mode 100644 tests/cli/cli-anthropic-account-threshold.test.ts diff --git a/docs-site/src/content/docs/fr/reference/cli/providers-accounts.md b/docs-site/src/content/docs/fr/reference/cli/providers-accounts.md index 4a8824336de..3c43a780a24 100644 --- a/docs-site/src/content/docs/fr/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/fr/reference/cli/providers-accounts.md @@ -237,7 +237,11 @@ renvoient 1 ; une sonde de quota en amont qui échoue ou expire produit plutôt ### `ocx account auto-switch > [--json]` -Contrôle le seuil du pool Codex `openai`, ou enregistre celui d’un pool OAuth générique. `on` enregistre 80 %, `off` 0 % et `threshold ` accepte 0–100. Un seuil générique n’oriente la sélection que si `pool.kernel` est activé avec `strategy: "fill-first"` ; le drapeau désactivé, sa sauvegarde n’active pas le basculement par seuil. Dans les deux cas, elle ne change ni l’activation du fournisseur, ni la rotation réactive après une erreur 429. Pour les pools génériques, les sorties utilisent la réponse confirmée du serveur. Pour un pool générique, `poolEnabled` est le réglage enregistré (`null` signifie non spécifié), pas l’état effectif hérité. `inert: true` indique un seuil enregistré mais non appliqué, `inert: false` un seuil que le pool applique réellement. L’absence d’`inert` signale une capacité inconnue, qui ne produit jamais `enabled: true`. Les fournisseurs à clé API, Anthropic et les valeurs invalides sont refusés. +Contrôle le seuil du pool Codex `openai`, ou enregistre celui d’un pool OAuth générique. `on` enregistre 80 %, `off` 0 % et `threshold ` accepte 0–100. Un seuil générique n’oriente la sélection que si `pool.kernel` est activé avec `strategy: "fill-first"` ; le drapeau désactivé, sa sauvegarde n’active pas le basculement par seuil. Dans les deux cas, elle ne change ni l’activation du fournisseur, ni la rotation réactive après une erreur 429. Pour les pools génériques, les sorties utilisent la réponse confirmée du serveur. Pour un pool générique, `poolEnabled` est le réglage enregistré (`null` signifie non spécifié), pas l’état effectif hérité. `inert: true` indique un seuil enregistré mais non appliqué, `inert: false` un seuil que le pool applique réellement. L’absence d’`inert` signale une capacité inconnue, qui ne produit jamais `enabled: true`. Les fournisseurs à clé API et les valeurs invalides sont refusés. + +### `ocx account auto-switch anthropic … --account ` + +Pour Anthropic OAuth, `ocx account auto-switch anthropic threshold 90 --account ` enregistre un entier de 0 à 100. `off --account ` vaut 0, `on --account ` vaut 80, `inherit --account ` rétablit l’héritage et `status --account ` lit sans écrire ; `--json` est disponible. La carte propose le même réglage. Une valeur absente/null hérite de `anthropicAccountPool.autoSwitchThreshold` (80 par défaut) ; 0 désactive seulement le basculement selon l’utilisation de ce compte. Le réglage survit au redémarrage et à la reconnexion, et disparaît avec le compte. Les priorités manuelle/affinité, les replis en cas de quota inconnu ou de comptes épuisés et les routes de modèles restent inchangés. Les seuils sont inactifs si le pool est désactivé ; pause et reprise après 429 restent actives. ```text openai: { provider, autoSwitchThreshold: number, enabled: boolean } diff --git a/docs-site/src/content/docs/fr/reference/management-api.md b/docs-site/src/content/docs/fr/reference/management-api.md index 207e6a4c965..8f0e9a10215 100644 --- a/docs-site/src/content/docs/fr/reference/management-api.md +++ b/docs-site/src/content/docs/fr/reference/management-api.md @@ -412,3 +412,13 @@ L'accès HTTP direct est surtout utile aux intégrations qui exigent les contrat ## Sessions distantes et rotation des clés de données `POST /api/keys/rotate {id}` démarre un chevauchement de dix minutes et renvoie le nouveau secret une seule fois. `POST /api/keys/rotate/commit {id,rotationId}` valide; `DELETE /api/keys/rotate {id,rotationId}` annule. L'authentification de gestion est obligatoire et une clé de données ne suffit pas. `POST /api/session/logout` exige la `gui-session` courante, l'Origin correspondante et CSRF. Un jeton admin reçoit 403 et ne peut jamais créer une session de consentement. + +## Seuil d’utilisation par compte Anthropic + +`PUT /api/oauth/accounts/auto-switch` + +Anthropic OAuth uniquement. `{ provider: "anthropic", accountId, threshold }` : entier 0–100 ou null pour hériter ; champ absent invalide. Conservé au redémarrage, supprimé avec le compte. + +Le DTO inclut `autoSwitchThresholdOverride` (entier/null), `autoSwitchThreshold` (défaut du pool) et `effectiveAutoSwitchThreshold`. 0 désactive seulement le basculement selon l’utilisation ; pause et reprise après 429 restent actives. + +HTTP: 400 invalid/unsupported; 404 missing account; `oauth_mutation_busy` on lock contention. diff --git a/docs-site/src/content/docs/ja/reference/cli/providers-accounts.md b/docs-site/src/content/docs/ja/reference/cli/providers-accounts.md index 2d4b72755a6..a97def7a7c6 100644 --- a/docs-site/src/content/docs/ja/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/ja/reference/cli/providers-accounts.md @@ -175,7 +175,11 @@ OAuth プロバイダーと API キー プロバイダーの場合、これに ### `ocx account auto-switch > [--json]` -`openai` Codex プールのしきい値を制御するか、汎用 OAuth プールのしきい値を保存します。`on` は 80%、`off` は 0%、`threshold ` は 0–100 を保存します。汎用プールのしきい値は `pool.kernel` が有効で `strategy: "fill-first"` の場合にのみ選択へ反映されます。フラグが無効なら、保存してもしきい値による切り替えは有効になりません。いずれの場合もプロバイダーの有効化設定と 429 エラー時のローテーションは変更されません。汎用プールの照会と変更の結果はサーバーの確認値を使用します。汎用プールの `poolEnabled` は保存された設定で、`null` は未指定です。継承後の実効状態ではありません。`inert: true` は保存済みで未適用、`inert: false` はプールが適用中であることを示します。`inert` が無い場合は機能が不明であり、その場合も `enabled: true` とは表示しません。API キープロバイダー、Anthropic、不正な値は拒否されます。 +`openai` Codex プールのしきい値を制御するか、汎用 OAuth プールのしきい値を保存します。`on` は 80%、`off` は 0%、`threshold ` は 0–100 を保存します。汎用プールのしきい値は `pool.kernel` が有効で `strategy: "fill-first"` の場合にのみ選択へ反映されます。フラグが無効なら、保存してもしきい値による切り替えは有効になりません。いずれの場合もプロバイダーの有効化設定と 429 エラー時のローテーションは変更されません。汎用プールの照会と変更の結果はサーバーの確認値を使用します。汎用プールの `poolEnabled` は保存された設定で、`null` は未指定です。継承後の実効状態ではありません。`inert: true` は保存済みで未適用、`inert: false` はプールが適用中であることを示します。`inert` が無い場合は機能が不明であり、その場合も `enabled: true` とは表示しません。API キープロバイダー、不正な値は拒否されます。 + +### `ocx account auto-switch anthropic … --account ` + +Anthropic OAuth では `ocx account auto-switch anthropic threshold 90 --account ` でアカウント別の整数 0–100 を保存します。`off --account ` は 0、`on --account ` は 80、`inherit --account ` は継承へ戻し、`status --account ` は読み取り専用です。`--json` も使えます。カードにも同じカスタム設定があります。未設定/null は `anthropicAccountPool.autoSwitchThreshold`（既定 80）を継承し、0 はそのアカウントの使用量による切り替えのみ無効にします。再起動・再ログインで保持され、削除時に消えます。手動選択、affinity、使用量不明・全候補消耗時のフォールバック、モデルルート制限は維持されます。プール無効時は適用されず、一時停止と 429 復旧は引き続き有効です。 ```text openai: { provider, autoSwitchThreshold: number, enabled: boolean } diff --git a/docs-site/src/content/docs/ja/reference/management-api.md b/docs-site/src/content/docs/ja/reference/management-api.md index 3fee2a3a801..cfdb48f8e77 100644 --- a/docs-site/src/content/docs/ja/reference/management-api.md +++ b/docs-site/src/content/docs/ja/reference/management-api.md @@ -354,3 +354,13 @@ account の selector binding は残るため、欠落中の exact route は fail ## リモートセッションとデータキー更新 `POST /api/keys/rotate {id}` は10分間の移行を開始し、新しい秘密値を一度だけ返します。`POST /api/keys/rotate/commit {id,rotationId}` で確定し、`DELETE /api/keys/rotate {id,rotationId}` で中止します。管理認証が必須で、データキーからは呼べません。`POST /api/session/logout` には現在の `gui-session`、一致する Origin、CSRF が必要です。管理トークンは 403 となり、同意セッションを作成できません。 + +## Anthropic アカウント使用量しきい値 + +`PUT /api/oauth/accounts/auto-switch` + +Anthropic OAuth のみ。`{ provider: "anthropic", accountId, threshold }`: 整数 0–100、null は継承、欠落はエラー。再起動後も保持され、アカウント削除時に消えます。 + +DTO は `autoSwitchThresholdOverride`（整数/null）、`autoSwitchThreshold`（プール既定値）、`effectiveAutoSwitchThreshold` を含みます。0 は使用量による切り替えのみ無効にし、一時停止と 429 復旧は維持します。 + +HTTP: 400 invalid/unsupported; 404 missing account; `oauth_mutation_busy` on lock contention. diff --git a/docs-site/src/content/docs/ko/reference/cli/providers-accounts.md b/docs-site/src/content/docs/ko/reference/cli/providers-accounts.md index 856390db88c..b87fb2dd9c6 100644 --- a/docs-site/src/content/docs/ko/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/ko/reference/cli/providers-accounts.md @@ -270,7 +270,11 @@ OAuth 및 API 키 제공자에는 제공자의 할당량 보고 엔드포인트 ### `ocx account auto-switch > [--json]` -`openai` Codex 풀의 임계값을 제어하거나 일반 OAuth 풀의 임계값을 저장합니다. `on`은 80%, `off`는 0%, `threshold `은 0–100을 저장합니다. 일반 풀의 임계값은 `pool.kernel`이 켜져 있고 `strategy: "fill-first"`일 때만 선택에 반영됩니다. 플래그가 꺼져 있으면 저장해도 임계값 기반 전환이 켜지지 않습니다. 어느 쪽이든 제공자 활성화 설정은 바뀌지 않고, 429 오류에 따른 회전도 비활성화되지 않습니다. 일반 풀의 조회와 변경 결과는 서버가 확인한 값을 사용합니다. 일반 풀의 `poolEnabled`는 저장된 제공자별 설정이며 `null`은 미지정입니다. 전역 설정을 상속한 실제 상태를 뜻하지 않습니다. `inert: true`는 임계값이 저장만 되고 적용되지 않는 상태, `inert: false`는 풀이 실제로 적용하고 있는 상태를 뜻합니다. `inert`가 아예 없으면 기능 지원을 알 수 없는 경우이며, 이때도 `enabled: true`로 표시하지 않습니다. API 키 제공자, Anthropic 및 잘못된 값은 거부합니다. +`openai` Codex 풀의 임계값을 제어하거나 일반 OAuth 풀의 임계값을 저장합니다. `on`은 80%, `off`는 0%, `threshold `은 0–100을 저장합니다. 일반 풀의 임계값은 `pool.kernel`이 켜져 있고 `strategy: "fill-first"`일 때만 선택에 반영됩니다. 플래그가 꺼져 있으면 저장해도 임계값 기반 전환이 켜지지 않습니다. 어느 쪽이든 제공자 활성화 설정은 바뀌지 않고, 429 오류에 따른 회전도 비활성화되지 않습니다. 일반 풀의 조회와 변경 결과는 서버가 확인한 값을 사용합니다. 일반 풀의 `poolEnabled`는 저장된 제공자별 설정이며 `null`은 미지정입니다. 전역 설정을 상속한 실제 상태를 뜻하지 않습니다. `inert: true`는 임계값이 저장만 되고 적용되지 않는 상태, `inert: false`는 풀이 실제로 적용하고 있는 상태를 뜻합니다. `inert`가 아예 없으면 기능 지원을 알 수 없는 경우이며, 이때도 `enabled: true`로 표시하지 않습니다. API 키 제공자 및 잘못된 값은 거부합니다. + +### `ocx account auto-switch anthropic … --account ` + +Anthropic OAuth는 `ocx account auto-switch anthropic threshold 90 --account `로 계정별 정수 0–100을 저장합니다. `off --account `는 0, `on --account `는 80, `inherit --account `는 상속 복원, `status --account `는 조회입니다. `--json`도 지원합니다. 계정 카드에서 같은 사용자 지정 임계값을 편집합니다. 미설정/null은 풀 기본값 `anthropicAccountPool.autoSwitchThreshold`(기본 80)를 상속하고, 0은 해당 계정의 사용량 기반 전환만 끕니다. 재시작·재로그인 후에도 유지되고 계정 삭제 시 제거됩니다. 수동 선택, 세션 affinity, 사용량 미확인·전체 소진 시 fallback, 모델 경로 제한은 유지됩니다. 풀이 꺼져 있으면 임계값은 적용되지 않으며 pause와 429 복구는 계속 동작합니다. ```text openai: { provider, autoSwitchThreshold: number, enabled: boolean } diff --git a/docs-site/src/content/docs/ko/reference/management-api.md b/docs-site/src/content/docs/ko/reference/management-api.md index 1a884ba64d4..a38309cb2cc 100644 --- a/docs-site/src/content/docs/ko/reference/management-api.md +++ b/docs-site/src/content/docs/ko/reference/management-api.md @@ -380,3 +380,13 @@ account의 selector binding은 남아 있어 계정이 없을 때 exact route가 ## 원격 세션과 데이터 키 교체 `POST /api/keys/rotate {id}`는 최대 10분의 전환을 시작하며 새 데이터 키를 한 번만 반환합니다. `POST /api/keys/rotate/commit {id,rotationId}`는 확정하고, `DELETE /api/keys/rotate {id,rotationId}`는 취소합니다. 모두 관리 인증이 필요하며 데이터 키로 호출할 수 없습니다. `POST /api/session/logout`은 현재 `gui-session`, 일치하는 Origin, CSRF가 필요합니다. 관리자 토큰은 403을 받고 동의 세션을 만들거나 교환할 수 없습니다. + +## Anthropic 계정 사용량 임계값 + +`PUT /api/oauth/accounts/auto-switch` + +Anthropic OAuth 전용. `{ provider: "anthropic", accountId, threshold }`: 정수 0–100, null은 상속, 누락은 오류. 재시작 후 유지되고 계정 삭제 시 제거됩니다. + +계정 DTO는 `autoSwitchThresholdOverride`(정수/null), `autoSwitchThreshold`(풀 기본값), `effectiveAutoSwitchThreshold`를 포함합니다. 0은 사용량 전환만 끄며 pause·429 복구는 유지합니다. + +HTTP: 400 invalid/unsupported; 404 missing account; `oauth_mutation_busy` on lock contention. diff --git a/docs-site/src/content/docs/reference/cli/providers-accounts.md b/docs-site/src/content/docs/reference/cli/providers-accounts.md index 02edd827e13..738fec91484 100644 --- a/docs-site/src/content/docs/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/reference/cli/providers-accounts.md @@ -504,7 +504,11 @@ instead (exit 0), matching the dashboard's quota bars. ### `ocx account auto-switch > [--json]` -Controls the `openai` Codex pool threshold, or stores a threshold for a generic OAuth pool. `on` stores 80%, `off` stores 0%, and `threshold ` accepts 0–100. A generic pool threshold steers selection only while `pool.kernel` is on with `strategy: "fill-first"`; with the flag off, saving one does not enable threshold-based switching. It never changes the provider enablement override or disables reactive 429 rotation. `status` and mutation output for generic pools use the confirmed server response. For generic pools, `poolEnabled` is the stored provider override (`null` means unspecified), not inherited effective state; `inert: true` means the threshold is stored but not applied, `inert: false` means the pool is applying it, and an absent `inert` is an unknown capability, which never reports `enabled: true`. API-key providers, Anthropic and invalid values are rejected. +Controls the `openai` Codex pool threshold, or stores a threshold for a generic OAuth pool. `on` stores 80%, `off` stores 0%, and `threshold ` accepts 0–100. A generic pool threshold steers selection only while `pool.kernel` is on with `strategy: "fill-first"`; with the flag off, saving one does not enable threshold-based switching. It never changes the provider enablement override or disables reactive 429 rotation. `status` and mutation output for generic pools use the confirmed server response. For generic pools, `poolEnabled` is the stored provider override (`null` means unspecified), not inherited effective state; `inert: true` means the threshold is stored but not applied, `inert: false` means the pool is applying it, and an absent `inert` is an unknown capability, which never reports `enabled: true`. API-key providers and invalid values are rejected. + +### `ocx account auto-switch anthropic … --account ` + +For Anthropic OAuth, use `ocx account auto-switch anthropic threshold 90 --account ` (integer 0–100), `off --account ` (0), `on --account ` (80), `inherit --account ` (reset), or `status --account ` (read-only); append `--json` for structured output. The account card offers the same custom-threshold toggle. Missing/null inherits `anthropicAccountPool.autoSwitchThreshold` (default 80); 0 disables usage-driven switching for that account, not pause or reactive 429 recovery. Overrides survive restart and re-login and are removed with the account. With pooling enabled, quota and fill-first compare each source/candidate against its own threshold in the selected quota window. Manual/affinity precedence, identity-less round-robin/fill-first behavior, unknown-quota fallback and all-drained fallback remain unchanged. Round-robin is not usage-driven; disabled pools ignore these thresholds. Model-route allowlists still constrain every candidate. ```text openai: { provider, autoSwitchThreshold: number, enabled: boolean } diff --git a/docs-site/src/content/docs/reference/management-api.md b/docs-site/src/content/docs/reference/management-api.md index 3c60b3505db..41e40d28302 100644 --- a/docs-site/src/content/docs/reference/management-api.md +++ b/docs-site/src/content/docs/reference/management-api.md @@ -744,3 +744,13 @@ Direct HTTP is most useful for integrations that need the exact endpoint contrac ## Remote sessions and data-key rotation `POST /api/keys/rotate {id}` starts a ten-minute overlap and returns the new data secret once. `POST /api/keys/rotate/commit {id,rotationId}` commits it; `DELETE /api/keys/rotate {id,rotationId}` aborts it. All require management authentication; data keys cannot call them. `POST /api/session/logout` requires the current `gui-session`, matching Origin, and CSRF. An admin token receives 403 and can never mint or exchange into a consent session. + +## Anthropic account usage threshold + +`PUT /api/oauth/accounts/auto-switch` + +Anthropic OAuth only; `{ provider: "anthropic", accountId, threshold }` accepts integer 0–100 or null to inherit. Missing threshold is invalid. Stored override survives restart and is removed with the account. + +Account-list DTOs include `autoSwitchThresholdOverride` (integer/null), `autoSwitchThreshold` (pool default), and `effectiveAutoSwitchThreshold`. 0 disables usage-driven switching only; it never disables pause or 429 recovery. + +HTTP: 400 invalid/unsupported; 404 missing account; `oauth_mutation_busy` on lock contention. diff --git a/docs-site/src/content/docs/ru/reference/cli/providers-accounts.md b/docs-site/src/content/docs/ru/reference/cli/providers-accounts.md index bd3daa00013..899eb314a8e 100644 --- a/docs-site/src/content/docs/ru/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/ru/reference/cli/providers-accounts.md @@ -215,7 +215,11 @@ quota-bar'ов дашборда. ### `ocx account auto-switch > [--json]` -Управляет порогом пула Codex `openai` или сохраняет порог общего пула OAuth. `on` сохраняет 80 %, `off` — 0 %, а `threshold ` принимает 0–100. Порог общего пула влияет на выбор только при включённом `pool.kernel` и `strategy: "fill-first"`; при выключенном флаге сохранение не включает переключение по порогу. В обоих случаях оно не меняет настройку включения провайдера и не отключает ротацию после ошибки 429. Для общего пула результат чтения и изменения берётся из подтверждённого ответа сервера. Для общего пула `poolEnabled` — сохранённая настройка провайдера (`null` означает отсутствие настройки), а не итоговое унаследованное состояние. `inert: true` означает, что порог сохранён, но не применяется, а `inert: false` — что пул его применяет. Отсутствие `inert` означает неизвестную возможность, которая также не даёт `enabled: true`. Провайдеры с ключом API, Anthropic и неверные значения отклоняются. +Управляет порогом пула Codex `openai` или сохраняет порог общего пула OAuth. `on` сохраняет 80 %, `off` — 0 %, а `threshold ` принимает 0–100. Порог общего пула влияет на выбор только при включённом `pool.kernel` и `strategy: "fill-first"`; при выключенном флаге сохранение не включает переключение по порогу. В обоих случаях оно не меняет настройку включения провайдера и не отключает ротацию после ошибки 429. Для общего пула результат чтения и изменения берётся из подтверждённого ответа сервера. Для общего пула `poolEnabled` — сохранённая настройка провайдера (`null` означает отсутствие настройки), а не итоговое унаследованное состояние. `inert: true` означает, что порог сохранён, но не применяется, а `inert: false` — что пул его применяет. Отсутствие `inert` означает неизвестную возможность, которая также не даёт `enabled: true`. Провайдеры с ключом API и неверные значения отклоняются. + +### `ocx account auto-switch anthropic … --account ` + +Для Anthropic OAuth команда `ocx account auto-switch anthropic threshold 90 --account ` сохраняет целое число 0–100. `off --account ` задаёт 0, `on --account ` — 80, `inherit --account ` восстанавливает наследование, а `status --account ` только читает; доступен `--json`. Карточка аккаунта предлагает тот же контроль. Отсутствующее/null значение наследует `anthropicAccountPool.autoSwitchThreshold` (по умолчанию 80); 0 отключает только переключение по использованию этого аккаунта. Настройка переживает перезапуск и повторный вход, удаляется вместе с аккаунтом. Ручной выбор, affinity, резервный выбор при неизвестных или исчерпанных квотах и ограничения маршрутов не меняются. При выключенном пуле пороги не действуют; пауза и восстановление после 429 сохраняются. ```text openai: { provider, autoSwitchThreshold: number, enabled: boolean } diff --git a/docs-site/src/content/docs/ru/reference/management-api.md b/docs-site/src/content/docs/ru/reference/management-api.md index 62b74ab5452..b7f827b36b6 100644 --- a/docs-site/src/content/docs/ru/reference/management-api.md +++ b/docs-site/src/content/docs/ru/reference/management-api.md @@ -402,3 +402,13 @@ fail closed, пока аккаунт отсутствует, а при повт ## Удалённые сессии и ротация ключей данных `POST /api/keys/rotate {id}` начинает десятиминутный overlap и один раз возвращает новый секрет. `POST /api/keys/rotate/commit {id,rotationId}` подтверждает, `DELETE /api/keys/rotate {id,rotationId}` отменяет. Требуется management auth; ключ данных не подходит. `POST /api/session/logout` требует текущую `gui-session`, совпадающий Origin и CSRF. Admin token получает 403 и не может создать consent session. + +## Порог использования аккаунта Anthropic + +`PUT /api/oauth/accounts/auto-switch` + +Только Anthropic OAuth. `{ provider: "anthropic", accountId, threshold }`: целое 0–100 или null для наследования; отсутствие поля — ошибка. Сохраняется при перезапуске и удаляется вместе с аккаунтом. + +DTO содержит `autoSwitchThresholdOverride` (целое/null), `autoSwitchThreshold` (порог пула) и `effectiveAutoSwitchThreshold`. 0 отключает только переключение по использованию; пауза и восстановление после 429 сохраняются. + +HTTP: 400 invalid/unsupported; 404 missing account; `oauth_mutation_busy` on lock contention. diff --git a/docs-site/src/content/docs/tr/reference/cli/providers-accounts.md b/docs-site/src/content/docs/tr/reference/cli/providers-accounts.md index 6c98e625a2d..df4f49ab949 100644 --- a/docs-site/src/content/docs/tr/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/tr/reference/cli/providers-accounts.md @@ -262,7 +262,11 @@ eşleşen null veya eski bir rapora düşer (çıkış 0). ### `ocx account auto-switch > [--json]` -`openai` Codex havuzunun eşiğini yönetir veya genel OAuth havuzunun eşiğini kaydeder. `on` %80, `off` %0 kaydeder; `threshold ` 0–100 kabul eder. Genel havuz eşiği yalnızca `pool.kernel` açıkken ve `strategy: "fill-first"` seçiliyken seçimi yönlendirir; bayrak kapalıyken kayıt işlemi eşik tabanlı geçişi etkinleştirmez. Her iki durumda da sağlayıcının etkinlik ayarını veya 429 hatasından sonraki otomatik hesap değişimini etkilemez. Genel havuz çıktısı sunucunun doğruladığı değerleri kullanır. Genel havuzlarda `poolEnabled`, kaydedilmiş sağlayıcı ayarıdır (`null` belirtilmemiş demektir); devralınmış etkin durumu göstermez. `inert: true` eşiğin kaydedildiğini ama uygulanmadığını, `inert: false` ise havuzun onu uyguladığını belirtir. `inert` yoksa yetenek bilinmiyordur ve bu durumda da `enabled: true` bildirilmez. API anahtarlı sağlayıcılar, Anthropic ve geçersiz değerler reddedilir. +`openai` Codex havuzunun eşiğini yönetir veya genel OAuth havuzunun eşiğini kaydeder. `on` %80, `off` %0 kaydeder; `threshold ` 0–100 kabul eder. Genel havuz eşiği yalnızca `pool.kernel` açıkken ve `strategy: "fill-first"` seçiliyken seçimi yönlendirir; bayrak kapalıyken kayıt işlemi eşik tabanlı geçişi etkinleştirmez. Her iki durumda da sağlayıcının etkinlik ayarını veya 429 hatasından sonraki otomatik hesap değişimini etkilemez. Genel havuz çıktısı sunucunun doğruladığı değerleri kullanır. Genel havuzlarda `poolEnabled`, kaydedilmiş sağlayıcı ayarıdır (`null` belirtilmemiş demektir); devralınmış etkin durumu göstermez. `inert: true` eşiğin kaydedildiğini ama uygulanmadığını, `inert: false` ise havuzun onu uyguladığını belirtir. `inert` yoksa yetenek bilinmiyordur ve bu durumda da `enabled: true` bildirilmez. API anahtarlı sağlayıcılar ve geçersiz değerler reddedilir. + +### `ocx account auto-switch anthropic … --account ` + +Anthropic OAuth için `ocx account auto-switch anthropic threshold 90 --account ` komutu 0–100 arasında tam sayı kaydeder. `off --account ` 0, `on --account ` 80 kaydeder; `inherit --account ` devralmayı geri getirir, `status --account ` yalnızca okur. `--json` desteklenir. Hesap kartı aynı ayarı sunar. Eksik/null değer `anthropicAccountPool.autoSwitchThreshold` varsayılanını (80) devralır; 0 yalnızca bu hesabın kullanıma dayalı geçişini kapatır. Yeniden başlatma ve girişte korunur, hesap silinince kaldırılır. Manuel seçim, affinity, bilinmeyen/tükenmiş kota yedek davranışı ve model rotaları değişmez. Havuz kapalıyken eşikler uygulanmaz; duraklatma ve 429 kurtarması sürer. ```text openai: { provider, autoSwitchThreshold: number, enabled: boolean } diff --git a/docs-site/src/content/docs/tr/reference/management-api.md b/docs-site/src/content/docs/tr/reference/management-api.md index 0a19213029c..1666fb0a8de 100644 --- a/docs-site/src/content/docs/tr/reference/management-api.md +++ b/docs-site/src/content/docs/tr/reference/management-api.md @@ -438,3 +438,13 @@ entegrasyonlar için en yararlıdır. ## Uzak oturumlar ve veri anahtarı döndürme `POST /api/keys/rotate {id}` on dakikalık geçişi başlatır ve yeni sırrı yalnızca bir kez döndürür. `POST /api/keys/rotate/commit {id,rotationId}` onaylar, `DELETE /api/keys/rotate {id,rotationId}` iptal eder. Yönetim kimlik doğrulaması gerekir; veri anahtarı bunları çağıramaz. `POST /api/session/logout` mevcut `gui-session`, eşleşen Origin ve CSRF ister. Admin token 403 alır ve onay oturumu oluşturamaz. + +## Anthropic hesap kullanım eşiği + +`PUT /api/oauth/accounts/auto-switch` + +Yalnızca Anthropic OAuth. `{ provider: "anthropic", accountId, threshold }`: 0–100 tam sayı veya devralmak için null; eksik alan hatadır. Yeniden başlatmada korunur, hesapla birlikte silinir. + +DTO: `autoSwitchThresholdOverride` (tam sayı/null), `autoSwitchThreshold` (havuz varsayılanı), `effectiveAutoSwitchThreshold`. 0 yalnızca kullanıma dayalı geçişi kapatır; duraklatma ve 429 kurtarması sürer. + +HTTP: 400 invalid/unsupported; 404 missing account; `oauth_mutation_busy` on lock contention. diff --git a/docs-site/src/content/docs/zh-cn/reference/cli/providers-accounts.md b/docs-site/src/content/docs/zh-cn/reference/cli/providers-accounts.md index 1806ae38843..4c1cc0afeda 100644 --- a/docs-site/src/content/docs/zh-cn/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/zh-cn/reference/cli/providers-accounts.md @@ -196,7 +196,11 @@ token，也不是简单重读账号列表。`--json` 返回 ### `ocx account auto-switch > [--json]` -控制 `openai` Codex 账户池阈值，或保存通用 OAuth 账户池阈值。`on` 保存 80%，`off` 保存 0%，`threshold ` 接受 0–100。通用池的阈值只有在 `pool.kernel` 打开且 `strategy: "fill-first"` 时才参与选择；标志关闭时，保存阈值不会启用阈值切换。两种情况下都不会改变提供方启用设置或禁用 429 错误后的轮换。通用池的查询和修改结果使用服务器确认值。通用池的 `poolEnabled` 是已保存的提供方设置，`null` 表示未指定，并不代表继承后的实际状态。`inert: true` 表示阈值已保存但未应用，`inert: false` 表示账户池正在应用它。没有 `inert` 字段表示能力未知，此时同样不会报告 `enabled: true`。API 密钥提供方、Anthropic 和无效值会被拒绝。 +控制 `openai` Codex 账户池阈值，或保存通用 OAuth 账户池阈值。`on` 保存 80%，`off` 保存 0%，`threshold ` 接受 0–100。通用池的阈值只有在 `pool.kernel` 打开且 `strategy: "fill-first"` 时才参与选择；标志关闭时，保存阈值不会启用阈值切换。两种情况下都不会改变提供方启用设置或禁用 429 错误后的轮换。通用池的查询和修改结果使用服务器确认值。通用池的 `poolEnabled` 是已保存的提供方设置，`null` 表示未指定，并不代表继承后的实际状态。`inert: true` 表示阈值已保存但未应用，`inert: false` 表示账户池正在应用它。没有 `inert` 字段表示能力未知，此时同样不会报告 `enabled: true`。API 密钥提供方和无效值会被拒绝。 + +### `ocx account auto-switch anthropic … --account ` + +Anthropic OAuth 使用 `ocx account auto-switch anthropic threshold 90 --account ` 保存账户专属整数 0–100。`off --account ` 设为 0，`on --account ` 设为 80，`inherit --account ` 恢复继承，`status --account ` 只读查询；可加 `--json`。账户卡片提供相同控制。未设置/null 继承 `anthropicAccountPool.autoSwitchThreshold`（默认 80）；0 只禁用该账户按用量切换。设置在重启和重新登录后保留，删除账户时移除。手动选择、affinity、未知或全部耗尽时的后备行为与模型路由限制不变。池禁用时不应用阈值；暂停与 429 恢复仍有效。 ```text openai: { provider, autoSwitchThreshold: number, enabled: boolean } diff --git a/docs-site/src/content/docs/zh-cn/reference/management-api.md b/docs-site/src/content/docs/zh-cn/reference/management-api.md index 4abf420ad36..a9809b951f2 100644 --- a/docs-site/src/content/docs/zh-cn/reference/management-api.md +++ b/docs-site/src/content/docs/zh-cn/reference/management-api.md @@ -348,3 +348,13 @@ OpenAI 也遵循此规则：开关不会选择特殊的 922k 模式。有效上 ## 远程会话与数据密钥轮换 `POST /api/keys/rotate {id}` 开始十分钟重叠期，并只返回一次新密钥。`POST /api/keys/rotate/commit {id,rotationId}` 提交，`DELETE /api/keys/rotate {id,rotationId}` 中止。它们都需要管理认证，数据密钥不能调用。`POST /api/session/logout` 需要当前 `gui-session`、匹配的 Origin 和 CSRF。Admin token 会收到 403，永远不能创建用户同意会话。 + +## Anthropic 账户用量阈值 + +`PUT /api/oauth/accounts/auto-switch` + +仅 Anthropic OAuth。`{ provider: "anthropic", accountId, threshold }`：整数 0–100 或 null 继承；缺少字段无效。重启后保留，随账户删除。 + +DTO 包含 `autoSwitchThresholdOverride`（整数/null）、`autoSwitchThreshold`（池默认值）、`effectiveAutoSwitchThreshold`。0 只禁用按用量切换；暂停和 429 恢复不变。 + +HTTP: 400 invalid/unsupported; 404 missing account; `oauth_mutation_busy` on lock contention. diff --git a/docs-site/src/content/docs/zh-tw/reference/cli/providers-accounts.md b/docs-site/src/content/docs/zh-tw/reference/cli/providers-accounts.md index 6abdcc7e7bc..09ead299f34 100644 --- a/docs-site/src/content/docs/zh-tw/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/zh-tw/reference/cli/providers-accounts.md @@ -172,7 +172,11 @@ ocx account resume google-antigravity ### `ocx account auto-switch > [--json]` -控制 `openai` Codex 帳戶池閾值，或儲存通用 OAuth 帳戶池閾值。`on` 儲存 80%，`off` 儲存 0%，`threshold ` 接受 0–100。通用池的閾值只有在 `pool.kernel` 開啟且 `strategy: "fill-first"` 時才參與選擇；旗標關閉時，儲存閾值不會啟用閾值切換。兩種情況下都不會改變供應商啟用設定或停用 429 錯誤後的輪替。通用池的查詢與修改結果使用伺服器確認值。通用池的 `poolEnabled` 是已儲存的供應商設定，`null` 表示未指定，並不代表繼承後的實際狀態。`inert: true` 表示閾值已儲存但未套用，`inert: false` 表示帳戶池正在套用它。沒有 `inert` 欄位表示能力未知，此時同樣不會回報 `enabled: true`。API 金鑰供應商、Anthropic 與無效值會被拒絕。 +控制 `openai` Codex 帳戶池閾值，或儲存通用 OAuth 帳戶池閾值。`on` 儲存 80%，`off` 儲存 0%，`threshold ` 接受 0–100。通用池的閾值只有在 `pool.kernel` 開啟且 `strategy: "fill-first"` 時才參與選擇；旗標關閉時，儲存閾值不會啟用閾值切換。兩種情況下都不會改變供應商啟用設定或停用 429 錯誤後的輪替。通用池的查詢與修改結果使用伺服器確認值。通用池的 `poolEnabled` 是已儲存的供應商設定，`null` 表示未指定，並不代表繼承後的實際狀態。`inert: true` 表示閾值已儲存但未套用，`inert: false` 表示帳戶池正在套用它。沒有 `inert` 欄位表示能力未知，此時同樣不會回報 `enabled: true`。API 金鑰供應商與無效值會被拒絕。 + +### `ocx account auto-switch anthropic … --account ` + +Anthropic OAuth 使用 `ocx account auto-switch anthropic threshold 90 --account ` 儲存帳戶專屬整數 0–100。`off --account ` 設為 0，`on --account ` 設為 80，`inherit --account ` 恢復繼承，`status --account ` 唯讀查詢；可加 `--json`。帳戶卡片提供相同控制。未設定/null 繼承 `anthropicAccountPool.autoSwitchThreshold`（預設 80）；0 只停用該帳戶依用量切換。設定在重啟和重新登入後保留，刪除帳戶時移除。手動選擇、affinity、未知或全部耗盡時的後備行為與模型路由限制不變。集區停用時不套用門檻；暫停與 429 復原仍有效。 ```text openai: { provider, autoSwitchThreshold: number, enabled: boolean } diff --git a/docs-site/src/content/docs/zh-tw/reference/management-api.md b/docs-site/src/content/docs/zh-tw/reference/management-api.md index e987e1f8a72..2ef37691a42 100644 --- a/docs-site/src/content/docs/zh-tw/reference/management-api.md +++ b/docs-site/src/content/docs/zh-tw/reference/management-api.md @@ -327,3 +327,13 @@ OpenAI 也遵循此規則：開關不會選擇特殊的 922k 模式。生效中 ## 遠端工作階段與資料金鑰輪替 `POST /api/keys/rotate {id}` 開始十分鐘重疊期，且只回傳一次新金鑰。`POST /api/keys/rotate/commit {id,rotationId}` 提交，`DELETE /api/keys/rotate {id,rotationId}` 中止。全部都需要管理驗證，資料金鑰不能呼叫。`POST /api/session/logout` 需要目前的 `gui-session`、相符的 Origin 與 CSRF。Admin token 會收到 403，永遠不能建立使用者同意工作階段。 + +## Anthropic 帳戶用量門檻 + +`PUT /api/oauth/accounts/auto-switch` + +僅 Anthropic OAuth。`{ provider: "anthropic", accountId, threshold }`：整數 0–100 或 null 繼承；缺少欄位無效。重啟後保留，隨帳戶刪除。 + +DTO 包含 `autoSwitchThresholdOverride`（整數/null）、`autoSwitchThreshold`（集區預設值）、`effectiveAutoSwitchThreshold`。0 只停用依用量切換；暫停和 429 復原不變。 + +HTTP: 400 invalid/unsupported; 404 missing account; `oauth_mutation_busy` on lock contention. diff --git a/gui/src/components/AccountAutoSwitchControl.tsx b/gui/src/components/AccountAutoSwitchControl.tsx index 6f0343bdb44..078b65dec46 100644 --- a/gui/src/components/AccountAutoSwitchControl.tsx +++ b/gui/src/components/AccountAutoSwitchControl.tsx @@ -10,6 +10,7 @@ export interface AccountAutoSwitchControlProps { override: number | null; disabled?: boolean; inputId: string; + hintText?: string; onChange(threshold: number | null): Promise; } @@ -20,6 +21,7 @@ export default function AccountAutoSwitchControl({ override, disabled = false, inputId, + hintText, onChange, }: AccountAutoSwitchControlProps) { const t = useT(); @@ -40,7 +42,7 @@ export default function AccountAutoSwitchControl({ draft: String(current.override ?? globalThreshold), })); const blocked = disabled || saving; - const hint = t("accountPool.autoSwitchHint"); + const hint = hintText ?? t("accountPool.autoSwitchHint"); const hintId = useId(); const write = async (next: number | null) => { diff --git a/gui/src/components/provider-workspace/AnthropicAccountPoolSettings.tsx b/gui/src/components/provider-workspace/AnthropicAccountPoolSettings.tsx index 84120b54c41..15b5e20b9e1 100644 --- a/gui/src/components/provider-workspace/AnthropicAccountPoolSettings.tsx +++ b/gui/src/components/provider-workspace/AnthropicAccountPoolSettings.tsx @@ -2,7 +2,7 @@ * Opt-in Anthropic OAuth account pool controls (#294). * Experimental — shows a strong warning because the feature is not battle-tested. */ -import { useCallback, useEffect, useState } from "react"; +import { useCallback, useEffect, useLayoutEffect, useRef, useState } from "react"; import { useT } from "../../i18n/shared"; import { getPoolSettings, putPoolSettings } from "../../pool-settings"; import { @@ -37,9 +37,11 @@ type PoolState = { export default function AnthropicAccountPoolSettings({ apiBase, accountCount, + onThresholdChange, }: { apiBase: string; accountCount: number; + onThresholdChange?: (threshold: number) => void; }) { const t = useT(); const [state, setState] = useState(null); @@ -48,6 +50,24 @@ export default function AnthropicAccountPoolSettings({ const [saving, setSaving] = useState(false); const [error, setError] = useState(null); const [loadError, setLoadError] = useState(false); + const onThresholdChangeRef = useRef(onThresholdChange); + const mountedRef = useRef(true); + const apiBaseRef = useRef(apiBase); + const saveAbortRef = useRef(null); + + useLayoutEffect(() => { + mountedRef.current = true; + return () => { + mountedRef.current = false; + saveAbortRef.current?.abort(); + saveAbortRef.current = null; + }; + }, []); + + useLayoutEffect(() => { + onThresholdChangeRef.current = onThresholdChange; + apiBaseRef.current = apiBase; + }, [apiBase, onThresholdChange]); useEffect(() => { let cancelled = false; @@ -79,6 +99,7 @@ export default function AnthropicAccountPoolSettings({ quotaWindow: normalizeAccountPoolQuotaWindow(json.quotaWindow), }); setDraft(String(nextThreshold)); + onThresholdChangeRef.current?.(nextThreshold); setStickyDraft(String(nextSticky)); setLoadError(false); }) @@ -99,6 +120,12 @@ export default function AnthropicAccountPoolSettings({ stickyLimit: number; quotaWindow: AccountPoolQuotaWindow; }) => { + const requestApiBase = apiBase; + saveAbortRef.current?.abort(); + const controller = new AbortController(); + saveAbortRef.current = controller; + const currentRequest = () => mountedRef.current && apiBaseRef.current === requestApiBase + && saveAbortRef.current === controller && !controller.signal.aborted; const previousState = state; setState({ enabled: next.enabled, @@ -112,27 +139,31 @@ export default function AnthropicAccountPoolSettings({ try { // The client owns the field mapping: `threshold` becomes `autoSwitchThreshold` and the // provider is always sent, so no call site can forget either. - const json = await putPoolSettings(apiBase, "anthropic", { + const json = await putPoolSettings(requestApiBase, "anthropic", { enabled: next.enabled, threshold: next.threshold, strategy: next.strategy, stickyLimit: next.stickyLimit, quotaWindow: next.quotaWindow, - }); + }, (input, init) => fetch(input, init), { signal: controller.signal }); + if (!currentRequest()) return; if (!json) throw new Error("save"); + const savedThreshold = typeof json.autoSwitchThreshold === "number" ? json.autoSwitchThreshold : next.threshold; const savedStrategy = normalizeAccountPoolStrategy(json?.strategy ?? next.strategy); const savedSticky = normalizeAccountPoolStickyLimit(json?.stickyLimit ?? next.stickyLimit); const savedWindow = normalizeAccountPoolQuotaWindow(json?.quotaWindow ?? next.quotaWindow); setState({ enabled: next.enabled, - threshold: next.threshold, + threshold: savedThreshold, strategy: savedStrategy, stickyLimit: savedSticky, quotaWindow: savedWindow, }); - setDraft(String(next.threshold)); + setDraft(String(savedThreshold)); + onThresholdChangeRef.current?.(savedThreshold); setStickyDraft(String(savedSticky)); } catch { + if (!currentRequest()) return; setError(t("anthropicPool.saveFailed")); if (previousState) { setState(previousState); @@ -140,7 +171,9 @@ export default function AnthropicAccountPoolSettings({ setStickyDraft(String(previousState.stickyLimit)); } } finally { - setSaving(false); + const ownsSave = saveAbortRef.current === controller; + if (ownsSave) saveAbortRef.current = null; + if (ownsSave && mountedRef.current && apiBaseRef.current === requestApiBase) setSaving(false); } }, [apiBase, state, t]); diff --git a/gui/src/components/provider-workspace/ProviderAuthPanel.tsx b/gui/src/components/provider-workspace/ProviderAuthPanel.tsx index 58b7eb214e8..440d30653f0 100644 --- a/gui/src/components/provider-workspace/ProviderAuthPanel.tsx +++ b/gui/src/components/provider-workspace/ProviderAuthPanel.tsx @@ -9,6 +9,7 @@ import { IconLock, IconRefresh, IconTrash } from "../../icons"; import type { WorkspaceItem } from "../../provider-workspace/catalog"; import { oauthAccountDisplayLabel, providerAuthSurface } from "../../provider-workspace/auth"; import { displayAccountId } from "../../lib/privacy"; +import AccountAutoSwitchControl from "../AccountAutoSwitchControl"; import { formatOAuthHealthLabel, formatOAuthHealthSummary, @@ -428,7 +429,12 @@ export default function ProviderAuthPanel({ {isOauth && ( <> {item.name === "anthropic" && ( - + { void authHandlers?.onAccountPoolThreshold?.(item.name, threshold); }} + /> )} {item.name === "google-antigravity" && (
@@ -621,6 +627,16 @@ export default function ProviderAuthPanel({
+ {item.name === "anthropic" && account.autoSwitchThresholdOverride !== undefined + && account.autoSwitchThreshold !== undefined && authHandlers.onAccountThreshold && ( + authHandlers.onAccountThreshold!(item.name, account, threshold)} + /> + )}
diff --git a/gui/src/components/provider-workspace/types.ts b/gui/src/components/provider-workspace/types.ts index 0fbca409b7c..697e43114ea 100644 --- a/gui/src/components/provider-workspace/types.ts +++ b/gui/src/components/provider-workspace/types.ts @@ -61,6 +61,9 @@ export type OAuthAccountRow = AccountQuotaReading & { autoSelectable?: boolean; skipReason?: "needs_reauth" | "paused" | "suspended" | "cooldown" | "quota_exhausted"; paused?: boolean; + autoSwitchThresholdOverride?: number | null; + autoSwitchThreshold?: number; + effectiveAutoSwitchThreshold?: number; health?: { status: OAuthAccountHealthStatus; reason?: string; until?: string }; healthLabel?: string; healthSummary?: string; @@ -91,6 +94,8 @@ export interface ProviderAuthHandlers { onReauth: (provider: string, accountId?: string) => void | Promise; onSwitchAccount: (provider: string, account: OAuthAccountRow) => void | Promise; onPauseAccount: (provider: string, account: OAuthAccountRow, paused: boolean) => void | Promise; + onAccountThreshold?: (provider: string, account: OAuthAccountRow, threshold: number | null) => Promise; + onAccountPoolThreshold?: (provider: string, threshold: number) => void | Promise; onRemoveAccount: (provider: string, account: OAuthAccountRow) => void | Promise; onRetryAccounts?: (provider: string) => void | Promise; onAddApiKey: (provider: string, key: string) => Promise; diff --git a/gui/src/hooks/useProviderAccountPools.ts b/gui/src/hooks/useProviderAccountPools.ts index a901a0ae133..d9a04b3e453 100644 --- a/gui/src/hooks/useProviderAccountPools.ts +++ b/gui/src/hooks/useProviderAccountPools.ts @@ -23,6 +23,9 @@ export interface OAuthAccount extends AccountQuotaReading { autoSelectable?: boolean; skipReason?: "needs_reauth" | "paused" | "suspended" | "cooldown" | "quota_exhausted"; paused?: boolean; + autoSwitchThresholdOverride?: number | null; + autoSwitchThreshold?: number; + effectiveAutoSwitchThreshold?: number; expiresAt?: number; health?: { status: "healthy" | "cooldown" | "reauth_required" | "warning"; reason?: string; until?: string }; healthLabel?: string; @@ -362,8 +365,28 @@ export function useProviderAccountPools(deps: { return key; }; + const setAccountPoolThreshold = async (provider: string, threshold: number): Promise => { + if (!aliveRef.current || !mountedRef.current || serverRef.current !== apiBase) return false; + // Pool settings and roster reads describe one server value. Invalidate older reads before + // publishing the confirmed save, then refresh so a concurrent external write can still win. + invalidateSelectionReads(provider, "oauth"); + setAccountSets(current => { + const existing = current[provider]; + return !existing ? current : { ...current, [provider]: { ...existing, + accounts: existing.accounts.map(row => ({ ...row, + autoSwitchThreshold: threshold, + effectiveAutoSwitchThreshold: typeof row.autoSwitchThresholdOverride === "number" + ? row.autoSwitchThresholdOverride : threshold, + })) } }; + }); + // Restart the full roster path, not only the cheap membership read. The settings card can + // resolve before the initial account load; cancelling that load without replacing its quota + // enrichment would leave usage bars empty until a manual refresh or remount. + return fetchAccountSets([provider]); + }; + const switchAccount = async (provider: string, account: OAuthAccount) => { - if (account.active || account.needsReauth || account.paused || switchingAccountRef.current || pausingAccountRef.current) return; + if (account.active || account.needsReauth || account.paused || switchingAccountRef.current || pausingAccountRef.current || selectionMutationsRef.current.has(`oauth:${provider}`)) return; const target = { provider, accountId: account.id }; switchingAccountRef.current = target; setSwitchingAccount(target); @@ -403,8 +426,58 @@ export function useProviderAccountPools(deps: { } }; + const setAccountThreshold = async (provider: string, account: OAuthAccount, threshold: number | null): Promise => { + const key = `oauth:${provider}`; + if (switchingAccountRef.current || pausingAccountRef.current || selectionMutationsRef.current.has(key)) return false; + const mutationKey = invalidateSelectionReads(provider, "oauth"); + const mutation = Symbol(); + selectionMutationsRef.current.set(mutationKey, mutation); + const currentMutation = () => aliveRef.current && mountedRef.current && serverRef.current === apiBase + && selectionMutationsRef.current.get(mutationKey) === mutation; + const label = oauthAccountDisplayLabel(accountSets[provider]?.accounts ?? [account], account, t); + try { + const bounded = createBoundedFetch(20_000); + requestsRef.current.add(bounded.controller); + let result: Pick; + try { + const res = await fetch(`${apiBase}/api/oauth/accounts/auto-switch`, { + method: "PUT", headers: { "Content-Type": "application/json" }, signal: bounded.signal, + body: JSON.stringify({ provider, accountId: account.id, threshold }), + }); + if (!res.ok) throw new Error("account threshold write failed"); + result = await res.json() as typeof result; + if (bounded.signal.aborted) throw new Error("account threshold deadline exceeded"); + } finally { + bounded.clear(); + requestsRef.current.delete(bounded.controller); + } + const validPercent = (value: unknown) => typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 100; + if (!result || (result.autoSwitchThresholdOverride !== null && !validPercent(result.autoSwitchThresholdOverride)) + || !validPercent(result.autoSwitchThreshold) || !validPercent(result.effectiveAutoSwitchThreshold)) throw new Error("invalid threshold response"); + if (!currentMutation()) return false; + invalidateSelectionReads(provider, "oauth"); + setAccountSets(current => { + const existing = current[provider]; + return !existing ? current : { ...current, [provider]: { ...existing, + accounts: existing.accounts.map(row => row.id === account.id ? { ...row, + autoSwitchThresholdOverride: result.autoSwitchThresholdOverride, + autoSwitchThreshold: result.autoSwitchThreshold, effectiveAutoSwitchThreshold: result.effectiveAutoSwitchThreshold } : row) } }; + }); + return true; + } catch { + if (currentMutation()) notify(t("accountPool.autoSwitchUpdateFailed", { email: label }), false); + return false; + } finally { + if (currentMutation()) { + invalidateSelectionReads(provider, "oauth"); + selectionMutationsRef.current.delete(mutationKey); + void refreshAccountRosters({ provider, kind: "oauth" }); + } + } + }; + const pauseAccount = async (provider: string, account: OAuthAccount, paused: boolean) => { - if (switchingAccountRef.current || pausingAccountRef.current) return; + if (switchingAccountRef.current || pausingAccountRef.current || selectionMutationsRef.current.has(`oauth:${provider}`)) return; const target = { provider, accountId: account.id }; pausingAccountRef.current = target; setPausingAccount({ ...target, paused }); @@ -623,7 +696,7 @@ export function useProviderAccountPools(deps: { return { accountSets, accountLoadStates, switchingAccount, pausingAccount, openAccounts, keyPools, addingKeyFor, newKeyValue, setAccountSets, setAccountLoadStates, setSwitchingAccount, setOpenAccounts, setKeyPools, setAddingKeyFor, setNewKeyValue, - fetchAccountSets, fetchKeyPools, refreshAccountRosters, switchAccount, pauseAccount, switchApiKey, removeApiKey, addApiKeyValue, addApiKey, editCredentialAlias, removeAccount, + fetchAccountSets, fetchKeyPools, refreshAccountRosters, switchAccount, pauseAccount, setAccountPoolThreshold, setAccountThreshold, switchApiKey, removeApiKey, addApiKeyValue, addApiKey, editCredentialAlias, removeAccount, oauthCardProviders, keyCardProviders, activeAccountNeedsReauth, }; } diff --git a/gui/src/i18n/de.ts b/gui/src/i18n/de.ts index 42ef7f400fc..6ddb8ba1d59 100644 --- a/gui/src/i18n/de.ts +++ b/gui/src/i18n/de.ts @@ -5,6 +5,7 @@ import type { TKey } from "./en"; * German i18n catalog, generated from en.ts. Must match the `TKey` set (compile-checked). */ export const de: Record = { + "pws.anthropicAccountThresholdHint": "Überschreibt den Standard des Claude-Pools. 0 deaktiviert den nutzungsbasierten Wechsel nur für dieses Konto; Pause und Wiederherstellung bei Ratenlimits gelten weiterhin.", "kiroLogin.title": "Bei Kiro anmelden", "kiroLogin.chooseMethod": "Anmeldemethode wählen", "kiroLogin.cli": "Mit Kiro CLI anmelden", diff --git a/gui/src/i18n/en.ts b/gui/src/i18n/en.ts index 11db36231fa..cd0bf7c4953 100644 --- a/gui/src/i18n/en.ts +++ b/gui/src/i18n/en.ts @@ -6,6 +6,7 @@ * `{var}` are plain interpolations. */ export const en = { + "pws.anthropicAccountThresholdHint": "Overrides the Claude pool default. 0 disables usage-based switching only for this account; pause and rate-limit recovery still apply.", "kiroLogin.title": "Sign in to Kiro", "kiroLogin.chooseMethod": "Choose a sign-in method", "kiroLogin.cli": "Kiro CLI", diff --git a/gui/src/i18n/fr.ts b/gui/src/i18n/fr.ts index fdc2b1be7de..b146c51151c 100644 --- a/gui/src/i18n/fr.ts +++ b/gui/src/i18n/fr.ts @@ -4,6 +4,7 @@ import type { TKey } from "./en"; * French i18n catalog. Must match the `TKey` set. */ export const fr: Record = { + "pws.anthropicAccountThresholdHint": "Remplace le seuil par défaut du pool Claude. 0 désactive le basculement selon l’utilisation uniquement pour ce compte ; la pause et la reprise après limitation restent actives.", "kiroLogin.title": "Se connecter à Kiro", "kiroLogin.chooseMethod": "Choisir une méthode de connexion", "kiroLogin.cli": "Importer avec Kiro CLI", diff --git a/gui/src/i18n/ja.ts b/gui/src/i18n/ja.ts index 256be8bae52..479576d4484 100644 --- a/gui/src/i18n/ja.ts +++ b/gui/src/i18n/ja.ts @@ -4,6 +4,7 @@ import type { TKey } from "./en"; * Japanese i18n catalog; must match the `TKey` set (compile-checked). */ export const ja: Record = { + "pws.anthropicAccountThresholdHint": "Claude プールの既定値を上書きします。0 はこのアカウントだけで使用量による切り替えを無効にします。一時停止とレート制限からの復旧は引き続き適用されます。", "kiroLogin.title": "Kiro にログイン", "kiroLogin.chooseMethod": "ログイン方法を選択", "kiroLogin.cli": "Kiro CLI から取り込む", diff --git a/gui/src/i18n/ko.ts b/gui/src/i18n/ko.ts index 843d22de50e..55ad082da30 100644 --- a/gui/src/i18n/ko.ts +++ b/gui/src/i18n/ko.ts @@ -4,6 +4,7 @@ import type { TKey } from "./en"; * Korean i18n catalog; must match the `TKey` set (compile-checked). */ export const ko: Record = { + "pws.anthropicAccountThresholdHint": "Claude 풀 기본값을 재정의합니다. 0은 이 계정의 사용량 기반 전환만 끄며, 일시정지와 요청 제한 복구는 계속 적용됩니다.", "kiroLogin.title": "Kiro에 로그인", "kiroLogin.chooseMethod": "로그인 방법 선택", "kiroLogin.cli": "Kiro CLI에서 가져오기", diff --git a/gui/src/i18n/ru.ts b/gui/src/i18n/ru.ts index f9e134bb6aa..545641e5a71 100644 --- a/gui/src/i18n/ru.ts +++ b/gui/src/i18n/ru.ts @@ -4,6 +4,7 @@ import type { TKey } from "./en"; * Russian i18n catalog; must match the `TKey` set (compile-checked). */ export const ru: Record = { + "pws.anthropicAccountThresholdHint": "Переопределяет порог пула Claude для этого аккаунта. 0 отключает переключение по использованию только для этого аккаунта; пауза и восстановление после 429 продолжают работать.", "kiroLogin.title": "Войти в Kiro", "kiroLogin.chooseMethod": "Выберите способ входа", "kiroLogin.cli": "Импортировать через Kiro CLI", diff --git a/gui/src/i18n/tr.ts b/gui/src/i18n/tr.ts index 70f22b95bee..6de30bf8766 100644 --- a/gui/src/i18n/tr.ts +++ b/gui/src/i18n/tr.ts @@ -5,6 +5,7 @@ import type { TKey } from "./en"; * Turkish i18n catalog. Must match the `TKey` set (compile-checked). */ export const tr: Record = { + "pws.anthropicAccountThresholdHint": "Claude havuzunun varsayılan eşiğini geçersiz kılar. 0, yalnızca bu hesap için kullanıma dayalı geçişi kapatır; duraklatma ve hız sınırı kurtarması geçerliliğini korur.", "kiroLogin.title": "Kiro oturumu aç", "kiroLogin.chooseMethod": "Oturum açma yöntemi seç", "kiroLogin.cli": "Kiro CLI ile içe aktar", diff --git a/gui/src/i18n/vi.ts b/gui/src/i18n/vi.ts index 59ec009d37d..23a799cd258 100644 --- a/gui/src/i18n/vi.ts +++ b/gui/src/i18n/vi.ts @@ -6,6 +6,7 @@ import type { TKey } from "./en"; * Technical terms and model identifiers intentionally remain English. */ export const vi: Record = { + "pws.anthropicAccountThresholdHint": "Ghi đè ngưỡng mặc định của nhóm Claude. 0 chỉ tắt chuyển đổi dựa trên mức sử dụng của tài khoản này; tạm dừng và khôi phục khi bị giới hạn vẫn áp dụng.", "kiroLogin.title": "Đăng nhập Kiro", "kiroLogin.chooseMethod": "Chọn cách đăng nhập", "kiroLogin.cli": "Nhập từ Kiro CLI", diff --git a/gui/src/i18n/zh-TW.ts b/gui/src/i18n/zh-TW.ts index 6be095cb4a4..b8b8786e243 100644 --- a/gui/src/i18n/zh-TW.ts +++ b/gui/src/i18n/zh-TW.ts @@ -2,6 +2,7 @@ import type { TKey } from "./en"; /** Traditional Chinese (Taiwan) UI strings — keys must match `en.ts` 1:1. */ export const zhTW: Record = { + "pws.anthropicAccountThresholdHint": "覆寫 Claude 集區預設門檻。0 僅停用此帳戶的依用量切換；暫停和速率限制復原仍然適用。", "kiroLogin.title": "登入 Kiro", "kiroLogin.chooseMethod": "選擇登入方式", "kiroLogin.cli": "從 Kiro CLI 匯入", diff --git a/gui/src/i18n/zh.ts b/gui/src/i18n/zh.ts index 446b16bb0d0..c93b513d65a 100644 --- a/gui/src/i18n/zh.ts +++ b/gui/src/i18n/zh.ts @@ -4,6 +4,7 @@ import type { TKey } from "./en"; * Chinese i18n catalog; must match the `TKey` set (compile-checked). */ export const zh: Record = { + "pws.anthropicAccountThresholdHint": "覆盖 Claude 池默认阈值。0 仅禁用此账户的按用量切换；暂停和速率限制恢复仍然生效。", "kiroLogin.title": "登录 Kiro", "kiroLogin.chooseMethod": "选择登录方式", "kiroLogin.cli": "从 Kiro CLI 导入", diff --git a/gui/src/pages/Providers.tsx b/gui/src/pages/Providers.tsx index da2c7787f9e..f45544e0376 100644 --- a/gui/src/pages/Providers.tsx +++ b/gui/src/pages/Providers.tsx @@ -386,7 +386,7 @@ export default function Providers({ apiBase }: { apiBase: string }) { const { accountSets, setAccountSets, accountLoadStates, switchingAccount, pausingAccount, keyPools, fetchAccountSets, fetchKeyPools, refreshAccountRosters, oauthCardProviders, keyCardProviders, - switchAccount, pauseAccount, switchApiKey, removeApiKey, addApiKeyValue, editCredentialAlias, + switchAccount, pauseAccount, setAccountPoolThreshold, setAccountThreshold, switchApiKey, removeApiKey, addApiKeyValue, editCredentialAlias, removeAccount, activeAccountNeedsReauth, } = pools; const refreshSelection = useCallback((target?: AccountSelectionTarget) => { @@ -659,6 +659,8 @@ export default function Providers({ apiBase }: { apiBase: string }) { onReauth: (provider, accountId) => requestLoginOAuth(provider, true, accountId), onSwitchAccount: switchAccount, onPauseAccount: pauseAccount, + onAccountPoolThreshold: setAccountPoolThreshold, + onAccountThreshold: setAccountThreshold, onRemoveAccount: removeAccount, onRetryAccounts: async provider => { await fetchAccountSets([provider]); }, onAddApiKey: addApiKeyValue, diff --git a/gui/tests/anthropic-pool-quota-window.test.tsx b/gui/tests/anthropic-pool-quota-window.test.tsx index c8b2a86c8fc..c50ae3506de 100644 --- a/gui/tests/anthropic-pool-quota-window.test.tsx +++ b/gui/tests/anthropic-pool-quota-window.test.tsx @@ -79,7 +79,7 @@ function stubPool(initial: PoolPayload): Record[] { return puts; } -async function mountPool(): Promise { +async function mountPool(onThresholdChange?: (threshold: number) => void): Promise { const host = testWindow.document.createElement("div"); testWindow.document.body.appendChild(host as never); const { createRoot } = await import("react-dom/client"); @@ -88,7 +88,7 @@ async function mountPool(): Promise { mountedRoots.push(root); root.render( - + , ); }); @@ -120,6 +120,43 @@ afterEach(async () => { }); describe("Anthropic account pool quota window", () => { + test("only confirmed pool defaults seed account override controls", async () => { + let fail = false; + globalThis.fetch = (async (_input, init) => init?.method === "PUT" + ? fail ? new Response(null, { status: 500 }) : Response.json({ enabled: false, autoSwitchThreshold: 73 }) + : Response.json({ enabled: true, autoSwitchThreshold: 64, strategy: "quota", stickyLimit: 1, quotaWindow: "five-hour" })) as typeof fetch; + const values: number[] = []; + const host = await mountPool(value => { values.push(value); }); + expect(values).toEqual([64]); + const toggle = host.querySelector('button[aria-pressed]') as HTMLButtonElement; + await act(async () => { toggle.click(); await flush(); }); + expect(values).toEqual([64, 73]); + fail = true; + await act(async () => { toggle.click(); await flush(); }); + expect(values).toEqual([64, 73]); + }); + + test("an unmounted settings card aborts its save without publishing the old server value", async () => { + let aborted = false; + globalThis.fetch = (async (_input, init) => { + if (init?.method !== "PUT") return Response.json({ enabled: true, autoSwitchThreshold: 64, strategy: "quota", stickyLimit: 1, quotaWindow: "five-hour" }); + return new Promise((_resolve, reject) => { + init.signal?.addEventListener("abort", () => { + aborted = true; + reject(new Error("aborted")); + }, { once: true }); + }); + }) as typeof fetch; + const values: number[] = []; + const host = await mountPool(value => { values.push(value); }); + const toggle = host.querySelector('button[aria-pressed]') as HTMLButtonElement; + await act(async () => { toggle.click(); await Promise.resolve(); }); + const root = mountedRoots.pop(); + await act(async () => { root?.unmount(); await Promise.resolve(); }); + expect(aborted).toBe(true); + expect(values).toEqual([64]); + }); + test("quota window selector renders for quota and fill-first strategies", async () => { stubPool({ enabled: true, diff --git a/gui/tests/provider-account-pause-refresh.test.tsx b/gui/tests/provider-account-pause-refresh.test.tsx index 3bcdf9cee69..9580be1e6d0 100644 --- a/gui/tests/provider-account-pause-refresh.test.tsx +++ b/gui/tests/provider-account-pause-refresh.test.tsx @@ -54,6 +54,92 @@ afterEach(async () => { } }); +test("confirmed threshold persists in UI when follow-up read fails; failed writes preserve prior state", async () => { + let fail = false; const bodies: unknown[] = []; + respond = async (_url, init) => { + if (init?.method !== "PUT") return new Response(null, { status: 503 }); + bodies.push(JSON.parse(String(init.body))); + return fail ? new Response(null, { status: 500 }) : Response.json({ autoSwitchThresholdOverride: 0, autoSwitchThreshold: 70, effectiveAutoSwitchThreshold: 0 }); + }; + await act(async () => { expect(await pools.setAccountThreshold("fixture", row("b", false), 0)).toBe(true); }); + expect(pools.accountSets.fixture.accounts[1]?.autoSwitchThresholdOverride).toBe(0); + expect(bodies[0]).toEqual({ provider: "fixture", accountId: "b", threshold: 0 }); + fail = true; + await act(async () => { expect(await pools.setAccountThreshold("fixture", row("b", false), null)).toBe(false); }); + expect(pools.accountSets.fixture.accounts[1]?.autoSwitchThresholdOverride).toBe(0); + expect(notices.some(notice => notice.key === "accountPool.autoSwitchUpdateFailed")).toBe(true); +}); + +test("pending threshold owns its roster generation and blocks conflicting pause", async () => { + let settle!: (response: Response) => void; let writes = 0; + respond = async (_url, init) => { + if (init?.method !== "PUT") return new Response(null, { status: 503 }); + writes++; return new Promise(resolve => { settle = resolve; }); + }; + let pending!: Promise; + await act(async () => { pending = pools.setAccountThreshold("fixture", row("b", false), 40); }); + await act(async () => { await pools.pauseAccount("fixture", row("b", false), true); }); + expect(writes).toBe(1); + await act(async () => { settle(Response.json({ autoSwitchThresholdOverride: 40, autoSwitchThreshold: 70, effectiveAutoSwitchThreshold: 40 })); await pending; }); + expect(pools.accountSets.fixture.accounts[1]?.autoSwitchThresholdOverride).toBe(40); +}); + +test("a confirmed pool threshold invalidates an older roster while later external changes still win", async () => { + let settleStale!: (response: Response) => void; + let reads = 0; + const urls: string[] = []; + respond = async (url, init) => { + if (init?.method === "PUT") return new Response(null, { status: 500 }); + urls.push(url); + reads++; + if (reads === 1) return new Promise(resolve => { settleStale = resolve; }); + const threshold = reads <= 3 ? 70 : 55; + return Response.json({ activeAccountId: "a", accounts: [ + { ...row("a", true), quotaMode: "probe", autoSwitchThresholdOverride: null, autoSwitchThreshold: threshold, effectiveAutoSwitchThreshold: threshold }, + { ...row("b", false), quotaMode: "probe", autoSwitchThresholdOverride: 40, autoSwitchThreshold: threshold, effectiveAutoSwitchThreshold: 40 }, + ] }); + }; + + let stale!: Promise; + await act(async () => { stale = pools.refreshAccountRosters({ provider: "fixture", kind: "oauth" }); }); + await act(async () => { expect(await pools.setAccountPoolThreshold("fixture", 70)).toBe(true); }); + await act(async () => { await Promise.resolve(); }); + expect(urls.some(url => url.includes("quota=1"))).toBe(true); + expect(pools.accountSets.fixture.accounts[0]?.autoSwitchThreshold).toBe(70); + expect(pools.accountSets.fixture.accounts[1]?.effectiveAutoSwitchThreshold).toBe(40); + + await act(async () => { + settleStale(Response.json({ activeAccountId: "a", accounts: [ + { ...row("a", true), quotaMode: "probe", autoSwitchThresholdOverride: null, autoSwitchThreshold: 65, effectiveAutoSwitchThreshold: 65 }, + { ...row("b", false), quotaMode: "probe", autoSwitchThresholdOverride: 40, autoSwitchThreshold: 65, effectiveAutoSwitchThreshold: 40 }, + ] })); + await stale; + }); + expect(pools.accountSets.fixture.accounts[0]?.autoSwitchThreshold).toBe(70); + + await act(async () => { expect(await pools.refreshAccountRosters({ provider: "fixture", kind: "oauth" })).toBe(true); }); + expect(pools.accountSets.fixture.accounts[0]?.autoSwitchThreshold).toBe(55); +}); + +test("a stalled account threshold write is aborted when the hook unmounts", async () => { + let aborted = false; + respond = async (_url, init) => new Promise((_resolve, reject) => { + expect(init?.signal).toBeInstanceOf(AbortSignal); + init?.signal?.addEventListener("abort", () => { + aborted = true; + reject(new Error("aborted")); + }, { once: true }); + }); + let pending!: Promise; + await act(async () => { + pending = pools.setAccountThreshold("fixture", row("b", false), 40); + await Promise.resolve(); + }); + await act(async () => { root?.unmount(); root = null; }); + expect(await pending).toBe(false); + expect(aborted).toBe(true); +}); + test("a saved pause stays visible and only the failed roster refresh is reported", async () => { respond = async (_url, init) => init?.method === "PUT" ? Response.json({ ok: true, activeAccountId: "a", activeAccountChanged: false }) @@ -86,4 +172,3 @@ test("a rejected save reports the pause failure and leaves the row unpaused", as expect(pools.accountSets.fixture.accounts.find(account => account.id === "b")?.paused).toBe(false); expect(notices).toEqual([{ key: "codexAuth.pauseFailed", ok: false }]); }); - diff --git a/gui/tests/provider-quota-refresh-controls.test.tsx b/gui/tests/provider-quota-refresh-controls.test.tsx index 2c1d868550a..0bab2820572 100644 --- a/gui/tests/provider-quota-refresh-controls.test.tsx +++ b/gui/tests/provider-quota-refresh-controls.test.tsx @@ -92,6 +92,23 @@ test("the usage tab reports the real outcome, not the click", async () => { expect(host.textContent).toContain("Quota check completed"); }); +test("Anthropic account threshold editor uses its pool default, preserves zero, and resets with null", async () => { + const calls: Array = []; + const item = { ...oauthItem, name: "anthropic", adapter: "anthropic" }; + const handlers = authHandlers({ onAccountThreshold: async (_provider, _account, value) => { calls.push(value); return true; } }); + const row = { id: "threshold-account", active: true, autoSwitchThresholdOverride: null, autoSwitchThreshold: 65 }; + await render(); + const toggle = () => host.querySelector('[aria-label^="Override global usage threshold"]') as HTMLButtonElement; + expect(toggle()).not.toBeNull(); + await act(async () => { toggle().click(); }); expect(calls).toEqual([65]); + await render(); + const input = host.querySelector('#anthropic-threshold-threshold-account') as HTMLInputElement; + expect(input.value).toBe("0"); + await act(async () => { toggle().click(); }); expect(calls).toEqual([65, null]); + await render(); + expect(toggle()).toBeNull(); // Old servers do not acquire a synthetic capability. +}); + test("a failed read is reported as a failure", async () => { const { handler, settle } = deferredHandler(); await render(); diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index c0898f6e213..b955f7fffe4 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -102,7 +102,9 @@ "anthropic-account-pool.test.ts": "adapters/anthropic", "anthropic-account-pause-outbound.test.ts": "adapters/anthropic", "anthropic-account-pause.test.ts": "adapters/anthropic", + "anthropic-account-threshold.test.ts": "adapters/anthropic", "anthropic-combo-account-cooldown.test.ts": "adapters/anthropic", + "cli-anthropic-account-threshold.test.ts": "cli", "anthropic-model-routes.test.ts": "adapters/anthropic", "anthropic-agentrouter-language-framing.test.ts": "adapters/anthropic", "anthropic-baseurl-override.test.ts": "adapters/anthropic", diff --git a/skills/ocx/references/01_management_surface.md b/skills/ocx/references/01_management_surface.md index 110b56be1e9..15c52cd6c71 100644 --- a/skills/ocx/references/01_management_surface.md +++ b/skills/ocx/references/01_management_surface.md @@ -919,15 +919,19 @@ Show or set the usage percentage at which a pool moves to another account. | PUT | `/api/codex-auth/auto-switch` | | GET | `/api/oauth/accounts/pool` | | PUT | `/api/oauth/accounts/pool` | +| GET | `/api/oauth/accounts` | +| PUT | `/api/oauth/accounts/auto-switch` | | Flag | Value | Meaning | |---|---|---| | `--json` | boolean | Emit the stored threshold and whether it is applied. | +| `--account` | string | Anthropic account ID; inherit restores the pool default, off stores zero. | JSON mode: `envelope`. - A bare invocation reads and never writes. - `on` stores 80%, `off` stores 0%, and `threshold ` accepts 0-100. +- Anthropic requires --account ; inherit sends null to restore its pool default. Manual/affinity precedence and pool-off recovery are unchanged. - For a generic OAuth pool, `inert: true` means the threshold is stored but not applied, `inert: false` means the pool is applying it, and an absent `inert` is an unknown capability. ### `ocx storage cleanup` diff --git a/src/cli/account-anthropic-threshold.ts b/src/cli/account-anthropic-threshold.ts new file mode 100644 index 00000000000..31b7176ac67 --- /dev/null +++ b/src/cli/account-anthropic-threshold.ts @@ -0,0 +1,41 @@ +import { apiError, apiJson, proxyUnreachable, resolveBaseUrl, type AccountDeps } from "./account-api"; + +/** A separate account selector prevents accidentally changing the whole Anthropic pool. */ +export async function cmdAnthropicAccountThreshold(args: string[], action: string, wantsJson: boolean, deps: AccountDeps): Promise { + const selector = args.indexOf("--account"); + const accountId = selector >= 0 ? args[selector + 1] : undefined; + if (selector >= 0) args.splice(selector, 2); + let threshold: number | null | undefined; + if (action === "inherit" && args.length === 0) threshold = null; + else if (action === "off" && args.length === 0) threshold = 0; + else if (action === "on" && args.length === 0) threshold = 80; + else if (action === "threshold" && args.length === 1 && /^\d+$/.test(args[0]!)) threshold = Number(args[0]); + else if (action !== "status" || args.length !== 0) return invalid(); + if (!accountId?.trim() || accountId.startsWith("--") || (threshold !== undefined && threshold !== null && threshold > 100)) return invalid(); + const base = await resolveBaseUrl(deps); + if (!base) return proxyUnreachable(); + const response = action === "status" + ? await apiJson(deps, base, "GET", "/api/oauth/accounts?provider=anthropic") + : await apiJson(deps, base, "PUT", "/api/oauth/accounts/auto-switch", { provider: "anthropic", accountId, threshold }); + if (response.status === 0) return proxyUnreachable(response.transportError); + if (response.status !== 200) return apiError(response.json, "failed to update account threshold", response.status); + if (!response.json || typeof response.json !== "object" || Array.isArray(response.json)) return apiError({}, "invalid account threshold response", 400); + const result = action === "status" + ? (Array.isArray(response.json.accounts) ? response.json.accounts : []).find((row: { id?: string } | null) => row?.id === accountId) + : response.json; + if (!result || typeof result !== "object") return apiError({}, "account not found", 404); + if (!Object.hasOwn(result, "autoSwitchThresholdOverride")) return apiError({}, "proxy does not support Anthropic account thresholds; upgrade and restart it", 400); + const validPercent = (value: unknown) => typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 100; + if ((result.autoSwitchThresholdOverride !== null && !validPercent(result.autoSwitchThresholdOverride)) + || !validPercent(result.effectiveAutoSwitchThreshold)) return apiError({}, "invalid account threshold response", 400); + const payload = { provider: "anthropic", accountId, autoSwitchThresholdOverride: result.autoSwitchThresholdOverride, + effectiveAutoSwitchThreshold: result.effectiveAutoSwitchThreshold }; + if (wantsJson) console.log(JSON.stringify(payload, null, 2)); + else console.log(`auto-switch: ${payload.autoSwitchThresholdOverride === null ? "inherited" : "custom"} (${payload.effectiveAutoSwitchThreshold === 0 ? "usage-based switching disabled" : `${payload.effectiveAutoSwitchThreshold}%`})`); + return 0; +} + +function invalid(): number { + console.error("Usage: ocx account auto-switch anthropic > --account [--json]"); + return 2; +} diff --git a/src/cli/account-api.ts b/src/cli/account-api.ts index 43199c43a23..6ad19b3105a 100644 --- a/src/cli/account-api.ts +++ b/src/cli/account-api.ts @@ -347,6 +347,7 @@ interface OAuthAccountDto { needsReauth?: boolean; /** Present only for providers that support operator pause (generic OAuth pools). */ paused?: boolean; + autoSwitchThresholdOverride?: number | null; autoSelectable?: boolean; skipReason?: unknown; /** Always sent by the management route; explicitly `null` when the tier is unknown. */ @@ -388,6 +389,7 @@ async function fetchOAuthRows( active: a.active ?? a.id === activeId, needsReauth: a.needsReauth, ...(a.paused === true ? { paused: true } : {}), + ...(name === "anthropic" && Object.hasOwn(a, "autoSwitchThresholdOverride") ? { autoSwitchThresholdOverride: a.autoSwitchThresholdOverride } : {}), ...(name === "kiro" && typeof a.autoSelectable === "boolean" ? { autoSelectable: a.autoSelectable } : {}), ...(name === "kiro" && a.autoSelectable === false && isKiroSkipReason(a.skipReason) diff --git a/src/cli/account-extended.ts b/src/cli/account-extended.ts index 5260b979a99..bc61bfb3483 100644 --- a/src/cli/account-extended.ts +++ b/src/cli/account-extended.ts @@ -1,4 +1,5 @@ import { loadConfig } from "../config"; +import { cmdAnthropicAccountThreshold } from "./account-anthropic-threshold"; import { isReservedCodexAccountWord, reportCodexAccountTargetError, resolveCodexAccountTarget } from "./account-target"; import { hasPassiveAccountQuota } from "../providers/quota"; import { closeSync, openSync, readSync, readFileSync, statSync } from "node:fs"; @@ -40,6 +41,7 @@ const AUTO_NOTE = "auto (no pin — lowest-usage account is selected per request const EXTENDED_USAGE = `Usage: ocx account refresh [--json] ocx account auto-switch > [--json] + ocx account auto-switch anthropic > --account [--json] ocx account alias [--json] ocx account priority [<-100..100|first|earlier|normal|later|last|reset>] [--json] ocx account pause [--json] @@ -358,6 +360,7 @@ export async function cmdAutoSwitch(args: string[], deps: AccountDeps): Promise< const classified = configAndType(deps, name); // Anthropic keeps its threshold on its own pool contract; generic OAuth providers (#695) // and the Codex pool are accepted here. + if (!("error" in classified) && classified.type === "oauth" && name === "anthropic") return cmdAnthropicAccountThreshold(args, action, wantsJson, deps); if ("error" in classified || classified.type === "api-key" || name === "anthropic") { return usage("Error: auto-switch only applies to the openai Codex account pool or a generic OAuth provider pool"); } diff --git a/src/cli/account.ts b/src/cli/account.ts index ab9a2089f7d..c702bd48fbb 100644 --- a/src/cli/account.ts +++ b/src/cli/account.ts @@ -49,6 +49,7 @@ const ACCOUNT_USAGE = `Usage: ocx account clear [--json] ocx account refresh [--json] ocx account auto-switch > [--json] + ocx account auto-switch anthropic > --account [--json] ocx account alias [--json] ocx account priority [<-100..100|first|earlier|normal|later|last|reset>] [--json] ocx account pause [--json] diff --git a/src/cli/capabilities.ts b/src/cli/capabilities.ts index c64edc31d30..484454a802f 100644 --- a/src/cli/capabilities.ts +++ b/src/cli/capabilities.ts @@ -636,13 +636,17 @@ export const CAPABILITIES: readonly Capability[] = [ { method: "PUT", path: "/api/codex-auth/auto-switch" }, { method: "GET", path: "/api/oauth/accounts/pool" }, { method: "PUT", path: "/api/oauth/accounts/pool" }, + { method: "GET", path: "/api/oauth/accounts" }, + { method: "PUT", path: "/api/oauth/accounts/auto-switch" }, ], - flags: [{ name: "--json", value: "boolean", summary: "Emit the stored threshold and whether it is applied." }], + flags: [{ name: "--json", value: "boolean", summary: "Emit the stored threshold and whether it is applied." }, + { name: "--account", value: "string", summary: "Anthropic account ID; inherit restores the pool default, off stores zero." }], mutates: true, json: "envelope", details: [ "A bare invocation reads and never writes.", "`on` stores 80%, `off` stores 0%, and `threshold ` accepts 0-100.", + "Anthropic requires --account ; inherit sends null to restore its pool default. Manual/affinity precedence and pool-off recovery are unchanged.", "For a generic OAuth pool, `inert: true` means the threshold is stored but not applied, `inert: false` means the pool is applying it, and an absent `inert` is an unknown capability.", ], }, diff --git a/src/lib/account-selection-events.ts b/src/lib/account-selection-events.ts index c3381b91060..0272fb3ebe4 100644 --- a/src/lib/account-selection-events.ts +++ b/src/lib/account-selection-events.ts @@ -5,8 +5,20 @@ export type AccountSelectionEvent = { revision: number; }; +export type OAuthAccountSelectionSnapshot = Readonly<{ + accountId: string; + revision?: string; +}>; + +export type OAuthAccountRoutingPolicyChange = Readonly<{ + provider: string; + before: OAuthAccountSelectionSnapshot; + after: OAuthAccountSelectionSnapshot; +}>; + const listeners = new Set<(event: AccountSelectionEvent) => void>(); const oauthPauseListeners = new Set<(provider: string) => void>(); +const oauthRoutingPolicyListeners = new Set<(event: OAuthAccountRoutingPolicyChange) => void>(); let revision = 0; /** Call only after the authoritative selection has been persisted. */ @@ -41,6 +53,22 @@ export function subscribeOAuthAccountPauseChanges(listener: (provider: string) = return () => { oauthPauseListeners.delete(subscription); }; } +/** + * Internal post-persistence signal for policy-only mutations that advance the + * selection generation without changing the operator's selected account. + */ +export function publishOAuthAccountRoutingPolicyChange(event: OAuthAccountRoutingPolicyChange): void { + for (const listener of [...oauthRoutingPolicyListeners]) { + try { listener(event); } catch { /* A process-local policy observer cannot undo persistence. */ } + } +} + +export function subscribeOAuthAccountRoutingPolicyChanges(listener: (event: OAuthAccountRoutingPolicyChange) => void): () => void { + const subscription = (event: OAuthAccountRoutingPolicyChange) => listener(event); + oauthRoutingPolicyListeners.add(subscription); + return () => { oauthRoutingPolicyListeners.delete(subscription); }; +} + export function currentAccountSelectionRevision(): number { return revision; } diff --git a/src/oauth/anthropic-account-threshold.ts b/src/oauth/anthropic-account-threshold.ts new file mode 100644 index 00000000000..5aaf5c05078 --- /dev/null +++ b/src/oauth/anthropic-account-threshold.ts @@ -0,0 +1,13 @@ +import type { OcxConfig } from "../types"; +import type { ProviderAccount } from "./types"; + +/** Shared strict boundary for persisted rows and management writes. Zero is not exhaustion. */ +export function parseAnthropicAccountThreshold(value: unknown): number | null { + return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 100 ? value : null; +} + +/** Missing or malformed legacy metadata inherits; concrete zero must survive nullish fallback. */ +export function effectiveAnthropicAccountThreshold(config: OcxConfig, account?: Pick): number { + return parseAnthropicAccountThreshold(account?.autoSwitchThresholdOverride) + ?? parseAnthropicAccountThreshold(config.anthropicAccountPool?.autoSwitchThreshold) ?? 80; +} diff --git a/src/oauth/anthropic-routing.ts b/src/oauth/anthropic-routing.ts index e0a0db6970a..050d51a6097 100644 --- a/src/oauth/anthropic-routing.ts +++ b/src/oauth/anthropic-routing.ts @@ -34,7 +34,8 @@ import type { OcxAccountPoolQuotaWindow, OcxAccountPoolRotationStrategy, OcxConf import { sweepExpiredOnWrite } from "../lib/state-store-sweeper"; import { retainedUtf8Bytes } from "../lib/admission"; import { routeCandidates, type AnthropicRouteDecision } from "./anthropic-model-routes"; -import { subscribeOAuthAccountPauseChanges } from "../lib/account-selection-events"; +import { subscribeAccountSelections, subscribeOAuthAccountPauseChanges, subscribeOAuthAccountRoutingPolicyChanges } from "../lib/account-selection-events"; +import { effectiveAnthropicAccountThreshold } from "./anthropic-account-threshold"; /** * The read side of a `Headers` object, so a caller can pass the live upstream response's @@ -116,6 +117,11 @@ export function anthropicAutoSwitchThreshold(config: OcxConfig): number { return DEFAULT_AUTO_SWITCH_THRESHOLD; } +/** Read live policy at selection, not a credential snapshot captured before an await. */ +export function anthropicAccountAutoSwitchThreshold(config: OcxConfig, accountId: string): number { + return effectiveAnthropicAccountThreshold(config, getAccountSet(PROVIDER)?.accounts.find(row => row.id === accountId)); +} + /** Strict parse for management APIs — returns null instead of defaulting. */ export function parseAccountPoolQuotaWindow(raw: unknown): OcxAccountPoolQuotaWindow | null { if (typeof raw === "string" && VALID_QUOTA_WINDOWS.has(raw as OcxAccountPoolQuotaWindow)) { @@ -314,6 +320,23 @@ const QUORUM_CACHE_TTL_MS = 2_000; let quorumCache: { value: boolean; readAt: number } | null = null; // Pause changes eligibility, not health. Do not reset cooldowns or cancel sent turns. subscribeOAuthAccountPauseChanges(provider => { if (provider === PROVIDER) quorumCache = null; }); +// A threshold write must fence in-flight automatic proposals, but it does not +// revoke an operator's one-dispatch choice. Rebase only that still-owned choice; +// an intervening account change clears it, so an ABA selection is not resurrected. +subscribeOAuthAccountRoutingPolicyChanges(event => { + if (event.provider !== PROVIDER || !manualPreference) return; + if (manualPreference.accountId !== event.before.accountId + || manualPreference.revision !== event.before.revision) return; + manualPreference = event.after.accountId === manualPreference.accountId ? { ...event.after } : null; +}); +// Any non-policy selection generation supersedes the pending one-shot choice. +// Threshold mutations rebase it first, before this generic notification runs. +subscribeAccountSelections(event => { + if (event.provider !== PROVIDER || event.kind !== "oauth" || !manualPreference) return; + const current = captureOAuthAccountSelection(PROVIDER); + if (current?.accountId !== manualPreference.accountId + || current.revision !== manualPreference.revision) manualPreference = null; +}); /** * Whether a 429 has somewhere to go: two or more accounts that could serve traffic if asked. @@ -401,8 +424,15 @@ function compareScoredAccounts(a: ScoredAccount, b: ScoredAccount): number { function pickLowestUsage(config: OcxConfig, excludeId: string | undefined, now: number, decision: AnthropicRouteDecision | null = null): string | null { const window = anthropicQuotaWindow(anthropicAccountPoolConfig(config)); const unfiltered = routeCandidates(getEligibleAnthropicAccounts(now), decision).filter(id => id !== excludeId); - const available = window === "weekly" ? unfiltered.filter(id => !exhausted5h(id)) : unfiltered; - const eligible = available.length > 0 ? available : unfiltered; + const available = window === "weekly" ? unfiltered.filter(id => !exhausted5h(id) + || isAnthropicAccountPoolEnabled(config) && anthropicAccountAutoSwitchThreshold(config, id) === 0) : unfiltered; + const availableOrFallback = available.length > 0 ? available : unfiltered; + // Thresholds are preferences, never eligibility. Keep the old lowest-usage fallback + // when every candidate is drained, and keep pool-off reactive recovery policy inert. + const hasKnownUnderThreshold = isAnthropicAccountPoolEnabled(config) + && availableOrFallback.some(id => hasKnownUsage(config, id) && isActiveUnderFillFirstThreshold(config, id)); + const eligible = hasKnownUnderThreshold + ? availableOrFallback.filter(id => isActiveUnderFillFirstThreshold(config, id)) : availableOrFallback; if (eligible.length === 0) return null; const scored: ScoredAccount[] = eligible.map(accountId => ({ accountId, @@ -435,7 +465,8 @@ function pickNextFillFirstAnthropicAccount( decision: AnthropicRouteDecision | null, ): string | null { const window = anthropicQuotaWindow(anthropicAccountPoolConfig(config)); - const available = window === "weekly" ? eligible.filter(id => !exhausted5h(id)) : eligible; + const available = window === "weekly" ? eligible.filter(id => !exhausted5h(id) + || anthropicAccountAutoSwitchThreshold(config, id) === 0) : eligible; const candidates = available.length > 0 ? available : eligible; if (candidates.length === 0) return null; const routeOrder = usesDeclaredRouteOrder(eligible, decision); @@ -517,7 +548,7 @@ function anthropicPoolStrategy(config: OcxConfig): OcxAccountPoolRotationStrateg } function isActiveUnderFillFirstThreshold(config: OcxConfig, accountId: string): boolean { - const threshold = anthropicAutoSwitchThreshold(config); + const threshold = anthropicAccountAutoSwitchThreshold(config, accountId); if (threshold <= 0) return true; const window = anthropicQuotaWindow(anthropicAccountPoolConfig(config)); if (window === "weekly" && exhausted5h(accountId)) return false; @@ -676,7 +707,7 @@ export function resolveAnthropicAccountForSession( return { accountId: strategyPick.accountId, reason: strategyPick.reason, routePosition: decision?.position }; } - const threshold = anthropicAutoSwitchThreshold(config); + const threshold = anthropicAccountAutoSwitchThreshold(config, set.activeAccountId); const activeOk = set.accounts.some(a => a.id === set.activeAccountId && a.needsReauth !== true) && !isCooled(set.activeAccountId, now) && eligible.includes(set.activeAccountId); diff --git a/src/oauth/store.ts b/src/oauth/store.ts index 7d085c96034..0d5233f9124 100644 --- a/src/oauth/store.ts +++ b/src/oauth/store.ts @@ -27,13 +27,14 @@ import { atomicWriteFileNoFollowUnclaimed } from "../config/atomic-write"; import { assertNotRealHomeUnderTest } from "../lib/test-home-guard"; import { recordOwnedConfigPath } from "../lib/config-ownership"; import { MAX_PENDING_OAUTH_MUTATIONS } from "../lib/translator-budget"; -import { publishAccountSelection, publishOAuthAccountPauseChange } from "../lib/account-selection-events"; +import { publishAccountSelection, publishOAuthAccountPauseChange, publishOAuthAccountRoutingPolicyChange } from "../lib/account-selection-events"; import { captureConfigGeneration, type GenerationContext, } from "../lib/state-store-sweeper"; import { validateCopilotApiBaseUrl } from "./github-copilot"; import { validateDevinApiBaseUrl } from "./devin/api-base"; +import { parseAnthropicAccountThreshold } from "./anthropic-account-threshold"; import type { OAuthAccountSelection, OAuthCredentialSource, OAuthCredentials, ProviderAccount, ProviderAccountSet } from "./types"; export type AuthStore = Record; @@ -626,6 +627,8 @@ function normalizeAccount(value: unknown): ProviderAccount | null { if (typeof candidate.alias === "string" && candidate.alias.trim()) account.alias = candidate.alias.trim(); if (candidate.needsReauth === true) account.needsReauth = true; if (candidate.paused === true) account.paused = true; + const threshold = parseAnthropicAccountThreshold(candidate.autoSwitchThresholdOverride); + if (threshold !== null) account.autoSwitchThresholdOverride = threshold; if (typeof candidate.addedAt === "number") account.addedAt = candidate.addedAt; if (typeof candidate.loginId === "string" && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(candidate.loginId)) { @@ -783,7 +786,7 @@ function serializeMutation(work: () => Promise, retainedValues: readonly u drainOAuthMutations(); return result; } -export function mutateStore(fn:(store:AuthStore)=>T|Promise, retainedValues: readonly unknown[] = [], options?: { waitMs?: number; assertBeforePersist?: () => void; scrubLegacyBackup?: (result: T) => readonly string[]; finalizeResult?: (result: T, store: AuthStore) => void }):Promise{return serializeMutation(async()=>{const guard=await createOAuthFileLock({path:getAuthStoreLockPath(),staleAfterMs:30000}).acquire();try{ +export function mutateStore(fn:(store:AuthStore)=>T|Promise, retainedValues: readonly unknown[] = [], options?: { waitMs?: number; assertBeforePersist?: () => void; scrubLegacyBackup?: (result: T) => readonly string[]; finalizeResult?: (result: T, store: AuthStore) => void; afterPersist?: (result: T) => void }):Promise{return serializeMutation(async()=>{const guard=await createOAuthFileLock({path:getAuthStoreLockPath(),staleAfterMs:30000}).acquire();try{ const { store, hadLegacy } = loadAuthStoreInternal(); if (hadLegacy) backupLegacyOnce(); const selections = new Map(Object.entries(store).map(([provider, set]) => [provider, { @@ -821,6 +824,9 @@ export function mutateStore(fn:(store:AuthStore)=>T|Promise, retainedValue options?.finalizeResult?.(result, store); persist(store); if (scrubbedProviders.length > 0) scrubLegacyBackup(scrubbedProviders); + // A committed observer may establish ordering before the generic selection + // publication, but its failure can never turn a durable write into a reported failure. + try { options?.afterPersist?.(result); } catch { /* The authoritative write already committed. */ } for (const provider of changedProviders) publishAccountSelection(provider, "oauth"); return result; }finally{guard.release();}}, retainedValues, options?.waitMs); @@ -1224,6 +1230,39 @@ export type SetAccountPausedResult = | { status: "unchanged"; activeAccountId: string; activeAccountChanged: boolean } | { status: "not-found" }; +/** Serialize policy with refresh/removal; stale pre-wait selection proposals must retry. */ +export async function setAnthropicAccountThreshold( + accountId: string, + threshold: number | null, + options: { assertBeforePersist?: () => void } = {}, +): Promise { + const normalizedThreshold = threshold === null ? null : parseAnthropicAccountThreshold(threshold); + if (threshold !== null && normalizedThreshold === null) { + throw new Error("threshold must be an integer 0-100 or null"); + } + const result = await mutateStore(store => { + const set = store.anthropic; + const account = set?.accounts.find(row => row.id === accountId); + if (!set || !account) return { status: "not-found" as const }; + if ((account.autoSwitchThresholdOverride ?? null) === normalizedThreshold) return { status: "unchanged" as const }; + const before = accountSelection(set); + if (normalizedThreshold === null) delete account.autoSwitchThresholdOverride; + else account.autoSwitchThresholdOverride = normalizedThreshold; + set.selectionRevision = randomUUID(); + return { status: "updated" as const, before, after: accountSelection(set) }; + }, [accountId, normalizedThreshold], { assertBeforePersist: options.assertBeforePersist, afterPersist: result => { + if (result.status !== "updated") return; + // Publish the policy-owned transition before the generic selection event. This + // preserves an exact previous-revision manual intent without opening an ABA gap. + publishOAuthAccountRoutingPolicyChange(Object.freeze({ + provider: "anthropic", + before: Object.freeze(result.before), + after: Object.freeze(result.after), + })); + } }); + return result.status !== "not-found"; +} + /** Persist an operator pause and move an active account to the next usable unpaused slot when available. */ export async function setAccountPaused( provider: string, @@ -1323,6 +1362,7 @@ export async function replaceProviderAccountSet( ...(account.alias ? { alias: account.alias } : {}), ...(account.needsReauth ? { needsReauth: true } : {}), ...(account.paused ? { paused: true } : {}), + ...(account.autoSwitchThresholdOverride !== undefined ? { autoSwitchThresholdOverride: account.autoSwitchThresholdOverride } : {}), ...(account.addedAt !== undefined ? { addedAt: account.addedAt } : {}), ...(account.loginId ? { loginId: account.loginId } : {}), })), diff --git a/src/oauth/types.ts b/src/oauth/types.ts index 73af11f0a63..f53139bd595 100644 --- a/src/oauth/types.ts +++ b/src/oauth/types.ts @@ -90,6 +90,8 @@ export interface ProviderAccount { needsReauth?: boolean; /** Operator exclusion from generic OAuth account selection until explicitly resumed. */ paused?: boolean; + /** Anthropic-only usage-switch override; absent inherits its pool default, zero disables it. */ + autoSwitchThresholdOverride?: number; addedAt?: number; } diff --git a/src/server/management/anthropic-account-threshold.ts b/src/server/management/anthropic-account-threshold.ts new file mode 100644 index 00000000000..4c0e5a368c8 --- /dev/null +++ b/src/server/management/anthropic-account-threshold.ts @@ -0,0 +1,27 @@ +import type { OcxConfig } from "../../types"; +import { effectiveAnthropicAccountThreshold, parseAnthropicAccountThreshold } from "../../oauth/anthropic-account-threshold"; +import { OAUTH_PROVIDERS } from "../../oauth"; +import { setAnthropicAccountThreshold } from "../../oauth/store"; +import { jsonResponse } from "../auth-cors"; +import { readManagementJsonBodyOr } from "./body"; + +/** Account-owned policy writes share the auth-store lock, not a config/auth split transaction. */ +export async function handleAnthropicAccountThreshold(req: Request, config: OcxConfig): Promise { + const body = await readManagementJsonBodyOr(req, {}); + if (!body || typeof body !== "object" || Array.isArray(body)) return jsonResponse({ error: "body must be an object" }, 400); + const fields = body as Record; + const definition = OAUTH_PROVIDERS.anthropic; + const effectiveProvider = config.providers.anthropic + ?? definition?.resolveProviderConfig?.(config) ?? definition?.providerConfig; + if (fields.provider !== "anthropic" || effectiveProvider?.authMode !== "oauth") { + return jsonResponse({ error: "account threshold requires Anthropic OAuth" }, 400); + } + if (typeof fields.accountId !== "string" || !fields.accountId.trim()) return jsonResponse({ error: "missing accountId" }, 400); + const threshold = parseAnthropicAccountThreshold(fields.threshold); + if (fields.threshold !== null && threshold === null) return jsonResponse({ error: "threshold must be an integer 0-100 or null" }, 400); + if (!await setAnthropicAccountThreshold(fields.accountId, threshold)) return jsonResponse({ error: "account not found" }, 404); + return jsonResponse({ ok: true, provider: "anthropic", accountId: fields.accountId, + autoSwitchThresholdOverride: threshold, + effectiveAutoSwitchThreshold: effectiveAnthropicAccountThreshold(config, { autoSwitchThresholdOverride: threshold ?? undefined }), + autoSwitchThreshold: effectiveAnthropicAccountThreshold(config) }); +} diff --git a/src/server/management/oauth-account-routes.ts b/src/server/management/oauth-account-routes.ts index 836105e1817..f2ae4aef429 100644 --- a/src/server/management/oauth-account-routes.ts +++ b/src/server/management/oauth-account-routes.ts @@ -1,4 +1,6 @@ import { parseAnthropicModelRoutes, readAnthropicModelRoutes } from "../../oauth/anthropic-model-routes"; +import { effectiveAnthropicAccountThreshold } from "../../oauth/anthropic-account-threshold"; +import { handleAnthropicAccountThreshold } from "./anthropic-account-threshold"; import { randomBytes, randomUUID } from "node:crypto"; import { readFileSync } from "node:fs"; import type { CatalogModel } from "../../codex/catalog"; @@ -418,6 +420,9 @@ export async function handleOauthAccountRoutes(ctx: ManagementContext): Promise< }); return { ...summary, ...oauthAccountHealthFields(provider, summary.id, health), quotaMode, ...(supportsPause ? { paused: full?.paused === true } : {}), + ...(provider === "anthropic" && supportsPause ? { autoSwitchThresholdOverride: full?.autoSwitchThresholdOverride ?? null, + effectiveAutoSwitchThreshold: effectiveAnthropicAccountThreshold(config, full), + autoSwitchThreshold: effectiveAnthropicAccountThreshold(config) } : {}), ...(provider === "kiro" && full ? kiroAutoSelection(full) : {}) }; }), }; @@ -493,6 +498,7 @@ export async function handleOauthAccountRoutes(ctx: ManagementContext): Promise< return jsonResponse({ ok: true, provider, activeAccountId: body.accountId }); } + if (url.pathname === "/api/oauth/accounts/auto-switch" && req.method === "PUT") return handleAnthropicAccountThreshold(req, config); if (url.pathname === "/api/oauth/accounts/pause" && req.method === "PUT") { const body = await readManagementJsonBodyOr(req, {}); if (!isPlainRecord(body)) return jsonResponse({ error: "body must be an object" }, 400); diff --git a/src/server/management/route-registry.ts b/src/server/management/route-registry.ts index a59319dd1bb..f41692503fe 100644 --- a/src/server/management/route-registry.ts +++ b/src/server/management/route-registry.ts @@ -323,6 +323,7 @@ export const MANAGEMENT_ROUTES: readonly ManagementRoute[] = [ { method: "PUT", path: "/api/oauth/accounts/active", module: "server/management/oauth-account-routes", mutates: true }, { method: "PUT", path: "/api/oauth/accounts/alias", module: "server/management/oauth-account-routes", mutates: true }, { method: "PUT", path: "/api/oauth/accounts/pause", module: "server/management/oauth-account-routes", mutates: true }, + { method: "PUT", path: "/api/oauth/accounts/auto-switch", module: "server/management/oauth-account-routes", mutates: true }, { method: "PUT", path: "/api/oauth/accounts/pool", module: "server/management/oauth-account-routes", mutates: true }, { method: "PUT", path: "/api/providers/keys/active", module: "server/management/oauth-account-routes", mutates: true }, { method: "PUT", path: "/api/providers/keys/alias", module: "server/management/oauth-account-routes", mutates: true }, diff --git a/src/server/responses/request-transport.ts b/src/server/responses/request-transport.ts index c340c0612a9..ebabd087625 100644 --- a/src/server/responses/request-transport.ts +++ b/src/server/responses/request-transport.ts @@ -193,9 +193,13 @@ export async function prepareResponsesTransport( // Resolve that choice, not the rejected candidate, before trying admission again. oauthSelection = captureOAuthAccountSelection(route.providerName); if (!oauthSelection) return null; - const revisedAnthropicId = route.providerName === "anthropic" && anthropicRouteDecision - ? resolveAnthropicAccountForSession(anthropicSessionKey, config, Date.now(), anthropicRouteDecision).accountId : null; - if (route.providerName === "anthropic" && anthropicRouteDecision && !revisedAnthropicId) return null; + // A revision also changes on per-account policy edits. Re-evaluate the selector + // after credential waits even without a model route, rather than reusing stale active. + const revisedAnthropic = route.providerName === "anthropic" + ? resolveAnthropicAccountForSession(anthropicSessionKey, config, Date.now(), anthropicRouteDecision) : null; + const revisedAnthropicId = revisedAnthropic?.accountId; + if (route.providerName === "anthropic" && !revisedAnthropicId) return null; + if (revisedAnthropic) anthropicReason = revisedAnthropic.reason; candidate = route.providerName === "anthropic" ? await getAnthropicPoolAccessSnapshot(revisedAnthropicId ?? oauthSelection.accountId) : await getValidAccessSnapshotForAccount(route.providerName, oauthSelection.accountId, { requireUsableAccount: true }); diff --git a/structure/INDEX.md b/structure/INDEX.md index 2f5e29efc0a..53f9e34cebd 100644 --- a/structure/INDEX.md +++ b/structure/INDEX.md @@ -56,6 +56,7 @@ Per-vendor contracts and the adapter authority that constructs them. | Doc | Scope | | --- | --- | +| [`providers/anthropic-account-thresholds.md`](providers/anthropic-account-thresholds.md) | Account-owned usage thresholds, inheritance, routing boundaries and durable policy changes. | | [`providers-and-adapters.md`](providers-and-adapters.md) | Provider and adapter selection, the adapter inventory, live model discovery, and the hosted-search continuation bridge. | | [`providers/anthropic-account-pool.md`](providers/anthropic-account-pool.md) | Anthropic OAuth account pause, model routes, and quota labels. | | [`providers/openai-tiers.md`](providers/openai-tiers.md) | Pool/Direct account modes, API-key separation, and the public provider and quota contract. | @@ -135,7 +136,7 @@ A source area can be described by more than one doc, because these docs are orga | `src/lab/` | [`runtime.md`](runtime.md)
[`adapters/compatibility-lab.md`](adapters/compatibility-lab.md) | | `src/lib/` | [`overview.md`](overview.md)
[`runtime.md`](runtime.md)
[`transports/byte-accounting.md`](transports/byte-accounting.md)
[`transports/responses-wire-shapes.md`](transports/responses-wire-shapes.md)
[`transports/responses-failover.md`](transports/responses-failover.md)
[`transports/responses-spend.md`](transports/responses-spend.md)
[`transports/inventory.md`](transports/inventory.md)
[`gui-and-management-api.md`](gui-and-management-api.md)
[`dashboard-and-usage.md`](dashboard-and-usage.md)
[`clients/integrations.md`](clients/integrations.md)
[`ops/service-and-sidecars.md`](ops/service-and-sidecars.md)
[`ops/docs-and-release.md`](ops/docs-and-release.md) | | `src/link/` | [`remote-link.md`](remote-link.md) | -| `src/oauth/` | [`runtime.md`](runtime.md)
[`transports/inventory.md`](transports/inventory.md)
[`providers-and-adapters.md`](providers-and-adapters.md)
[`providers/anthropic-account-pool.md`](providers/anthropic-account-pool.md)
[`providers/xai-grok.md`](providers/xai-grok.md) | +| `src/oauth/` | [`runtime.md`](runtime.md)
[`transports/inventory.md`](transports/inventory.md)
[`providers/anthropic-account-thresholds.md`](providers/anthropic-account-thresholds.md)
[`providers-and-adapters.md`](providers-and-adapters.md)
[`providers/anthropic-account-pool.md`](providers/anthropic-account-pool.md)
[`providers/xai-grok.md`](providers/xai-grok.md) | | `src/plugins/` | [`ops/plugins.md`](ops/plugins.md) | | `src/protocols/` | [`data-planes/protocol-paths.md`](data-planes/protocol-paths.md) | | `src/providers/` | [`runtime.md`](runtime.md)
[`subagents.md`](subagents.md)
[`transports/inventory.md`](transports/inventory.md)
[`providers-and-adapters.md`](providers-and-adapters.md)
[`providers/anthropic-account-pool.md`](providers/anthropic-account-pool.md)
[`providers/xai-grok.md`](providers/xai-grok.md) | diff --git a/structure/decisions/ADR-6014-anthropic-account-threshold.md b/structure/decisions/ADR-6014-anthropic-account-threshold.md new file mode 100644 index 00000000000..072e8d00374 --- /dev/null +++ b/structure/decisions/ADR-6014-anthropic-account-threshold.md @@ -0,0 +1,49 @@ +# ADR-6014 — decision recorded under "Anthropic account thresholds" + +- Contract owner: [Anthropic account thresholds](../providers/anthropic-account-thresholds.md) + +## Decision Log + +- Purpose and intent: Give individual Claude subscriptions an optional usage-switch policy, + completing the second vertical slice of issue #6013 without changing manual pause semantics. +- Existing implementation and constraints: Anthropic owns its active/manual/affinity selector, + three strategies and three quota windows. Model routes constrain the roster. The first slice + stores pause on the OAuth account; credentials, policy changes and deletion must serialize. +- Alternatives considered: A config-side account-id map mirrors Codex but creates an auth/config + split write and orphan cleanup problem. Hard eligibility would strand all-drained or unknown + quota requests. Reusing the generic pool threshold would change every account. +- Selected approach: Store optional `autoSwitchThresholdOverride` on the protected OAuth row. + Missing/null inherits the current Anthropic pool default; validate integer 0..100, retaining + concrete zero as usage-driven switching disabled. The auth mutation lock bumps selection + revision on changes, rejecting old admission proposals without touching credentials or health. + A post-persistence policy event carries the exact previous/current selection revisions and runs + before the generic selection publication. It rebases only a one-shot manual preference that + still owns the previous revision; every ordinary selection generation clears stale ownership, + including A→B→A. Re-login and refresh preserve metadata; account/provider deletion owns cleanup. +- Why this approach: One account-owned record provides atomic lifetime and restart persistence. + The existing cache remains the freshness/unknown authority. Known below-threshold candidates + are preferred using their own policy, but usage never makes an account ineligible. + Manual/affinity and identity-less strategy fast paths keep their existing priority. Quota and + fill-first use policy with pooling enabled; round-robin and pool-off recovery stay unchanged. +- Benefits, tradeoffs, and impact: No config migration or additional secret store. Account DTOs + expose override/default/effective values without credentials. The dedicated PUT, explicit CLI + `--account` and reused compact card control share the contract. Same-provider mutation ownership + and roster generations protect GUI reads. GET and PUT both use the built-in Anthropic OAuth + definition when the explicit provider row is absent, and each PUT response projects the value + committed by that request instead of re-reading a later concurrent write. Focus/draft behavior + uses existing React components with a localized Anthropic hint. The confirmed pool default lives + in the shared account-roster state: settings reads/saves invalidate older roster generations and + start a fresh read, so late responses cannot overwrite a save while later external changes remain + observable. Per-account writes use the roster client's bounded, abortable request lifecycle, so an + unmount or API-base change cannot leave mutation ownership locked. Additional auth-store reads occur + at selection boundaries. + Thresholds are soft preferences, not spend caps; admitted/sent requests are not cancelled. + +## Verification + +Focused tests cover every strategy/window, zero/inheritance, unknown/reset-expired quotas, +route scope, pool-off recovery, generation invalidation, active and non-active policy edits after +a manual selection, observer ordering, A→B→A and same-id generations, rejected persistence, +successive policy edits, refresh/pause races, restart and deletion in +`tests/adapters/anthropic/anthropic-account-threshold.test.ts`. CLI and mounted GUI tests verify +surface parity. Validation uses isolated homes, not the live proxy. diff --git a/structure/gui-and-management-api.md b/structure/gui-and-management-api.md index 384fa6fa2cb..fb6136f84d3 100644 --- a/structure/gui-and-management-api.md +++ b/structure/gui-and-management-api.md @@ -1,5 +1,17 @@ # GUI And Management API +Anthropic OAuth account DTOs include `autoSwitchThresholdOverride` (integer or null), +`autoSwitchThreshold` (pool default) and `effectiveAutoSwitchThreshold`. The dedicated +`PUT /api/oauth/accounts/auto-switch` accepts `{ provider: "anthropic", accountId, threshold }`; +explicit null restores inheritance and missing/invalid values fail. Other providers are rejected. +`src/server/management/anthropic-account-threshold.ts` validates; the auth store serializes writes. +CLI `ocx account auto-switch anthropic` requires `--account ` with status, inherit, on, off or +threshold. The dashboard reuses `AccountAutoSwitchControl` below account actions, retaining +focus/draft semantics and translated copy. The hook protects same-provider selection mutations +and stale roster reads; confirmed pool-setting changes seed new overrides immediately, without +overwriting an existing custom draft. Old servers do not show a synthetic control. See +[Anthropic threshold semantics](providers/anthropic-account-thresholds.md). + Anthropic account rows now expose the shared boolean `paused` DTO and use the existing `PUT /api/oauth/accounts/pause` body `{ provider, accountId, paused }`. The dashboard's `ProviderAuthPanel` and `useProviderAccountPools` reuse the translated pause/resume actions, diff --git a/structure/manifest.json b/structure/manifest.json index d200b1c1e29..533183c97bd 100644 --- a/structure/manifest.json +++ b/structure/manifest.json @@ -267,6 +267,13 @@ "src/protocols/" ] }, + { + "path": "providers/anthropic-account-thresholds.md", + "tier": 4, + "title": "Anthropic Account Thresholds", + "scope": "Account-owned usage thresholds, inheritance, routing boundaries and durable policy changes.", + "documents": ["src/oauth/"] + }, { "path": "providers-and-adapters.md", "tier": 4, diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index 3fee43be69d..a96f7c56e5e 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -3,6 +3,7 @@ Anthropic account pause, model routes, and quota labels follow the [Anthropic account-pool contract](providers/anthropic-account-pool.md). +Per-account usage thresholds follow the [Anthropic account thresholds contract](providers/anthropic-account-thresholds.md). An Anthropic 429 records the served account's cooldown even when the request has used its allowed retry sends. That final account remains excluded on the next request; combo target cooling is skipped only after the matching account cooldown is present. GitHub Copilot `modelContextTiers` is selected per upstream model. The Chat and Responses diff --git a/structure/providers/anthropic-account-thresholds.md b/structure/providers/anthropic-account-thresholds.md new file mode 100644 index 00000000000..813aaf55dc9 --- /dev/null +++ b/structure/providers/anthropic-account-thresholds.md @@ -0,0 +1,31 @@ +# Anthropic Account Thresholds + +`src/oauth/anthropic-account-threshold.ts` resolves `ProviderAccount.autoSwitchThresholdOverride` +against `anthropicAccountPool.autoSwitchThreshold` (default 80). Stored integers 0..100 survive +refresh, re-login and restart. Missing/null inherits; malformed disk values normalize to absent. +`setAnthropicAccountThreshold` in `src/oauth/store.ts` serializes writes with pause, refresh and +deletion and advances selection revision when policy changes. Account deletion removes it. +`src/server/responses/request-transport.ts` re-evaluates Anthropic selection after a revision +conflict during credential resolution, with or without a model route, before physical dispatch. +The policy-only post-persistence signal runs before the generic selection event and carries exact +previous/current revisions. A pending one-shot manual choice may adopt the new revision only while +it still owns the previous one; this fences old automatic proposals without silently discarding +operator intent. Ordinary selection publications clear stale ownership, including A→B→A and +same-account revision replacement. + +Management reads and writes resolve Anthropic OAuth eligibility through the same configured-or- +built-in provider definition, so a separately persisted account store remains editable when the +explicit provider row is absent. A successful write returns its validated committed override and +derived effective value directly; it never re-reads a newer concurrent mutation into the response. + +`src/oauth/anthropic-routing.ts` compares quota and fill-first source/candidates against each +account's effective threshold. Zero disables usage-driven switching for that account, including +the weekly five-hour exhaustion guard. Unknown source usage does not force a switch. Existing +reset-aware last-good normalization remains the freshness authority. Known below-threshold +candidates are preferred; unknown and all-drained sets retain the legacy ranking/fallback. +Thresholds are preferences, not exclusions: model routes do not widen merely because candidates +are drained. Manual/affinity and identity-less RR/fill-first priorities remain unchanged; +round-robin is not usage-driven. With pooling disabled, reactive recovery ignores stored policy. +Pause, reauthentication, cooldown and credential admission continue to take precedence over zero. + +> Decision record: [ADR-6014](../decisions/ADR-6014-anthropic-account-threshold.md) diff --git a/tests/adapters/anthropic/anthropic-account-threshold.test.ts b/tests/adapters/anthropic/anthropic-account-threshold.test.ts new file mode 100644 index 00000000000..cfe603ca8d0 --- /dev/null +++ b/tests/adapters/anthropic/anthropic-account-threshold.test.ts @@ -0,0 +1,312 @@ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { mkdtempSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { clearPoolRotationState } from "../../../src/codex/pool-rotation"; +import { subscribeAccountSelections } from "../../../src/lib/account-selection-events"; +import { effectiveAnthropicAccountThreshold, parseAnthropicAccountThreshold } from "../../../src/oauth/anthropic-account-threshold"; +import { bindAnthropicSessionAffinity, clearAnthropicAccountPoolState, promoteAnthropicActiveAccount, + resetAnthropicRoutingForManualSelection, resolveAnthropicAccountForSession, rotateAnthropicAccountOn429 } from "../../../src/oauth/anthropic-routing"; +import { captureOAuthAccountSelection, getAccountCredential, getAccountSet, removeAccount, replaceProviderAccountSet, + saveAccountCredential, saveCredential, setAccountPaused, setActiveAccount, setAnthropicAccountThreshold } from "../../../src/oauth/store"; +import { clearAccountQuotaCache, setCachedProviderAccountQuotaForTests } from "../../../src/providers/quota"; +import type { OcxConfig, OcxAccountPoolQuotaWindow, OcxAccountPoolRotationStrategy } from "../../../src/types"; +import { removeTreeWithRetry } from "../../helpers/remove-tree"; +import { handleAnthropicAccountThreshold } from "../../../src/server/management/anthropic-account-threshold"; +import { handleOauthAccountRoutes } from "../../../src/server/management/oauth-account-routes"; +import type { ManagementContext } from "../../../src/server/management/context"; + +const oldHome = process.env.OPENCODEX_HOME; +let home: string; +let ids: [string, string, string]; +function config(strategy: OcxAccountPoolRotationStrategy = "quota", quotaWindow: OcxAccountPoolQuotaWindow = "five-hour", enabled = true): OcxConfig { + return { port: 0, defaultProvider: "anthropic", providers: { + anthropic: { adapter: "anthropic", baseUrl: "https://api.anthropic.com", authMode: "oauth" }, + }, anthropicAccountPool: { enabled, strategy, quotaWindow, autoSwitchThreshold: 80 } }; +} +function quota(id: string, percent: number, resetAt = Date.now() + 3600_000) { + setCachedProviderAccountQuotaForTests("anthropic", id, { fiveHourPercent: percent, weeklyPercent: percent, + fiveHourResetAt: resetAt, weeklyResetAt: resetAt, updatedAt: Date.now() }); +} +beforeEach(async () => { + home = mkdtempSync(join(tmpdir(), "ocx-anthropic-threshold-")); process.env.OPENCODEX_HOME = home; + clearAnthropicAccountPoolState(); clearPoolRotationState(); clearAccountQuotaCache(); + for (let i = 0; i < 3; i++) await saveCredential("anthropic", { access: `synthetic-${i}`, refresh: `synthetic-refresh-${i}`, + expires: Date.now() + 3600_000, accountId: `threshold-${i}` }); + ids = getAccountSet("anthropic")!.accounts.map(row => row.id).sort() as typeof ids; + await setActiveAccount("anthropic", ids[0]); + const pick = resolveAnthropicAccountForSession("setup", config()); + await promoteAnthropicActiveAccount(pick.accountId!, captureOAuthAccountSelection("anthropic"), { config: config(), reason: pick.reason }); +}); +afterEach(() => { + clearAnthropicAccountPoolState(); clearPoolRotationState(); clearAccountQuotaCache(); + if (oldHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = oldHome; + removeTreeWithRetry(home); +}); + +test("inheritance, null reset, zero and strict integer range", async () => { + for (const invalid of [undefined, null, "80", true, [], {}, -1, 101, 2.5, NaN, Infinity]) expect(parseAnthropicAccountThreshold(invalid)).toBeNull(); + for (const valid of [0, 1, 80, 100]) expect(parseAnthropicAccountThreshold(valid)).toBe(valid); + const cfg = config(); + expect(effectiveAnthropicAccountThreshold({ ...cfg, anthropicAccountPool: {} })).toBe(80); + await setAnthropicAccountThreshold(ids[0], 0); + expect(effectiveAnthropicAccountThreshold(cfg, getAccountSet("anthropic")!.accounts.find(row => row.id === ids[0]))).toBe(0); + await setAnthropicAccountThreshold(ids[0], null); cfg.anthropicAccountPool!.autoSwitchThreshold = 63; + expect(effectiveAnthropicAccountThreshold(cfg, getAccountSet("anthropic")!.accounts.find(row => row.id === ids[0]))).toBe(63); + expect(await setAnthropicAccountThreshold("missing", 50)).toBe(false); +}); + +for (const strategy of ["quota", "fill-first"] as const) for (const window of ["five-hour", "weekly", "max-utilization"] as const) { + test(`${strategy}/${window}: source and successor use their own thresholds`, async () => { + const [a, b, c] = ids; const cfg = config(strategy, window); + quota(a, 60); quota(b, 30); quota(c, 70); + await setAnthropicAccountThreshold(a, 50); await setAnthropicAccountThreshold(b, 20); await setAnthropicAccountThreshold(c, 90); + expect(resolveAnthropicAccountForSession("new", cfg).accountId).toBe(c); + await setAnthropicAccountThreshold(a, 0); + expect(resolveAnthropicAccountForSession("new", cfg).accountId).toBe(a); + await setAnthropicAccountThreshold(a, 100); quota(a, 99); + expect(resolveAnthropicAccountForSession("new", cfg).accountId).toBe(a); + quota(a, 100); + expect(resolveAnthropicAccountForSession("new", cfg).accountId).toBe(c); + }); + test(`${strategy}/${window}: unknown and reset-expired source does not force switching`, async () => { + await setAnthropicAccountThreshold(ids[0], 1); quota(ids[1], 0); quota(ids[2], 0); + expect(resolveAnthropicAccountForSession("unknown", config(strategy, window)).accountId).toBe(ids[0]); + quota(ids[0], 100, Date.now() - 1000); + expect(resolveAnthropicAccountForSession("expired", config(strategy, window)).accountId).toBe(ids[0]); + }); +} + +for (const window of ["five-hour", "weekly", "max-utilization"] as const) test(`round-robin/${window} is not usage-driven`, async () => { + const cfg = config("round-robin", window); + const before = resolveAnthropicAccountForSession("unbound", cfg); + for (const id of ids) { quota(id, 99); await setAnthropicAccountThreshold(id, 1); } + expect(resolveAnthropicAccountForSession("unbound", cfg)).toEqual(before); +}); + +test("manual, affinity and identity-less strategy priorities remain unchanged", async () => { + const [a, b] = ids; quota(a, 70); quota(b, 10); await setAnthropicAccountThreshold(a, 20); + bindAnthropicSessionAffinity("bound", a); + expect(resolveAnthropicAccountForSession("bound", config()).reason).toBe("affinity"); + for (const strategy of ["round-robin", "fill-first"] as const) expect(resolveAnthropicAccountForSession(null, config(strategy)).accountId).toBe(a); + await setActiveAccount("anthropic", a); resetAnthropicRoutingForManualSelection(a); + expect(resolveAnthropicAccountForSession("new", config())).toMatchObject({ accountId: a, reason: "manual" }); +}); + +test.each(["active", "non-active"] as const)("%s threshold edits preserve the pending manual dispatch", async target => { + const [a, b, c] = ids; + quota(a, 90); quota(b, 10); quota(c, 70); + await setActiveAccount("anthropic", a); + resetAnthropicRoutingForManualSelection(a); + + await setAnthropicAccountThreshold(target === "active" ? a : b, target === "active" ? 20 : 50); + const first = resolveAnthropicAccountForSession("manual-after-policy", config()); + expect(first).toMatchObject({ accountId: a, reason: "manual" }); + expect(await promoteAnthropicActiveAccount(a, captureOAuthAccountSelection("anthropic"), { + config: config(), sessionKey: "manual-after-policy", reason: first.reason, + })).not.toBeNull(); + + // The operator's one-shot intent is now consumed; the edited quota policy owns + // the next unbound session and moves traffic to the lower-usage account. + expect(resolveAnthropicAccountForSession("policy-after-manual", config())).toMatchObject({ accountId: b, reason: "lowest-usage" }); +}); + +test("policy ownership is visible before the generic selection event", async () => { + const [a, b, c] = ids; + quota(a, 90); quota(b, 10); quota(c, 70); + await setActiveAccount("anthropic", a); + resetAnthropicRoutingForManualSelection(a); + const observed: ReturnType[] = []; + const unsubscribe = subscribeAccountSelections(event => { + if (event.provider === "anthropic" && event.kind === "oauth") { + observed.push(resolveAnthropicAccountForSession("inside-selection-event", config())); + } + }); + try { + await setAnthropicAccountThreshold(b, 50); + } finally { + unsubscribe(); + } + expect(observed).toEqual([expect.objectContaining({ accountId: a, reason: "manual" })]); + expect(resolveAnthropicAccountForSession("after-selection-event", config())).toMatchObject({ accountId: a, reason: "manual" }); +}); + +test.each(["aba", "same-id"] as const)("ordinary %s revisions cannot be adopted by a later policy event", async transition => { + const [a, b, c] = ids; + quota(a, 90); quota(b, 10); quota(c, 70); + await setActiveAccount("anthropic", a); + resetAnthropicRoutingForManualSelection(a); + if (transition === "aba") await setActiveAccount("anthropic", b); + await setActiveAccount("anthropic", a); + await setAnthropicAccountThreshold(b, 50); + expect(resolveAnthropicAccountForSession(`after-${transition}`, config())).toMatchObject({ accountId: b, reason: "lowest-usage" }); +}); + +test("a consumed manual choice stays consumed across successive policy edits", async () => { + const [a, b, c] = ids; + quota(a, 90); quota(b, 10); quota(c, 70); + await setActiveAccount("anthropic", a); + resetAnthropicRoutingForManualSelection(a); + const manual = resolveAnthropicAccountForSession("consume-before-policy", config()); + expect(manual).toMatchObject({ accountId: a, reason: "manual" }); + expect(await promoteAnthropicActiveAccount(a, captureOAuthAccountSelection("anthropic"), { + config: config(), sessionKey: "consume-before-policy", reason: manual.reason, + })).not.toBeNull(); + await setAnthropicAccountThreshold(b, 50); + await setAnthropicAccountThreshold(c, 60); + expect(resolveAnthropicAccountForSession("after-consumed-policy", config())).toMatchObject({ accountId: b, reason: "lowest-usage" }); +}); + +test("a rejected policy persistence neither advances selection nor consumes manual intent", async () => { + const [a, b, c] = ids; + quota(a, 90); quota(b, 10); quota(c, 70); + await setActiveAccount("anthropic", a); + resetAnthropicRoutingForManualSelection(a); + const before = captureOAuthAccountSelection("anthropic"); + await expect(setAnthropicAccountThreshold(b, 50, { + assertBeforePersist: () => { throw new Error("synthetic threshold persist refusal"); }, + })).rejects.toThrow("synthetic threshold persist refusal"); + expect(captureOAuthAccountSelection("anthropic")).toEqual(before); + expect(resolveAnthropicAccountForSession("after-rejected-policy", config())).toMatchObject({ accountId: a, reason: "manual" }); +}); + +test("successive policy revisions preserve the current manual choice exactly once", async () => { + const [a, b, c] = ids; + quota(a, 90); quota(b, 10); quota(c, 70); + await setActiveAccount("anthropic", a); + resetAnthropicRoutingForManualSelection(a); + await setAnthropicAccountThreshold(b, 50); + await setAnthropicAccountThreshold(c, 60); + const manual = resolveAnthropicAccountForSession("after-two-policies", config()); + expect(manual).toMatchObject({ accountId: a, reason: "manual" }); + expect(await promoteAnthropicActiveAccount(a, captureOAuthAccountSelection("anthropic"), { + config: config(), sessionKey: "after-two-policies", reason: manual.reason, + })).not.toBeNull(); + expect(resolveAnthropicAccountForSession("after-two-policies-consumed", config())).toMatchObject({ accountId: b, reason: "lowest-usage" }); +}); + +test("all-drained fallback remains available; zero candidate stays usable", async () => { + const [a, b, c] = ids; quota(a, 90); quota(b, 20); quota(c, 50); + for (const id of ids) await setAnthropicAccountThreshold(id, 10); + expect(resolveAnthropicAccountForSession("new", config()).accountId).toBe(b); + await setAnthropicAccountThreshold(c, 0); + expect(resolveAnthropicAccountForSession("new", config()).accountId).toBe(c); +}); + +test("unknown successors do not displace the legacy measured fallback without a known under-threshold candidate", async () => { + quota(ids[0], 90); quota(ids[1], 95); + expect(resolveAnthropicAccountForSession("unknown-successor", config()).accountId).toBe(ids[0]); +}); + +test("zero remains available even with a measured exhausted five-hour window under weekly routing", async () => { + quota(ids[0], 90); quota(ids[1], 100); quota(ids[2], 95); + await setAnthropicAccountThreshold(ids[1], 0); + for (const strategy of ["quota", "fill-first"] as const) expect(resolveAnthropicAccountForSession("new", config(strategy, "weekly")).accountId).toBe(ids[1]); +}); + +test("strict routes stay closed even when drained; fallback widens only empty eligibility", async () => { + const [a, b, c] = ids; for (const id of ids) { quota(id, 90); await setAnthropicAccountThreshold(id, 10); } + quota(c, 0); + const route = { position: 1, accounts: [b, a], fallback: true }; + expect(resolveAnthropicAccountForSession("new", config(), Date.now(), route).accountId).toBe(b); + await setAccountPaused("anthropic", a, true); await setAccountPaused("anthropic", b, true); + expect(resolveAnthropicAccountForSession("new", config(), Date.now(), { ...route, fallback: false }).accountId).toBeNull(); + expect(resolveAnthropicAccountForSession("new", config(), Date.now(), route).accountId).toBe(c); +}); + +test("pool-off proactive and reactive recovery ignore stored per-account thresholds", async () => { + const [a, b, c] = ids; quota(a, 90); quota(b, 20); quota(c, 50); + await setAnthropicAccountThreshold(a, 1); await setAnthropicAccountThreshold(b, 1); await setAnthropicAccountThreshold(c, 0); + expect(resolveAnthropicAccountForSession("new", config("fill-first", "five-hour", false)).accountId).toBe(a); + expect(rotateAnthropicAccountOn429(config("fill-first", "five-hour", false), a, "60")).toBe(b); +}); + +test("pool-on fill-first 429 successors use candidate thresholds without escaping route", async () => { + const [a, b, c] = ids; quota(b, 40); quota(c, 70); + await setAnthropicAccountThreshold(b, 30); await setAnthropicAccountThreshold(c, 80); + expect(rotateAnthropicAccountOn429(config("fill-first"), a, "60", null, Date.now(), null, + { position: 1, accounts: [a, b, c], fallback: false })).toBe(c); +}); + +test("threshold generation rejects a pre-wait proposal, including non-active candidate edits", async () => { + const captured = captureOAuthAccountSelection("anthropic"); + await setAnthropicAccountThreshold(ids[1], 25); + expect(await promoteAnthropicActiveAccount(ids[1], captured, { config: config() })).toBeNull(); + expect(getAccountSet("anthropic")!.activeAccountId).toBe(ids[0]); +}); + +test("idempotence and ABA generations; queued deletion cannot recreate an account", async () => { + const a = ids[0]; + await setAnthropicAccountThreshold(a, 0); + const before = captureOAuthAccountSelection("anthropic"); + await setAnthropicAccountThreshold(a, 0); + expect(captureOAuthAccountSelection("anthropic")).toEqual(before); + await setAnthropicAccountThreshold(a, null); await setAnthropicAccountThreshold(a, 0); + expect(captureOAuthAccountSelection("anthropic")?.revision).not.toBe(before?.revision); + expect(await promoteAnthropicActiveAccount(a, before, { config: config() })).toBeNull(); + const result = await Promise.all([setAnthropicAccountThreshold(a, 30), removeAccount("anthropic", a), setAnthropicAccountThreshold(a, 90)]); + expect(result).toEqual([true, true, false]); + expect(getAccountSet("anthropic")!.accounts.some(row => row.id === a)).toBe(false); +}); + +test("refresh, pause, replacement and fresh-process reads retain threshold; deletion cleans it", async () => { + const a = ids[0]; const credential = getAccountCredential("anthropic", a)!; + await Promise.all([setAnthropicAccountThreshold(a, 0), saveAccountCredential("anthropic", a, { ...credential, access: "synthetic-rotated" }), setAccountPaused("anthropic", a, true)]); + let account = getAccountSet("anthropic")!.accounts.find(row => row.id === a)!; + expect(account).toMatchObject({ autoSwitchThresholdOverride: 0, paused: true, credential: { access: "synthetic-rotated" } }); + await replaceProviderAccountSet("anthropic", getAccountSet("anthropic")); + if (process.platform !== "win32") expect(statSync(join(home, "auth.json")).mode & 0o777).toBe(0o600); + const child = Bun.spawnSync([process.execPath, "-e", `import { getAccountSet } from './src/oauth/store.ts'; console.log(getAccountSet('anthropic').accounts.find(a => a.id === ${JSON.stringify(a)}).autoSwitchThresholdOverride);`], { cwd: process.cwd(), env: process.env }); + expect(child.exitCode).toBe(0); expect(child.stdout.toString().trim()).toBe("0"); + await removeAccount("anthropic", a); + expect(await setAnthropicAccountThreshold(a, 60)).toBe(false); + expect(getAccountSet("anthropic")!.accounts.some(row => row.id === a)).toBe(false); +}); + +test("invalid persisted metadata normalizes to inherited without migrating other account state", async () => { + const path = join(home, "auth.json"); const store = JSON.parse(readFileSync(path, "utf8")); + store.anthropic.accounts[0].autoSwitchThresholdOverride = "0"; + writeFileSync(path, JSON.stringify(store)); + expect(getAccountSet("anthropic")!.accounts[0]!.autoSwitchThresholdOverride).toBeUndefined(); +}); + +test("API validates provider, account, integer, missing and null; reads durable policy", async () => { + const put = (body: unknown, cfg = config()) => handleAnthropicAccountThreshold(new Request("http://localhost/api/oauth/accounts/auto-switch", { + method: "PUT", body: JSON.stringify(body), headers: { "content-type": "application/json" }, + }), cfg); + for (const threshold of [undefined, "50", -1, 101, 0.5, true, {}, []]) expect((await put({ provider: "anthropic", accountId: ids[0], threshold })).status).toBe(400); + for (const threshold of [0, 100, null]) { + const response = await put({ provider: "anthropic", accountId: ids[0], threshold }); + expect(response.status).toBe(200); expect(await response.json()).toMatchObject({ autoSwitchThresholdOverride: threshold, effectiveAutoSwitchThreshold: threshold ?? 80 }); + } + expect((await put({ provider: "kiro", accountId: ids[0], threshold: 30 })).status).toBe(400); + expect((await put({ provider: "anthropic", accountId: "missing", threshold: 30 })).status).toBe(404); + const cfg = config(); cfg.providers.anthropic!.authMode = "api-key"; + expect((await put({ provider: "anthropic", accountId: ids[0], threshold: 30 }, cfg)).status).toBe(400); + const fallback = config(); delete fallback.providers.anthropic; + expect((await put({ provider: "anthropic", accountId: ids[0], threshold: 30 }, fallback)).status).toBe(200); +}); + +test("concurrent API writes each report its commit without assuming queue order", async () => { + const put = (threshold: number) => handleAnthropicAccountThreshold(new Request("http://localhost/api/oauth/accounts/auto-switch", { + method: "PUT", body: JSON.stringify({ provider: "anthropic", accountId: ids[0], threshold }), + headers: { "content-type": "application/json" }, + }), config()); + const [first, second] = await Promise.all([put(30), put(70)]); + expect(await first.json()).toMatchObject({ autoSwitchThresholdOverride: 30, effectiveAutoSwitchThreshold: 30 }); + expect(await second.json()).toMatchObject({ autoSwitchThresholdOverride: 70, effectiveAutoSwitchThreshold: 70 }); + expect([30, 70]).toContain(getAccountSet("anthropic")!.accounts.find(row => row.id === ids[0])?.autoSwitchThresholdOverride); +}); + +test("management dispatcher exposes the saved override/default/effective DTO without credentials", async () => { + const cfg = config(); + const call = (req: Request) => handleOauthAccountRoutes({ req, url: new URL(req.url), config: cfg, deps: {} } as ManagementContext); + const put = new Request("http://localhost/api/oauth/accounts/auto-switch", { method: "PUT", body: JSON.stringify({ provider: "anthropic", accountId: ids[0], threshold: 0 }) }); + expect((await call(put))?.status).toBe(200); + cfg.anthropicAccountPool!.autoSwitchThreshold = 60; + const response = await call(new Request("http://localhost/api/oauth/accounts?provider=anthropic")); + const dto = await response!.json(); + expect(dto.accounts.find((row: { id: string }) => row.id === ids[0])).toMatchObject({ autoSwitchThresholdOverride: 0, effectiveAutoSwitchThreshold: 0, autoSwitchThreshold: 60 }); + expect(dto.accounts.find((row: { id: string }) => row.id === ids[1])).toMatchObject({ autoSwitchThresholdOverride: null, effectiveAutoSwitchThreshold: 60, autoSwitchThreshold: 60 }); + expect(JSON.stringify(dto)).not.toContain("synthetic"); +}); diff --git a/tests/adapters/anthropic/anthropic-model-routes.test.ts b/tests/adapters/anthropic/anthropic-model-routes.test.ts index faaae0d123f..3eb71e9050b 100644 --- a/tests/adapters/anthropic/anthropic-model-routes.test.ts +++ b/tests/adapters/anthropic/anthropic-model-routes.test.ts @@ -1,4 +1,6 @@ -import { afterEach, beforeEach, expect, test } from "bun:test"; +import { afterEach, beforeEach, expect, spyOn, test } from "bun:test"; +import { OAUTH_PROVIDERS } from "../../../src/oauth"; +import { getAccountCredential, setAnthropicAccountThreshold } from "../../../src/oauth/store"; import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -92,6 +94,25 @@ test("bounded first-match globs and invalid rules", () => { expect(parseAnthropicModelRoutes([{ name: "a", match: "[bad]", accounts: ["a"] }]).ok).toBe(false); }); +test.each([false, true])("threshold edit during credential refresh reselects before send (route=%s)", async routed => { + const ids = await seed(); const [a, b, c] = ids as [string, string, string]; + const cfg = config(ids, () => answer()); + cfg.anthropicAccountPool = { enabled: true, strategy: "quota", ...(routed ? { routes: [{ name: "all", match: "claude-*", accounts: ids }] } : {}) }; + const initial = resolveAnthropicAccountForSession(null, cfg); + await promoteAnthropicActiveAccount(initial.accountId!, captureOAuthAccountSelection("anthropic"), { config: cfg, reason: initial.reason }); + for (const [id, percent] of [[a, 60], [b, 70], [c, 90]] as const) setCachedProviderAccountQuotaForTests("anthropic", id, { fiveHourPercent: percent, updatedAt: Date.now() }); + const credential = getAccountCredential("anthropic", a)!; + await saveAccountCredential("anthropic", a, { ...credential, expires: Date.now() - 1 }); + const refresh = spyOn(OAUTH_PROVIDERS.anthropic!, "refresh").mockImplementation(async () => { + await setAnthropicAccountThreshold(a, 50); + return { ...credential, expires: Date.now() + 3600_000 }; + }); + try { + const response = await post(cfg); expect(response.status).toBe(200); + expect(sends).toEqual(["Bearer synthetic-access-1"]); + } finally { refresh.mockRestore(); } +}); + test.each([true, false])("all-paused pool returns 403 without any send (enabled=%s)", async enabled => { const ids = await seed(); for (const id of ids) await setAccountPaused("anthropic", id, true); diff --git a/tests/cli/cli-account.test.ts b/tests/cli/cli-account.test.ts index 7772accc49d..211aac44d29 100644 --- a/tests/cli/cli-account.test.ts +++ b/tests/cli/cli-account.test.ts @@ -1089,13 +1089,13 @@ describe("ocx account CLI (issue #180 matrix)", () => { }); }); - test("21: auto-switch rejects wrong providers, invalid thresholds and missing providers", async () => { - const wrongProvider = await run(["auto-switch", "anthropic", "on"]); + test("21: auto-switch rejects missing Anthropic account selectors, invalid thresholds and missing providers", async () => { + const missingAnthropicAccount = await run(["auto-switch", "anthropic", "on"]); const invalidThreshold = await run(["auto-switch", "openai", "threshold", "101"]); const missingProvider = await run(["auto-switch"]); - expect(wrongProvider.code).toBe(1); - expect(wrongProvider.stderr).toContain("auto-switch only applies to the openai Codex account pool or a generic OAuth provider pool"); + expect(missingAnthropicAccount.code).toBe(2); + expect(missingAnthropicAccount.stderr).toContain("--account "); expect(invalidThreshold.code).toBe(1); expect(invalidThreshold.stderr).toContain("integer 0-100"); expect(missingProvider.code).toBe(1); diff --git a/tests/cli/cli-anthropic-account-threshold.test.ts b/tests/cli/cli-anthropic-account-threshold.test.ts new file mode 100644 index 00000000000..5b42b53e0ce --- /dev/null +++ b/tests/cli/cli-anthropic-account-threshold.test.ts @@ -0,0 +1,58 @@ +import { expect, test } from "bun:test"; +import { cmdAutoSwitch } from "../../src/cli/account-extended"; +import type { AccountDeps } from "../../src/cli/account-api"; + +test("Anthropic CLI writes explicit account policy and resets inheritance", async () => { + const calls: { path: string; body: unknown }[] = []; + const deps: AccountDeps = { baseUrl: "http://127.0.0.1:10100", + loadConfigImpl: () => ({ providers: { anthropic: { adapter: "anthropic", authMode: "oauth" } } }) as never, + fetchImpl: (async (url, init) => { + const body = JSON.parse(String(init?.body)); calls.push({ path: new URL(String(url)).pathname, body }); + return Response.json({ autoSwitchThresholdOverride: body.threshold, effectiveAutoSwitchThreshold: body.threshold ?? 80 }); + }) as typeof fetch }; + const log = console.log; const error = console.error; const output: string[] = []; + console.log = value => { output.push(String(value)); }; console.error = () => {}; + try { + for (const [action, value] of [["off", 0], ["on", 80], ["inherit", null]] as const) { + expect(await cmdAutoSwitch(["anthropic", action, "--account", "a", "--json"], deps)).toBe(0); + expect(JSON.parse(output.at(-1)!)).toMatchObject({ accountId: "a", autoSwitchThresholdOverride: value }); + expect(calls.at(-1)).toEqual({ path: "/api/oauth/accounts/auto-switch", body: { provider: "anthropic", accountId: "a", threshold: value } }); + } + expect(await cmdAutoSwitch(["anthropic", "threshold", "100", "--account", "a"], deps)).toBe(0); + const count = calls.length; + for (const args of [["threshold", "101"], ["threshold", "1.5"], ["threshold", "-1"], ["inherit", "extra"]]) { + expect(await cmdAutoSwitch(["anthropic", ...args, "--account", "a"], deps)).toBe(2); + } + expect(await cmdAutoSwitch(["anthropic", "off"], deps)).toBe(2); + expect(calls.length).toBe(count); + } finally { console.log = log; console.error = error; } +}); + +test("status is read-only and reports inheritance instead of guessing on an old proxy", async () => { + const calls: string[] = []; let supported = true; + const deps: AccountDeps = { baseUrl: "http://127.0.0.1:10100", + loadConfigImpl: () => ({ providers: { anthropic: { adapter: "anthropic", authMode: "oauth" } } }) as never, + fetchImpl: (async (_url, init) => { calls.push(init?.method ?? "GET"); return Response.json({ accounts: [{ id: "a", + ...(supported ? { autoSwitchThresholdOverride: null, effectiveAutoSwitchThreshold: 65 } : {}) }] }); }) as typeof fetch }; + const log = console.log; const error = console.error; const output: string[] = []; + console.log = value => { output.push(String(value)); }; console.error = () => {}; + try { + expect(await cmdAutoSwitch(["anthropic", "status", "--account", "a", "--json"], deps)).toBe(0); + expect(JSON.parse(output[0]!)).toMatchObject({ autoSwitchThresholdOverride: null, effectiveAutoSwitchThreshold: 65 }); + supported = false; + expect(await cmdAutoSwitch(["anthropic", "status", "--account", "a"], deps)).not.toBe(0); + expect(calls).toEqual(["GET", "GET"]); + } finally { console.log = log; console.error = error; } +}); + +test("malformed status responses fail without throwing or displaying an invented threshold", async () => { + const log = console.log; const error = console.error; console.log = () => {}; console.error = () => {}; + try { + for (const body of [null, [], {}, { accounts: [{ id: "a", autoSwitchThresholdOverride: null }] }]) { + const deps: AccountDeps = { baseUrl: "http://127.0.0.1:10100", + loadConfigImpl: () => ({ providers: { anthropic: { adapter: "anthropic", authMode: "oauth" } } }) as never, + fetchImpl: (async () => Response.json(body)) as typeof fetch }; + expect(await cmdAutoSwitch(["anthropic", "status", "--account", "a"], deps)).not.toBe(0); + } + } finally { console.log = log; console.error = error; } +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 5c52800bcbc..96dc540b2a4 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -109,7 +109,9 @@ "anthropic-account-pool.test.ts": "adapters/anthropic", "anthropic-account-pause-outbound.test.ts": "adapters/anthropic", "anthropic-account-pause.test.ts": "adapters/anthropic", + "anthropic-account-threshold.test.ts": "adapters/anthropic", "anthropic-combo-account-cooldown.test.ts": "adapters/anthropic", + "cli-anthropic-account-threshold.test.ts": "cli", "anthropic-model-routes.test.ts": "adapters/anthropic", "anthropic-agentrouter-language-framing.test.ts": "adapters/anthropic", "anthropic-baseurl-override.test.ts": "adapters/anthropic", From dad107c4ef9fcba4f4def70a72c9ca223384e836 Mon Sep 17 00:00:00 2001 From: ingwannu Date: Wed, 30 Sep 2026 05:40:18 +0900 Subject: [PATCH 23/26] fix(responses): support canonical non-streaming delivery (#6200) * fix(responses): support canonical nonstream delivery * fix(responses): fold canonical SSE for compaction clients * fix(responses): align buffered SSE failure semantics * fix(responses): cancel deferred replay on client disconnect * fix(responses): redact upstream terminal diagnostics before delivery * docs(responses): keep terminal contract within structure budget * fix(responses): redact full failed terminals and synthetic stream errors Delay buffered serving-route publication until deferred replay survives the final abort check. * fix(responses): mask credential in synthetic refusal code * test(responses): include terminal redaction in core owner inventory * fix(responses): mask pooled credential in buffered bare errors * test(responses): abort synchronously in the deferred-replay disconnect case --------- Co-authored-by: Ingwannu Co-authored-by: JUN --- .../docs/fr/reference/configuration/server.md | 2 +- .../docs/fr/reference/proxy-formats.md | 10 + .../docs/ja/reference/configuration/server.md | 2 +- .../docs/ja/reference/proxy-formats.md | 2 + .../docs/ko/reference/configuration/server.md | 2 +- .../docs/ko/reference/proxy-formats.md | 8 + .../docs/reference/configuration/server.md | 2 +- .../content/docs/reference/proxy-formats.md | 12 + .../docs/ru/reference/configuration/server.md | 2 +- .../docs/ru/reference/proxy-formats.md | 9 + .../docs/tr/reference/configuration/server.md | 2 +- .../docs/tr/reference/proxy-formats.md | 11 + .../zh-cn/reference/configuration/server.md | 2 +- .../docs/zh-cn/reference/proxy-formats.md | 7 + .../zh-tw/reference/configuration/server.md | 2 +- .../docs/zh-tw/reference/proxy-formats.md | 7 + scripts/test-layout/layout.json | 2 +- src/adapters/openai-responses/passthrough.ts | 10 + src/server/relay-eager.ts | 7 +- src/server/relay.ts | 350 ++++++-- src/server/responses/adapter-delivery.ts | 32 +- src/server/responses/buffered-sse-json.ts | 345 ++++++++ src/server/responses/core-codex-account.ts | 5 +- src/server/responses/core-lifetime.ts | 4 + src/server/responses/passthrough-delivery.ts | 315 ++++++- src/server/responses/passthrough-dispatch.ts | 64 +- src/server/responses/passthrough-error.ts | 4 +- .../responses/terminal-error-redaction.ts | 53 ++ .../decisions/ADR-6162-responses-http-sse.md | 12 + structure/providers-and-adapters.md | 2 +- structure/transports/responses-wire-shapes.md | 40 +- structure/transports/responses.md | 2 +- tests/fixtures/test-layout-expected.json | 1 + tests/helpers/codex-pool-retry.ts | 3 + tests/helpers/responses-core-source.ts | 2 + tests/responses/passthrough-abort.test.ts | 2 +- .../responses-canonical-nonstream.test.ts | 830 ++++++++++++++++++ .../responses-pool-401-refresh.test.ts | 88 ++ .../responses-preview-main-read-fence.test.ts | 71 +- tests/responses/sse-failed-tail.test.ts | 23 +- tests/server/server-auth.test.ts | 29 + tests/usage/request-log-nonstream.test.ts | 29 +- 42 files changed, 2226 insertions(+), 181 deletions(-) create mode 100644 src/server/responses/buffered-sse-json.ts create mode 100644 src/server/responses/terminal-error-redaction.ts create mode 100644 structure/decisions/ADR-6162-responses-http-sse.md create mode 100644 tests/responses/responses-canonical-nonstream.test.ts diff --git a/docs-site/src/content/docs/fr/reference/configuration/server.md b/docs-site/src/content/docs/fr/reference/configuration/server.md index 6fe894b567f..8dec2432524 100644 --- a/docs-site/src/content/docs/fr/reference/configuration/server.md +++ b/docs-site/src/content/docs/fr/reference/configuration/server.md @@ -14,7 +14,7 @@ exécute des fonctionnalités d'assistance autour des demandes du fournisseur. | `hostname?` | `string` | `"127.0.0.1"` | Adresse de liaison. Les liaisons hors bouclage nécessitent `OPENCODEX_API_AUTH_TOKEN`. | | `proxy?` | `string` | — | URL du proxy HTTP(S) ou SOCKS5 sortant (`socks5://host:port`) ou `${ENV_VAR}`. Les URL HTTP s’appliquent à `HTTP_PROXY` / `HTTPS_PROXY` si elles sont vides. Les URL SOCKS5 utilisent le tunnel SOCKS5 intégré et sont aussi exposées via `ALL_PROXY` (`ocx start --socks5`); `HTTP(S)_PROXY` héritées sont effacées dans ce processus. Le bouclage reste dans `NO_PROXY`. | | `emptyCompletionRetry?` | `boolean` | `false` | Active une nouvelle tentative Responses identique lorsqu’une réponse ne contient ni texte ni appel d’outil. Cette tentative peut être facturée. `OCX_EMPTY_COMPLETION_RETRY=0` la désactive sans modifier la configuration ; les combinaisons et les tours de compactage routés restent exclus. | -| `stallTimeoutSec?` | `number` | `300` (public) / désactivé (local) | Secondes sans progression utile en amont (Responses et Chat natif) avant la coupure du flux. Sans réglage, un amont **local** (loopback, privé ou nom `.local`/`.lan`) est désactivé par défaut et un amont public vaut 300 s ; une valeur positive s'applique aux deux (minimum 1 s) ; `0` désactive le watchdog partout. Les lectures de corps en attente de `/v1/responses/compact` partagent ce budget mais valent 300 s par défaut même pour un amont local ; une valeur explicite, y compris `0`, prime. | +| `stallTimeoutSec?` | `number` | `300` (public) / désactivé (local) | Secondes sans progression utile en amont (Responses et Chat natif) avant la coupure du flux. Sans réglage, un amont **local** (loopback, privé ou nom `.local`/`.lan`) est désactivé par défaut et un amont public vaut 300 s ; une valeur positive s'applique aux deux (minimum 1 s) ; `0` désactive partout le watchdog de silence. Pour Responses qui replie le SSE canonique ChatGPT en JSON non-streaming, un plafond total indépendant de 15 minutes subsiste même lorsque ce watchdog est désactivé. Les lectures de corps en attente de `/v1/responses/compact` partagent ce budget mais valent 300 s par défaut même pour un amont local ; une valeur explicite, y compris `0`, prime. | | `connectTimeoutMs?` | `number` | `200000` | Délai maximal par tentative pour DNS/TCP/TLS et les en-têtes finaux ; il prend fin avant la génération du corps. | | `shutdownTimeoutMs?` | `number` | `5000` | Délai de vidange gracieux avant l’annulation des tours actifs. | | `websockets?` | `boolean` | `false` | Annonce et autorise la route WebSocket Responses destinée aux clients. La valeur false maintient les clients sur HTTP/SSE ; elle ne désactive pas une optimisation WebSocket canonique admissible vers ChatGPT en amont. | diff --git a/docs-site/src/content/docs/fr/reference/proxy-formats.md b/docs-site/src/content/docs/fr/reference/proxy-formats.md index 4483284851b..3d47bfaeae4 100644 --- a/docs-site/src/content/docs/fr/reference/proxy-formats.md +++ b/docs-site/src/content/docs/fr/reference/proxy-formats.md @@ -75,6 +75,16 @@ Avec `stream: true`, la réponse est `text/event-stream`. Le pont émet des év Avec `stream: false` ou pas de `stream`, les mêmes événements d'adaptateur sont collectés dans une seule réponse JSON objet. Les deux formulaires préservent le modèle sélectionné, les éléments de sortie, l'état du terminal et l'utilisation. +La route canonique ChatGPT Codex n'accepte que SSE en amont ; seul l'appel amont utilise donc +`stream: true`. OpenCodex valide le flux terminal dans des limites bornées, puis le replie dans la +forme JSON demandée par le client sans modifier une valeur `store` explicite. Un échec de validation +renvoie une erreur plutôt qu'un JSON partiel avec HTTP 200. Les limites sont de 4 Mio par trame, +32 Mio pour le transcript et la source de reconstruction, 100 000 trames SSE et 10 000 éléments de +sortie reconstruits. `stallTimeoutSec` régit le premier octet du corps et les silences suivants. +Lorsqu'il vaut `0`, y compris par défaut pour un upstream local, il n'expire pas immédiatement : seul +le plafond indépendant de 15 minutes pour le tour mis en mémoire reste actif. Les clients streaming +restent inchangés. + Les trames SSE des réponses destinées au client sont limitées à 4 Mio par trame, mesuré en octets bruts avant la SSE délimiteur de bloc. Sur HTTP, une trame amont non terminée qui dépasse la limite échoue fermée avec un événement synthétique `response.failed` suivi de `data: [DONE]`. Sur les réponses WebSocket diff --git a/docs-site/src/content/docs/ja/reference/configuration/server.md b/docs-site/src/content/docs/ja/reference/configuration/server.md index 7eeac7e4776..c11d7af21c0 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/server.md +++ b/docs-site/src/content/docs/ja/reference/configuration/server.md @@ -14,7 +14,7 @@ description: リスナー、リモート アクセス、アドミッション | `proxy?` | `string` | — |送信 HTTP(S) または SOCKS5 プロキシ URL（`socks5://host:port`）または `${ENV_VAR}`。HTTP URL は未設定時のみ `HTTP_PROXY` / `HTTPS_PROXY` に適用されます。SOCKS5 URL は組み込みの SOCKS5 トンネルを使用し、`ALL_PROXY` にも適用されます（`ocx start --socks5`）。このプロセスで継承した `HTTP(S)_PROXY` はクリアされます。ループバックは `NO_PROXY` に残ります。 | | `emptyCompletionRetry?` | `boolean` | `false` | テキストもツール呼び出しもない Responses ターンを、ターミナルイベント前にストリームが終了した場合も含め、同一リクエストで 1 回再試行するよう明示的に有効化します。再試行は課金対象になる場合があります。`OCX_EMPTY_COMPLETION_RETRY=0` で設定を変更せず無効化できます。combo と routed-compaction turn は対象外です。 | | `dropCodexSafetyBuffering?` | `boolean` | `false` | Codex Responses パススルーから Codex の safety-buffering ヒントを除去します。対象は `x-codex-safety-buffering-enabled` / `x-codex-safety-buffering-faster-model` 応答ヘッダー、`safety_buffering` 型の `response.metadata` SSE イベント、およびその他の SSE イベントにある `safety_buffering` フィールドです。Codex TUI はこれらを、既定の操作でセッションをより弱いモデルに切り替える「より高速なモデルで再試行」プロンプトとして表示します。その他の `x-codex-*` ヘッダーと SSE イベントの内容は、そのフィールドの除去を除いて変更せずに転送されます。既定ではオフです。 | -| `stallTimeoutSec?` | `number` | `300`（public）/ 無効（local） | ストリームが遮断されるまでの、有効な上流進捗がない秒数（Responses とネイティブ Chat）。未設定では**ローカル**上流（loopback・プライベート・`.local`/`.lan` 名）は無効が既定、public 上流は 300 秒。正の値は両方に適用（最小 1 秒）、`0` で watchdog を全面無効化。`/v1/responses/compact` の保留ボディ読み取りもこの予算を共有するが、ローカル上流でも既定は 300 秒。明示値（`0` を含む）が優先される。 | +| `stallTimeoutSec?` | `number` | `300`（public）/ 無効（local） | ストリームが遮断されるまでの、有効な上流進捗がない秒数（Responses とネイティブ Chat）。未設定では**ローカル**上流（loopback・プライベート・`.local`/`.lan` 名）は無効が既定、public 上流は 300 秒。正の値は両方に適用（最小 1 秒）、`0` で無通信 watchdog を全面無効化。canonical ChatGPT SSE を非ストリーミング JSON にまとめる Responses リクエストには、watchdog が無効でも独立した 15 分の全体上限が残る。`/v1/responses/compact` の保留ボディ読み取りもこの予算を共有するが、ローカル上流でも既定は 300 秒。明示値（`0` を含む）が優先される。 | | `connectTimeoutMs?` | `number` | `200000` |試行ごとの DNS/TCP/TLS/最終ヘッダーの期限。本体が生成される前に終了します。 | | `shutdownTimeoutMs?` | `number` | `5000` |アクティブなターンが中止される前の正常な排出期限。 | | `websockets?` | `boolean` | `false` | クライアント向け Responses WebSocket パスを広告して許可します。false の場合クライアントは HTTP/SSE を使いますが、対象となる canonical ChatGPT upstream WS 最適化は無効にしません。 | diff --git a/docs-site/src/content/docs/ja/reference/proxy-formats.md b/docs-site/src/content/docs/ja/reference/proxy-formats.md index 7c5cbbb4277..db698fd90cb 100644 --- a/docs-site/src/content/docs/ja/reference/proxy-formats.md +++ b/docs-site/src/content/docs/ja/reference/proxy-formats.md @@ -63,6 +63,8 @@ provider events → internal adapter events → client dialect `stream: false` を指定するか、`stream` を指定しないと、同じアダプター イベントが 1 つの Responses JSON オブジェクトに収集されます。どちらの形式でも、選択したモデル、出力項目、端末の状態、使用状況が保存されます。 +canonical ChatGPT Codex ルートではアップストリームが SSE のみを受け付けるため、アップストリームへのリクエストだけを `stream: true` にします。OpenCodex は終端ストリームを制限内で検証し、クライアントが要求した JSON 形式へまとめます。明示された `store` は変更せず、検証に失敗した場合は不完全な JSON を HTTP 200 で返さずエラーにします。上限は 1 フレーム 4 MiB、transcript と再構築入力がそれぞれ 32 MiB、SSE フレーム 100,000 件、再構築される output item 10,000 件です。`stallTimeoutSec` は最初の body byte と以後の無通信時間の両方に適用されます。値が `0`、またはローカル upstream の既定値として無効な場合も即時には失効せず、独立した 15 分の全体上限だけが残ります。ストリーミング クライアントの動作は変わりません。 + クライアント向け Responses SSE フレームは、SSE ブロック区切りの前の生バイトで測って 1 フレームあたり 4 MiB に制限されます。HTTP では、区切りなしでこの上限を超えたアップストリーム フレームは、合成 `response.failed` イベントと続く `data: [DONE]` でフェイルクローズします。Responses WebSocket ブリッジでは、同じ条件で 502 `websocket_protocol_error` を送信し、アップストリーム リーダーをキャンセルします。完全な Responses 終端フレームがすでに到着している場合はそれが優先され、その後のサイズ超過または不正なバイトは、完了したターンをトランスポート障害に置き換えず破棄されます。 :::note diff --git a/docs-site/src/content/docs/ko/reference/configuration/server.md b/docs-site/src/content/docs/ko/reference/configuration/server.md index e166d1e50ba..ee24fd4f16e 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/server.md +++ b/docs-site/src/content/docs/ko/reference/configuration/server.md @@ -14,7 +14,7 @@ description: 리스너, 원격 접근, admission 키, 타임아웃, 저장소, | `proxy?` | `string` | — | 송신용 HTTP(S) 또는 SOCKS5 프록시 URL(`socks5://host:port`) 또는 `${ENV_VAR}`입니다. HTTP URL은 해당 변수가 비어 있을 때 `HTTP_PROXY` / `HTTPS_PROXY`에 적용됩니다. SOCKS5 URL은 내장 SOCKS5 터널을 사용하고 `ALL_PROXY`에도 적용되며(`ocx start --socks5`), 이 프로세스에서 상속된 `HTTP(S)_PROXY`를 지웁니다. 루프백은 `NO_PROXY`에 그대로 남습니다. | | `emptyCompletionRetry?` | `boolean` | `false` | 텍스트나 도구 호출이 없는 Responses 턴을, 터미널 이벤트 전에 스트림이 종료된 경우를 포함해 동일한 요청으로 한 번 재시도하도록 선택합니다. 재시도에는 비용이 발생할 수 있습니다. `OCX_EMPTY_COMPLETION_RETRY=0`은 설정을 바꾸지 않고 비활성화하며, combo 및 routed-compaction turn은 제외됩니다. | | `dropCodexSafetyBuffering?` | `boolean` | `false` | Canonical Codex Responses 응답의 선택적 safety-buffering 헤더 두 개와 SSE 힌트를 제거합니다. 공급자의 안전 정책이나 거절 응답은 바뀌지 않습니다. Native WS 메타데이터와 compact는 제외됩니다. | -| `stallTimeoutSec?` | `number` | `300`(public) / 비활성(local) | 스트림이 끊기기까지 유효한 업스트림 진행이 없는 시간(초, Responses 및 네이티브 Chat). 미설정 시 **로컬** 업스트림(loopback, private, `.local`/`.lan` 이름)은 비활성이 기본이고 공개 업스트림은 300초. 양수 값은 둘 다에 적용(최소 1초), `0`은 전면 비활성. `/v1/responses/compact`의 보류 바디 읽기도 이 예산을 공유하지만 로컬 업스트림에서도 기본은 300초. 명시 값(`0` 포함)이 우선한다. | +| `stallTimeoutSec?` | `number` | `300`(public) / 비활성(local) | 스트림이 끊기기까지 유효한 업스트림 진행이 없는 시간(초, Responses 및 네이티브 Chat). 미설정 시 **로컬** 업스트림(loopback, private, `.local`/`.lan` 이름)은 비활성이 기본이고 공개 업스트림은 300초. 양수 값은 둘 다에 적용(최소 1초), `0`은 무응답 watchdog을 전면 비활성화한다. canonical ChatGPT SSE를 비스트리밍 JSON으로 접는 Responses 요청에는 이 watchdog이 꺼져도 별도의 15분 전체 상한이 남는다. `/v1/responses/compact`의 보류 바디 읽기도 이 예산을 공유하지만 로컬 업스트림에서도 기본은 300초. 명시 값(`0` 포함)이 우선한다. | | `connectTimeoutMs?` | `number` | `200000` | 시도별 DNS/TCP/TLS/최종 헤더 기한입니다. 본문 생성 전에 끝납니다. | | `shutdownTimeoutMs?` | `number` | `5000` | 진행 중인 turn을 중단하기 전에 허용하는 정상 종료 드레인 기한입니다. | | `websockets?` | `boolean` | `false` | 클라이언트용 Responses WebSocket 경로를 광고하고 허용합니다. `false`이면 클라이언트는 HTTP/SSE를 사용하며, 적격 canonical ChatGPT 업스트림 WS 최적화는 비활성화하지 않습니다. | diff --git a/docs-site/src/content/docs/ko/reference/proxy-formats.md b/docs-site/src/content/docs/ko/reference/proxy-formats.md index 4848304cfbb..e93eb315644 100644 --- a/docs-site/src/content/docs/ko/reference/proxy-formats.md +++ b/docs-site/src/content/docs/ko/reference/proxy-formats.md @@ -73,6 +73,14 @@ deltas, 그리고 정확히 하나의 종료 `response.completed`, `response.fai `stream: false`이거나 `stream`이 없으면, 같은 adapter 이벤트가 하나의 Responses JSON 객체로 수집됩니다. 두 형식 모두 선택한 모델, output item, 종료 상태, usage를 보존합니다. +canonical ChatGPT Codex 경로는 upstream이 SSE만 받으므로 upstream 요청에만 `stream: true`를 사용합니다. +OpenCodex는 종료 스트림을 제한된 크기 안에서 검증한 뒤 클라이언트가 요청한 JSON 형태로 접습니다. +명시적인 `store` 값은 바꾸지 않으며, 검증에 실패하면 일부 JSON을 HTTP 200으로 반환하지 않고 오류로 +끝냅니다. 한도는 프레임당 4 MiB, transcript와 재구성 입력 각각 32 MiB, 100,000 SSE 프레임, 재구성 +output item 10,000개입니다. `stallTimeoutSec`는 첫 body byte와 이후 무응답 간격에 모두 적용됩니다. +값이 `0`이거나 로컬 upstream 기본값으로 비활성화된 경우 즉시 만료하지 않고, 독립된 15분 전체 상한만 +적용합니다. 스트리밍 클라이언트의 동작은 바뀌지 않습니다. + 클라이언트로 전달되는 Responses SSE 프레임은 SSE 블록 구분자 앞의 원시 바이트 기준으로 프레임당 4 MiB로 제한됩니다. HTTP에서는 구분자 없이 이 한도를 초과한 업스트림 프레임을 합성 `response.failed` 이벤트와 이어지는 `data: [DONE]`으로 fail closed 처리합니다. Responses WebSocket 브리지에서는 같은 조건에서 502 `websocket_protocol_error`를 보내고 업스트림 reader를 취소합니다. 완전한 Responses 종료 프레임이 이미 수신된 경우에는 그 종료가 우선하며, 이후의 과도한 크기 또는 잘못된 바이트는 완료된 턴을 전송 오류로 바꾸지 않고 버립니다. :::note diff --git a/docs-site/src/content/docs/reference/configuration/server.md b/docs-site/src/content/docs/reference/configuration/server.md index f0d92698dbf..8bbc6894c18 100644 --- a/docs-site/src/content/docs/reference/configuration/server.md +++ b/docs-site/src/content/docs/reference/configuration/server.md @@ -16,7 +16,7 @@ runs helper features around provider requests. | `noProxy?` | `string \| string[]` | — | Hosts that bypass `proxy`, merged with inherited `NO_PROXY` and loopback entries. A string may use comma-separated `NO_PROXY` syntax or `${ENV_VAR}`. | | `emptyCompletionRetry?` | `boolean` | `false` | Opt in to one identical Responses retry when a turn has no text or tool call, including a stream that ends before a terminal event. The retry may be billable. `OCX_EMPTY_COMPLETION_RETRY=0` disables it without changing config; combo and routed-compaction turns remain excluded. | | `dropCodexSafetyBuffering?` | `boolean` | `false` | Remove optional client-facing hints from canonical Codex Responses passthrough: the two `x-codex-safety-buffering-enabled` / `x-codex-safety-buffering-faster-model` response headers, `response.metadata` events whose metadata type is `safety_buffering`, and top-level `safety_buffering` fields. Other headers, response data, policy refusals and failures are preserved. This does not disable provider safety enforcement or upstream buffering. Native `codex.response.metadata.headers` WebSocket metadata and `/responses/compact` are outside this filter. | -| `stallTimeoutSec?` | `number` | `300` (public) / disabled (local) | Seconds without meaningful upstream progress (Responses and native Chat) before the stream is cut. Unset, a **local** upstream (loopback, private, or a `.local`/`.lan` name) defaults to disabled and a public upstream to 300 s; a positive value applies to both (minimum 1 s); `0` disables the watchdog everywhere. Disabled leaves a silent-but-healthy local model connected (keep-alives still flow). Pending `/v1/responses/compact` body reads share this budget but default to 300 s even for a local upstream — the route buffers the complete body while holding an active-turn lease — and an explicit value, including `0`, still wins. | +| `stallTimeoutSec?` | `number` | `300` (public) / disabled (local) | Seconds without meaningful upstream progress (Responses and native Chat) before the stream is cut. Unset, a **local** upstream (loopback, private, or a `.local`/`.lan` name) defaults to disabled and a public upstream to 300 s; a positive value applies to both (minimum 1 s); `0` disables the silence watchdog everywhere. Disabled leaves a silent-but-healthy local model connected (keep-alives still flow). Canonical ChatGPT Responses folded from SSE into non-streaming JSON retain a separate 15-minute whole-turn ceiling even when the silence watchdog is disabled. Pending `/v1/responses/compact` body reads share this budget but default to 300 s even for a local upstream — the route buffers the complete body while holding an active-turn lease — and an explicit value, including `0`, still wins. | | `oauthOpenBrowser?` | `boolean` | `true` | Whether a login may open a browser on the machine running the proxy. Absent and `true` both open, so an existing install is unchanged; only an explicit `false` declines. Decline when you need the authorization link in a different browser profile, or when the dashboard is not on the proxy's machine — the login still starts and the URL is still returned and displayed. `POST /api/oauth/login` and `POST /api/codex-auth/login` accept a per-request `openBrowser` boolean that overrides this, and the dashboard exposes the same choice beside the login button. Device-code flows never open a browser either way. | | `connectTimeoutMs?` | `number` | `200000` | Per-attempt DNS/TCP/TLS/final-header deadline; it ends before body generation. | | `shutdownTimeoutMs?` | `number` | `5000` | Graceful drain deadline before active turns are aborted. | diff --git a/docs-site/src/content/docs/reference/proxy-formats.md b/docs-site/src/content/docs/reference/proxy-formats.md index db7b765d6cd..bad29c7e258 100644 --- a/docs-site/src/content/docs/reference/proxy-formats.md +++ b/docs-site/src/content/docs/reference/proxy-formats.md @@ -212,6 +212,18 @@ With `stream: true`, the response is `text/event-stream`. The bridge emits Respo With `stream: false` or no `stream`, the same adapter events are collected into one Responses JSON object. Both forms preserve the selected model, output items, terminal status, and usage. +The canonical ChatGPT Codex route still uses `stream: true` on its upstream-only request because that +destination is SSE-only; OpenCodex boundedly validates and folds the terminal stream back into the +JSON shape the client requested. This transport coercion does not alter an explicit `store` value. +The first terminal must be valid; a later terminal cannot replace an invalid first one. Output indices +must be contiguous and covered by a completed item or the terminal output, so a text/tool delta left +open by a sparse terminal fails closed rather than becoming partial JSON. The path caps each frame at +4 MiB, each transcript and reconstruction source at 32 MiB, the stream at 100,000 frames, and +reconstructed output at 10,000 items. The configured `stallTimeoutSec` governs both the first upstream +body byte and later silent gaps. When that stall clock is disabled (`0`, including the default for a +local upstream), it does not expire immediately; only the independent 15-minute buffered-turn ceiling +remains. An EOF, malformed or oversized frame, read error, stall, cancellation, or missing terminal +returns an error instead of partial JSON with HTTP 200. Streaming callers are unchanged. When a provider filters or truncates a response, an unfinished tool call remains `incomplete` in both JSON and SSE. Partial output is preserved, and the bridge does not emit an argument diff --git a/docs-site/src/content/docs/ru/reference/configuration/server.md b/docs-site/src/content/docs/ru/reference/configuration/server.md index 2a4dcb20c68..73af6c0fbe8 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/server.md +++ b/docs-site/src/content/docs/ru/reference/configuration/server.md @@ -15,7 +15,7 @@ description: Listener, удалённый доступ, admission key, тайм | `proxy?` | `string` | — | URL исходящего HTTP(S) или SOCKS5-прокси (`socks5://host:port`) или `${ENV_VAR}`. HTTP URL пишутся в `HTTP_PROXY` / `HTTPS_PROXY`, если те не заданы. SOCKS5 используют встроенный SOCKS5-туннель и также пишутся в `ALL_PROXY` (`ocx start --socks5`); унаследованные `HTTP(S)_PROXY` сбрасываются в этом процессе. Loopback всегда остаётся в `NO_PROXY`. | | `emptyCompletionRetry?` | `boolean` | `false` | Явно включает один идентичный повтор Responses, если в turn нет ни текста, ни tool call, включая случай, когда stream завершается до terminal event. Повтор может тарифицироваться. `OCX_EMPTY_COMPLETION_RETRY=0` отключает его без изменения config; combo и routed-compaction turn исключены. | | `dropCodexSafetyBuffering?` | `boolean` | `false` | Удаляет подсказки Codex safety-buffering из passthrough-ответов Codex Responses: заголовки `x-codex-safety-buffering-enabled` / `x-codex-safety-buffering-faster-model`, SSE-события `response.metadata` типа `safety_buffering` и поле `safety_buffering` в других SSE-событиях. Codex TUI отображает их как предложение повторить запрос с более быстрой моделью, действие по умолчанию в котором переключает сессию на более слабую модель. Остальные заголовки `x-codex-*` и содержимое других SSE-событий передаются без изменений, кроме удаления этого поля. По умолчанию выключено. | -| `stallTimeoutSec?` | `number` | `300` (public) / выкл. (local) | Секунды без полезного прогресса upstream (Responses и нативный Chat) до обрыва потока. Без настройки **локальный** upstream (loopback, private, имя `.local`/`.lan`) по умолчанию выключен, публичный — 300 с; положительное значение действует на оба (минимум 1 с); `0` отключает watchdog везде. Ожидающие чтения тела `/v1/responses/compact` используют этот же бюджет, но по умолчанию 300 с даже для локального upstream; явное значение, включая `0`, имеет приоритет. | +| `stallTimeoutSec?` | `number` | `300` (public) / выкл. (local) | Секунды без полезного прогресса upstream (Responses и нативный Chat) до обрыва потока. Без настройки **локальный** upstream (loopback, private, имя `.local`/`.lan`) по умолчанию выключен, публичный — 300 с; положительное значение действует на оба (минимум 1 с); `0` отключает watchdog тишины везде. Для Responses, которые сворачивают canonical ChatGPT SSE в непотоковый JSON, даже при выключенном watchdog остаётся отдельный общий предел 15 минут. Ожидающие чтения тела `/v1/responses/compact` используют этот же бюджет, но по умолчанию 300 с даже для локального upstream; явное значение, включая `0`, имеет приоритет. | | `connectTimeoutMs?` | `number` | `200000` | Дедлайн одной попытки DNS/TCP/TLS/final-header; он завершается до генерации тела ответа. | | `shutdownTimeoutMs?` | `number` | `5000` | Дедлайн graceful-drain до принудительного прерывания активных turn'ов. | | `websockets?` | `boolean` | `false` | Объявляет и разрешает клиентский WebSocket-путь Responses. При false клиенты используют HTTP/SSE; это не отключает подходящую upstream WS-оптимизацию canonical ChatGPT. | diff --git a/docs-site/src/content/docs/ru/reference/proxy-formats.md b/docs-site/src/content/docs/ru/reference/proxy-formats.md index 976972081e2..510b3d6eadf 100644 --- a/docs-site/src/content/docs/ru/reference/proxy-formats.md +++ b/docs-site/src/content/docs/ru/reference/proxy-formats.md @@ -77,6 +77,15 @@ Translated-adapter'ы обрабатывают только известные При `stream: false` или при отсутствии `stream` те же события адаптера собираются в один JSON Responses. Обе формы сохраняют выбранную модель, output item'ы, terminal status и usage. +Канонический маршрут ChatGPT Codex принимает от upstream только SSE, поэтому `stream: true` +задаётся только для upstream-запроса. OpenCodex проверяет терминальный поток в заданных пределах и +сворачивает его в JSON, запрошенный клиентом, не меняя явно заданный `store`. Ошибка проверки не +возвращается как частичный JSON с HTTP 200. Ограничения: 4 MiB на frame, по 32 MiB на transcript и +источник реконструкции, 100 000 SSE-frame'ов и 10 000 восстановленных output item'ов. +`stallTimeoutSec` ограничивает ожидание первого body byte и последующие периоды тишины. Если значение +равно `0` или отключено по умолчанию для локального upstream, немедленного timeout нет и действует +только отдельный общий предел 15 минут. Поведение streaming-клиентов не меняется. + Клиентские frame'ы Responses SSE ограничены 4 MiB на frame, считая сырые байты до разделителя SSE-блока. В HTTP незавершённый upstream-frame, превысивший этот предел, завершается fail-closed синтетическим событием `response.failed`, после которого идёт `data: [DONE]`. В мосте Responses WebSocket то же условие даёт 502 `websocket_protocol_error` и отменяет upstream-reader. Если полноценный terminal-frame Responses уже получен, он остаётся авторитетным: слишком большие или некорректные байты после него отбрасываются и не заменяют завершённый ход транспортной ошибкой. :::note diff --git a/docs-site/src/content/docs/tr/reference/configuration/server.md b/docs-site/src/content/docs/tr/reference/configuration/server.md index a4d69c1f4a0..301faadfe16 100644 --- a/docs-site/src/content/docs/tr/reference/configuration/server.md +++ b/docs-site/src/content/docs/tr/reference/configuration/server.md @@ -15,7 +15,7 @@ yardımcı özellikleri nasıl çalıştıracağını kontrol eder. | `hostname?` | `string` | `"127.0.0.1"` | Bağlama adresi. Geri döngü olmayan bağlamalar `OPENCODEX_API_AUTH_TOKEN` gerektirir. | | `proxy?` | `string` | — | Giden HTTP(S) veya SOCKS5 proxy URL'si (`socks5://host:port`) ya da `${ENV_VAR}`. HTTP URL'leri değişkenler boşsa `HTTP_PROXY` / `HTTPS_PROXY`'ye yazılır. SOCKS5 URL'leri yerleşik gerçek SOCKS5 tünelini kullanır ve `ALL_PROXY`'ye de yazılır (`ocx start --socks5`); bu süreçte miras `HTTP(S)_PROXY` temizlenir. Geri döngü `NO_PROXY` içinde kalır. | | `emptyCompletionRetry?` | `boolean` | `false` | Metin veya araç çağrısı içermeyen bir Responses tamamlamasını aynı istekle bir kez yeniden denemeyi açıkça etkinleştirir. Yeniden deneme ücretlendirilebilir. `OCX_EMPTY_COMPLETION_RETRY=0`, yapılandırmayı değiştirmeden devre dışı bırakır; combo ve routed-compaction turları hariçtir. | -| `stallTimeoutSec?` | `number` | `300` (public) / kapalı (local) | Akış kesilmeden önce anlamlı üst sunucu ilerlemesi olmadan geçen saniye (Responses ve yerel Chat). Ayarlanmamışsa **yerel** üst sunucu (loopback, private, `.local`/`.lan` adı) için varsayılan kapalı, genel üst sunucu için 300 sn; pozitif değer ikisine de uygulanır (en az 1 sn); `0` watchdog'u her yerde kapatır. `/v1/responses/compact` için bekleyen gövde okumaları bu bütçeyi paylaşır ama yerel üst sunucuda bile varsayılan 300 sn'dir; açık değer (`0` dahil) önceliklidir. | +| `stallTimeoutSec?` | `number` | `300` (public) / kapalı (local) | Akış kesilmeden önce anlamlı üst sunucu ilerlemesi olmadan geçen saniye (Responses ve yerel Chat). Ayarlanmamışsa **yerel** üst sunucu (loopback, private, `.local`/`.lan` adı) için varsayılan kapalı, genel üst sunucu için 300 sn; pozitif değer ikisine de uygulanır (en az 1 sn); `0` sessizlik watchdog'unu her yerde kapatır. Canonical ChatGPT SSE'yi akışsız JSON'a katlayan Responses isteklerinde watchdog kapalı olsa bile bağımsız 15 dakikalık toplam tur sınırı kalır. `/v1/responses/compact` için bekleyen gövde okumaları bu bütçeyi paylaşır ama yerel üst sunucuda bile varsayılan 300 sn'dir; açık değer (`0` dahil) önceliklidir. | | `connectTimeoutMs?` | `number` | `200000` | Deneme başına DNS/TCP/TLS/nihai başlık son tarihi; gövde üretiminden önce biter. | | `shutdownTimeoutMs?` | `number` | `5000` | Aktif turlar iptal edilmeden önce zarif boşaltma süresi sınırı. | | `websockets?` | `boolean` | `false` | Responses WebSocket yolu için `supports_websockets` bildirin. False, HTTP/SSE'yi tutar. | diff --git a/docs-site/src/content/docs/tr/reference/proxy-formats.md b/docs-site/src/content/docs/tr/reference/proxy-formats.md index 90ed019103b..3178dfae456 100644 --- a/docs-site/src/content/docs/tr/reference/proxy-formats.md +++ b/docs-site/src/content/docs/tr/reference/proxy-formats.md @@ -81,6 +81,17 @@ olayı gibi Responses olaylarını yayar. Normal bir akış `data: [DONE]` ile b Responses JSON nesnesinde toplanır. Her iki form da seçilen modeli, çıktı öğelerini, terminal durumunu ve kullanımı korur. +Canonical ChatGPT Codex rotasının yukarı akışı yalnızca SSE kabul ettiğinden, +yalnızca yukarı akış isteği `stream: true` kullanır. OpenCodex terminal akışı +sınırlı boyutlar içinde doğrular ve istemcinin istediği JSON biçimine katlar; +açık bir `store` değeri değişmez. Doğrulama başarısız olursa HTTP 200 ile kısmi +JSON yerine hata döner. Sınırlar çerçeve başına 4 MiB, transcript ve yeniden +oluşturma kaynağı için ayrı ayrı 32 MiB, 100.000 SSE çerçevesi ve 10.000 yeniden +oluşturulmuş çıktı öğesidir. `stallTimeoutSec` hem ilk body byte'ını hem de +sonraki sessiz aralıkları sınırlar. Değer `0` olduğunda veya yerel upstream için +varsayılan olarak devre dışı bırakıldığında hemen zaman aşımına uğramaz; yalnızca +bağımsız 15 dakikalık toplam tur sınırı kalır. Streaming istemcileri değişmez. + İstemciye yönelik Responses SSE çerçeveleri, SSE blok sınırlayıcısından önceki ham bayt cinsinden ölçülen çerçeve başına 4 MiB ile sınırlandırılmıştır. HTTP üzerinde sınırı aşan sonlandırılmamış bir yukarı akış çerçevesi, ardından `data: diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/server.md b/docs-site/src/content/docs/zh-cn/reference/configuration/server.md index 1fc4ee8ccc7..dc96d8ff4b9 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/server.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/server.md @@ -15,7 +15,7 @@ description: 监听、远程访问、准入密钥、超时、存储、侧车、 | `proxy?` | `string` | — | 出站 HTTP(S) 或 SOCKS5 代理 URL（`socks5://host:port`），或 `${ENV_VAR}`。HTTP URL 仅在未设置时写入 `HTTP_PROXY` / `HTTPS_PROXY`。SOCKS5 URL 使用内置的真实 SOCKS5 隧道，也会写入 `ALL_PROXY`（`ocx start --socks5`）；并清除本进程继承的 `HTTP(S)_PROXY`。回环地址始终保留在 `NO_PROXY` 中。 | | `emptyCompletionRetry?` | `boolean` | `false` | 显式启用：当 Responses turn 既无文本也无工具调用时，使用相同请求重试一次，包括流在终止事件之前结束的情况。重试可能产生费用。`OCX_EMPTY_COMPLETION_RETRY=0` 可在不修改配置的情况下禁用；combo 与 routed-compaction turn 不参与。 | | `dropCodexSafetyBuffering?` | `boolean` | `false` | 从 Codex Responses 透传响应中移除 Codex safety-buffering 提示：`x-codex-safety-buffering-enabled` / `x-codex-safety-buffering-faster-model` 响应头、类型为 `safety_buffering` 的 `response.metadata` SSE 事件，以及其他 SSE 事件中的 `safety_buffering` 字段。Codex TUI 会将这些提示显示为“使用更快模型重试”的提示框，其默认操作会把会话切换到较弱的模型。其他 `x-codex-*` 响应头和其他所有 SSE 事件内容均保持不变，但会移除该字段。默认关闭。 | -| `stallTimeoutSec?` | `number` | `300`（public）/ 禁用（local） | 上游无有效进展（Responses 和原生 Chat）多少秒后切断流。未设置时**本地**上游（loopback、private、`.local`/`.lan` 名称）默认禁用，公网上游默认 300 秒；正值对两者生效（最小 1 秒）；`0` 全面禁用。`/v1/responses/compact` 的挂起响应体读取共享此预算，但即使本地上游也默认 300 秒；显式值（含 `0`）优先。 | +| `stallTimeoutSec?` | `number` | `300`（public）/ 禁用（local） | 上游无有效进展（Responses 和原生 Chat）多少秒后切断流。未设置时**本地**上游（loopback、private、`.local`/`.lan` 名称）默认禁用，公网上游默认 300 秒；正值对两者生效（最小 1 秒）；`0` 全面禁用静默 watchdog。对于把 canonical ChatGPT SSE 折叠为非流式 JSON 的 Responses 请求，即使 watchdog 已禁用，仍保留独立的 15 分钟整轮上限。`/v1/responses/compact` 的挂起响应体读取共享此预算，但即使本地上游也默认 300 秒；显式值（含 `0`）优先。 | | `connectTimeoutMs?` | `number` | `200000` | 每次尝试的 DNS/TCP/TLS/最终响应头截止时间；它在正文生成之前结束。 | | `shutdownTimeoutMs?` | `number` | `5000` | 优雅停机截止时间，超过后会中止仍在进行中的请求。 | | `websockets?` | `boolean` | `false` | 声明并允许面向客户端的 Responses WebSocket 路径。设为 false 时客户端使用 HTTP/SSE；它不会禁用符合条件的 canonical ChatGPT 上游 WS 优化。 | diff --git a/docs-site/src/content/docs/zh-cn/reference/proxy-formats.md b/docs-site/src/content/docs/zh-cn/reference/proxy-formats.md index c2d1871cfae..91ea424435d 100644 --- a/docs-site/src/content/docs/zh-cn/reference/proxy-formats.md +++ b/docs-site/src/content/docs/zh-cn/reference/proxy-formats.md @@ -72,6 +72,13 @@ Responses 表示是这座桥的中心。原生兼容的路由可以跳过部分 当 `stream: false` 或未提供 `stream` 时，同样的适配器事件会被收集为一个 Responses JSON 对象。两种形式都会保留所选模型、输出项、终止状态和 usage。 +canonical ChatGPT Codex 路由的上游只接受 SSE，因此仅对上游请求使用 `stream: true`。 +OpenCodex 会在有界限制内验证终止流，再将其折叠为客户端请求的 JSON 形式；显式 `store` 值不会 +改变。验证失败时会返回错误，而不会以 HTTP 200 返回部分 JSON。限制为：每帧 4 MiB、transcript +和重建源各 32 MiB、100,000 个 SSE 帧，以及 10,000 个重建输出项。`stallTimeoutSec` 同时控制 +首个 body byte 和后续静默间隔；当其为 `0`，或因本地上游默认禁用时，不会立即超时，只保留独立的 +15 分钟整轮上限。流式客户端的行为不变。 + 面向客户端的 Responses SSE 帧按 SSE 块分隔符之前的原始字节计算，每帧限制为 4 MiB。对于 HTTP，未终止的上游帧一旦超过该限制，会以合成的 `response.failed` 事件并随后发送 `data: [DONE]` 的方式 fail closed。对于 Responses WebSocket 桥，相同情况会发送 502 `websocket_protocol_error` 并取消上游 reader。已经完整到达的 Responses 终止帧具有优先权；其后的超大或格式错误字节会被丢弃，而不会把已经完成的轮次替换为传输失败。 :::note diff --git a/docs-site/src/content/docs/zh-tw/reference/configuration/server.md b/docs-site/src/content/docs/zh-tw/reference/configuration/server.md index ee1b7e72c77..dd8a0e472dc 100644 --- a/docs-site/src/content/docs/zh-tw/reference/configuration/server.md +++ b/docs-site/src/content/docs/zh-tw/reference/configuration/server.md @@ -13,7 +13,7 @@ description: 監聽器、遠端存取、許可金鑰、逾時、儲存、sidecar | `hostname?` | `string` | `"127.0.0.1"` | 綁定位址。非回送綁定需要 `OPENCODEX_API_AUTH_TOKEN`。 | | `proxy?` | `string` | — | 對外 HTTP(S) 或 SOCKS5 代理 URL（`socks5://host:port`）或 `${ENV_VAR}`。HTTP URL 僅在那些變數未設定時套用至 `HTTP_PROXY` / `HTTPS_PROXY`。SOCKS5 URL 使用內建的真實 SOCKS5 通道，也會套用至 `ALL_PROXY`（`ocx start --socks5`），並清除此行程繼承的 `HTTP(S)_PROXY`。回送保留在 `NO_PROXY` 中。 | | `emptyCompletionRetry?` | `boolean` | `false` | 明確啟用：當 Responses 完成時沒有文字或工具呼叫，以相同請求重試一次。重試可能產生費用。`OCX_EMPTY_COMPLETION_RETRY=0` 可在不變更設定的情況下停用；combo 與 routed-compaction turn 不適用。 | -| `stallTimeoutSec?` | `number` | `300`（public）/ 停用（local） | 上游無有效進展（Responses 與原生 Chat）多少秒後切斷串流。未設定時**本地**上游（loopback、private、`.local`/`.lan` 名稱）預設停用，公網上游預設 300 秒；正值對兩者生效（最小 1 秒）；`0` 全面停用。`/v1/responses/compact` 的擱置回應本文讀取共用此預算，但即使本地上游也預設 300 秒；明確值（含 `0`）優先。 | +| `stallTimeoutSec?` | `number` | `300`（public）/ 停用（local） | 上游無有效進展（Responses 與原生 Chat）多少秒後切斷串流。未設定時**本地**上游（loopback、private、`.local`/`.lan` 名稱）預設停用，公網上游預設 300 秒；正值對兩者生效（最小 1 秒）；`0` 全面停用靜默 watchdog。對於把 canonical ChatGPT SSE 折疊為非串流 JSON 的 Responses 請求，即使 watchdog 已停用，仍保留獨立的 15 分鐘整體上限。`/v1/responses/compact` 的擱置回應本文讀取共用此預算，但即使本地上游也預設 300 秒；明確值（含 `0`）優先。 | | `connectTimeoutMs?` | `number` | `200000` | 每次嘗試的 DNS/TCP/TLS/final-header 截止時間；它在 body 生成前結束。 | | `shutdownTimeoutMs?` | `number` | `5000` | 在中止活躍回合前的優雅排空截止時間。 | | `websockets?` | `boolean` | `false` | 廣告並允許面向 client 的 Responses WebSocket 路徑。False 時 client 使用 HTTP/SSE；不會停用符合條件的 canonical ChatGPT upstream WS 最佳化。 | diff --git a/docs-site/src/content/docs/zh-tw/reference/proxy-formats.md b/docs-site/src/content/docs/zh-tw/reference/proxy-formats.md index 8e90432b06f..5f1ca4e8a34 100644 --- a/docs-site/src/content/docs/zh-tw/reference/proxy-formats.md +++ b/docs-site/src/content/docs/zh-tw/reference/proxy-formats.md @@ -63,6 +63,13 @@ Responses 表示是橋接的中心。原生相容的路由可跳過部分轉譯 在 `stream: false` 或無 `stream` 時，相同的 adapter 事件被收集為一個 Responses JSON 物件。兩種形式都保留所選模型、輸出項目、終端狀態與 usage。 +canonical ChatGPT Codex 路由的上游只接受 SSE，因此僅對上游請求使用 `stream: true`。OpenCodex +會在有界限制內驗證終端串流，再將其折疊成客戶端要求的 JSON 形式；明確的 `store` 值不會改變。 +驗證失敗時會傳回錯誤，而不會以 HTTP 200 傳回部分 JSON。限制為每個 frame 4 MiB、transcript +與重建來源各 32 MiB、100,000 個 SSE frame，以及 10,000 個重建 output item。 +`stallTimeoutSec` 同時控制第一個 body byte 與後續靜默間隔；當它是 `0`，或因本機 upstream +預設停用時，不會立即逾時，只保留獨立的 15 分鐘整體上限。串流客戶端的行為不變。 + 每個終端 Responses usage 物件都包含兩個 detail 物件，即使供應商未回報那些細節： ```json diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index b955f7fffe4..80ddcdc8820 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -6,7 +6,7 @@ "codex-quota-query-backoff.test.ts": "codex-integration", "pnpm-command-isolation.test.ts": "update", "provider-antigravity-quota-retry.test.ts": "providers", "project-config-warning-snapshot.test.ts": "codex-integration", "codex-quota-auto-refresh-generation.test.ts": "codex-integration", "codex-account-clear-paused.test.ts": "codex-integration", "low-quota-protection.test.ts": "codex-integration", - "responses-compaction-recovery.test.ts": "responses", "compaction-recovery-settings.test.ts": "config", "responses-compaction-recovery-policy.test.ts": "responses", "plugin-loader.test.ts": "lib", "plugin-upstream-hooks.test.ts": "lib", + "responses-compaction-recovery.test.ts": "responses", "responses-canonical-nonstream.test.ts": "responses", "compaction-recovery-settings.test.ts": "config", "responses-compaction-recovery-policy.test.ts": "responses", "plugin-loader.test.ts": "lib", "plugin-upstream-hooks.test.ts": "lib", "cli-kiro-auto-selection.test.ts": "cli", "codebuddy-live-models.test.ts": "providers", "kiro-auto-selection.test.ts": "providers/kiro", "kiro-quota-metrics.test.ts": "providers/kiro", "management-provider-request-pacing.test.ts": "server", "desktop-supervised-restart.test.ts": "clients", "cli-restart-handoff.test.ts": "cli", "restart-replacement.test.ts": "server", "deepseek-artifact-tool-schema.test.ts": "providers", diff --git a/src/adapters/openai-responses/passthrough.ts b/src/adapters/openai-responses/passthrough.ts index d4533405dd0..9ef085c8cf2 100644 --- a/src/adapters/openai-responses/passthrough.ts +++ b/src/adapters/openai-responses/passthrough.ts @@ -510,6 +510,16 @@ export function createResponsesPassthroughAdapter(provider: OcxProviderConfig): const originalChoice = isPlainObject(parsed._rawBody) ? parsed._rawBody.tool_choice : undefined; finalBody = normalizeMuseToolChoice(finalBody, originalChoice); } + // The canonical ChatGPT Codex endpoint is SSE-only even though the public Responses surface + // permits an omitted/false `stream`. Keep the client's delivery preference on `parsed.stream` + // and coerce only this final upstream copy; passthrough delivery folds the terminal stream + // back into JSON for that client. `store` is intentionally untouched here because explicit + // caller storage semantics are independent of the transport required by the destination. + if (isCanonicalOpenAiForwardProvider(provider) + && isPlainObject(finalBody) + && finalBody.stream !== true) { + finalBody = { ...finalBody, stream: true }; + } if (isCanonicalOpenAiForwardProvider(provider)) { const routingHeaders = new Headers(headers); applyCodexRoutingHint(routingHeaders, finalBody); diff --git a/src/server/relay-eager.ts b/src/server/relay-eager.ts index e1fc209c563..dc787f0eb44 100644 --- a/src/server/relay-eager.ts +++ b/src/server/relay-eager.ts @@ -88,6 +88,8 @@ export type EagerRelayOptions = { upstreamError?: string; /** Optional client-facing hint policy; inspection retains original frames. */ terminalBoundary?: CodexSafetyBufferingFilterOptions; + /** Mask the selected outbound credential in synthetic failures. */ + maskCredential?: (text: string) => string; /** Injectable clock for tests. */ now?: () => number; }; @@ -123,7 +125,7 @@ export function relaySseEagerBounded( ?? (hooks.rewritePayload ? payloadRewriteAsBlockRewrite(hooks.rewritePayload) : undefined); const encodeFailedTail = (error: unknown): Uint8Array | null => { try { - return failedTailFrame(terminalEncoder, error); + return failedTailFrame(terminalEncoder, error, opts?.maskCredential); } catch { return null; } @@ -140,7 +142,7 @@ export function relaySseEagerBounded( return encodeFailedTail(error); } try { - return refusalFailedTailFrame(terminalEncoder, refusalMessage, refusalCode); + return refusalFailedTailFrame(terminalEncoder, refusalMessage, refusalCode, opts?.maskCredential); } catch { return null; } @@ -335,6 +337,7 @@ export function relaySseEagerBounded( terminalEncoder, upstreamError, terminalBoundary.upstreamRefusalCode(), + opts?.maskCredential, ); queuedBytes += upstreamErrorFrame.byteLength + terminalSentinel.byteLength; try { diff --git a/src/server/relay.ts b/src/server/relay.ts index c2cc93758f0..6d7041b402e 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -29,14 +29,18 @@ import { EMPTY_BYTES, joinSseFrameBytes, MAX_CLIENT_SSE_FRAME_BYTES, + SseFrameCountLimitError, } from "./sse-frame-buffer"; import { replaceSseDataPayload, sseDataPayload } from "./sse-payload-rewrite"; import { createBoundedResponseLogBody } from "./response-log-body"; import { clientWireLogOf } from "./inference/client-wire"; import { recordClientWireRequestLog } from "./inference/client-wire-log"; +import { nativeResponseFingerprint } from "./responses/native-response-json"; +import { nativeResponseOutput } from "./responses/native-response-output"; const nativePassthroughSseResponses = new WeakSet(); const eagerRelaySseResponses = new WeakSet(); +const preinspectedJsonResponses = new WeakSet(); export const MAX_INSPECTION_SSE_FRAME_BYTES = MAX_CLIENT_SSE_FRAME_BYTES; export const MAX_COMPLETED_OUTPUT_ITEMS = 256; @@ -124,12 +128,12 @@ export function relayWithAbort( }); } -export function buildFailedTailPayload(err: unknown): string { +export function buildFailedTailPayload(err: unknown, maskCredential?: (text: string) => string): string { const translatorOverflow = isTranslatorBudgetExceededError(err); - const message = (translatorOverflow + const diagnostic = redactSecretString((translatorOverflow ? "upstream translation buffer exceeded the safe limit" - : `Upstream stream terminated unexpectedly: ${err instanceof Error ? err.message : String(err)}`) - .slice(0, MAX_TAIL_ERROR_MESSAGE_CHARS); + : `Upstream stream terminated unexpectedly: ${err instanceof Error ? err.message : String(err)}`)); + const message = (maskCredential?.(diagnostic) ?? diagnostic).slice(0, MAX_TAIL_ERROR_MESSAGE_CHARS); const failure = { type: "upstream_error", code: translatorOverflow ? "translation_buffer_limit" : "upstream_reset", @@ -141,9 +145,9 @@ export function buildFailedTailPayload(err: unknown): string { }); } -function buildFailedTailPayloadOrFallback(err: unknown): string { +function buildFailedTailPayloadOrFallback(err: unknown, maskCredential?: (text: string) => string): string { try { - return buildFailedTailPayload(err); + return buildFailedTailPayload(err, maskCredential); } catch { // Error.message and String(error) may execute hostile accessors. Preserve a // bounded protocol terminal even when diagnostic serialization is unsafe. @@ -151,8 +155,8 @@ function buildFailedTailPayloadOrFallback(err: unknown): string { } } -export function failedTailFrame(encoder: TextEncoder, err: unknown): Uint8Array { - const payload = buildFailedTailPayloadOrFallback(err); +export function failedTailFrame(encoder: TextEncoder, err: unknown, maskCredential?: (text: string) => string): Uint8Array { + const payload = buildFailedTailPayloadOrFallback(err, maskCredential); return encoder.encode(`\n\nevent: response.failed\ndata: ${payload}\n\n${DONE_SSE_FRAME_TEXT}`); } @@ -170,17 +174,19 @@ export function upstreamErrorTailFrame( encoder: TextEncoder, message: string, refusalCode?: string, + maskCredential?: (text: string) => string, ): Uint8Array { return encoder.encode( - `event: response.failed\ndata: ${upstreamErrorFailedPayload(message, refusalCode)}\n\n`, + `event: response.failed\ndata: ${upstreamErrorFailedPayload(message, refusalCode, maskCredential)}\n\n`, ); } -function upstreamErrorFailedPayload(message: string, refusalCode?: string): string { +function upstreamErrorFailedPayload(message: string, refusalCode?: string, maskCredential?: (text: string) => string): string { + const diagnostic = redactSecretString(message); const error = { type: refusalCode === undefined ? "upstream_error" : "invalid_request_error", - code: refusalCode ?? "upstream_server_error", - message: redactSecretString(message).slice(0, MAX_TAIL_ERROR_MESSAGE_CHARS), + code: refusalCode === undefined ? "upstream_server_error" : (maskCredential?.(refusalCode) ?? refusalCode), + message: (maskCredential?.(diagnostic) ?? diagnostic).slice(0, MAX_TAIL_ERROR_MESSAGE_CHARS), }; return JSON.stringify({ type: "response.failed", @@ -206,8 +212,9 @@ export function refusalFailedTailFrame( encoder: TextEncoder, message: string, refusalCode: string, + maskCredential?: (text: string) => string, ): Uint8Array { - const payload = upstreamErrorFailedPayload(message, refusalCode); + const payload = upstreamErrorFailedPayload(message, refusalCode, maskCredential); return encoder.encode( `\n\nevent: response.failed\ndata: ${payload}\n\n${DONE_SSE_FRAME_TEXT}`, ); @@ -238,6 +245,8 @@ function boundedBareUpstreamErrorMessage(payload: unknown): string | undefined { function boundedBareUpstreamError(payload: unknown): { message: string; refusalCode: string | undefined; + errorType: string | undefined; + errorCode: string | undefined; } | undefined { const root = asJsonRecord(payload); if (!root || root.type !== "error") return undefined; @@ -247,21 +256,30 @@ function boundedBareUpstreamError(payload: unknown): { // that supplied the message also supplies the verdict. Taking the FIRST code // rather than searching for a refusal is what stops a refusal nested below a // transient one from overruling it. - const code = [ + const candidates = [ asJsonRecord(root.error), asJsonRecord(root.last_error), asJsonRecord(response?.error), asJsonRecord(response?.incomplete_details), root, - ] - .map(candidate => stringField(candidate, "code")) - .find(candidate => candidate !== undefined); + ]; + const code = candidates.map(candidate => stringField(candidate, "code")) + .find(candidate => candidate !== undefined)?.slice(0, 128); + // Root `type` is the SSE event discriminator (`"error"`), not an error class. Only nested + // error records can authoritatively name classes such as rate_limit_error or server_error. + const errorType = candidates.slice(0, -1).map(candidate => stringField(candidate, "type")) + .find(candidate => candidate !== undefined)?.slice(0, 128); const refusalCode = code !== undefined ? (isTerminalRefusalCode(code) ? code : undefined) : message === undefined ? undefined : safetyRefusalCodeFromMessage(message); - if (message !== undefined) return { message, refusalCode }; + if (message !== undefined) return { message, refusalCode, errorType, errorCode: code }; if (refusalCode === undefined) return undefined; - return { message: terminalRefusalFallbackMessage(refusalCode), refusalCode }; + return { + message: terminalRefusalFallbackMessage(refusalCode), + refusalCode, + errorType, + errorCode: code, + }; } export type SseTerminalOutputBoundary = { @@ -271,9 +289,30 @@ export type SseTerminalOutputBoundary = { doneSeen(): boolean; upstreamError(): string | undefined; upstreamRefusalCode(): string | undefined; + upstreamErrorType(): string | undefined; + upstreamErrorCode(): string | undefined; dispose(): void; }; +export class SseAggregateLimitError extends Error { + readonly maxBytes: number; + + constructor(maxBytes: number) { + super(`upstream SSE transcript exceeded ${maxBytes} bytes`); + this.name = "SseAggregateLimitError"; + this.maxBytes = maxBytes; + } +} + +export type SseTerminalOutputBoundaryOptions = CodexSafetyBufferingFilterOptions & { + /** Optional raw bytes admitted before the first terminal; omitted keeps streaming behavior. */ + maxInputBytes?: number; + /** Optional client-facing bytes emitted through the first terminal. */ + maxOutputBytes?: number; + /** Optional aggregate frame count through the first terminal. */ + maxFrames?: number; +}; + /** * Frame-aware client output boundary shared by both native Responses relays. * It buffers only the current incomplete SSE block under the same hard byte @@ -282,9 +321,21 @@ export type SseTerminalOutputBoundary = { * a terminal arrives so clean EOF can synthesize one terminal and one sentinel. */ export function createSseTerminalOutputBoundary( - options?: CodexSafetyBufferingFilterOptions, + options?: SseTerminalOutputBoundaryOptions, ): SseTerminalOutputBoundary { const dropSafetyBuffering = options?.dropCodexSafetyBuffering === true; + const maxInputBytes = options?.maxInputBytes; + const maxOutputBytes = options?.maxOutputBytes; + const maxFrames = options?.maxFrames; + if (maxInputBytes !== undefined && (!Number.isSafeInteger(maxInputBytes) || maxInputBytes <= 0)) { + throw new RangeError("maxInputBytes must be a positive safe integer"); + } + if (maxOutputBytes !== undefined && (!Number.isSafeInteger(maxOutputBytes) || maxOutputBytes <= 0)) { + throw new RangeError("maxOutputBytes must be a positive safe integer"); + } + if (maxFrames !== undefined && (!Number.isSafeInteger(maxFrames) || maxFrames <= 0)) { + throw new RangeError("maxFrames must be a positive safe integer"); + } const decoder = new TextDecoder(); const encoder = new TextEncoder(); const framer = new BoundedSseFrameBuffer(MAX_INSPECTION_SSE_FRAME_BYTES); @@ -294,12 +345,30 @@ export function createSseTerminalOutputBoundary( let disposed = false; let upstreamError: string | undefined; let upstreamRefusalCode: string | undefined; + let upstreamErrorType: string | undefined; + let upstreamErrorCode: string | undefined; + let inputBytes = 0; + let outputBytes = 0; + let framesSeen = 0; const processFrames = ( frames: ReturnType, ): Uint8Array => { if (disposed || terminal || frames.length === 0) return EMPTY_BYTES; + if (maxFrames !== undefined && frames.length > maxFrames - framesSeen) { + throw new SseFrameCountLimitError(maxFrames); + } + framesSeen += frames.length; const output: Uint8Array[] = []; + const appendOutput = (...parts: Uint8Array[]): void => { + for (const part of parts) { + if (maxOutputBytes !== undefined && part.byteLength > maxOutputBytes - outputBytes) { + throw new SseAggregateLimitError(maxOutputBytes); + } + outputBytes += part.byteLength; + output.push(part); + } + }; let responsesTerminal = false; for (const frame of frames) { const payload = sseDataPayload(decoder.decode(frame.block)); @@ -311,6 +380,8 @@ export function createSseTerminalOutputBoundary( if (bare !== undefined) { upstreamError = bare.message; upstreamRefusalCode = bare.refusalCode; + upstreamErrorType = bare.errorType; + upstreamErrorCode = bare.errorCode; } const safetyBuffering = dropSafetyBuffering && parsed !== undefined ? codexSafetyBufferingBlockAction(parsed) : "keep"; @@ -331,7 +402,7 @@ export function createSseTerminalOutputBoundary( if (isDone) { done = true; if (responsesTerminal) { - output.push(outboundBlock, frame.delimiter); + appendOutput(outboundBlock, frame.delimiter); } else if (!pendingDone) { // Do not expose a sentinel before a Responses terminal. If EOF // follows, the synthetic incomplete path owns the one sentinel; @@ -342,11 +413,11 @@ export function createSseTerminalOutputBoundary( } // Preserve every frame through the first Responses terminal. Every // later non-DONE frame is dropped. - if (!responsesTerminal) output.push(outboundBlock, frame.delimiter); + if (!responsesTerminal) appendOutput(outboundBlock, frame.delimiter); if (!responsesTerminal && payload && terminalStatusFromParsed(parsed)) { responsesTerminal = true; if (pendingDone) { - output.push(pendingDone.block, pendingDone.delimiter); + appendOutput(pendingDone.block, pendingDone.delimiter); pendingDone = null; } } @@ -361,7 +432,26 @@ export function createSseTerminalOutputBoundary( return { feed(chunk) { if (disposed || terminal) return EMPTY_BYTES; - return processFrames(framer.feed(chunk)); + if (maxInputBytes === undefined) return processFrames(framer.feed(chunk)); + // A fetch implementation may hand us one multi-megabyte chunk containing thousands of + // complete frames. Slice BEFORE framing so neither the frame array nor the final join can be + // allocated past the explicit transcript budget. Once a valid terminal appears, trailing + // bytes in the same network chunk retain the ordinary terminal-authoritative behavior. + const slices: Uint8Array[] = []; + for (let offset = 0; offset < chunk.byteLength && !terminal;) { + const remaining = maxInputBytes - inputBytes; + if (remaining <= 0) throw new SseAggregateLimitError(maxInputBytes); + const length = Math.min(64 * 1024, remaining, chunk.byteLength - offset); + const slice = chunk.subarray(offset, offset + length); + inputBytes += slice.byteLength; + offset += length; + const processed = processFrames(framer.feed(slice)); + if (processed.byteLength > 0) slices.push(processed); + if (!terminal && offset < chunk.byteLength && inputBytes >= maxInputBytes) { + throw new SseAggregateLimitError(maxInputBytes); + } + } + return joinSseFrameBytes(slices); }, finish() { if (disposed || terminal) return EMPTY_BYTES; @@ -379,6 +469,8 @@ export function createSseTerminalOutputBoundary( doneSeen: () => done, upstreamError: () => upstreamError, upstreamRefusalCode: () => upstreamRefusalCode, + upstreamErrorType: () => upstreamErrorType, + upstreamErrorCode: () => upstreamErrorCode, dispose() { if (disposed) return; disposed = true; @@ -400,7 +492,7 @@ export function relaySseWithFailedTail( body: ReadableStream, upstream: AbortController, onClientGone?: (reason?: unknown) => void, - opts?: { upstreamError?: string; terminalBoundary?: CodexSafetyBufferingFilterOptions }, + opts?: { upstreamError?: string; terminalBoundary?: CodexSafetyBufferingFilterOptions; maskCredential?: (text: string) => string }, ): ReadableStream { const reader = body.getReader(); const encoder = new TextEncoder(); @@ -454,6 +546,7 @@ export function relaySseWithFailedTail( encoder, upstreamError, terminalBoundary.upstreamRefusalCode(), + opts?.maskCredential, )); controller.enqueue(doneFrame(encoder)); } @@ -486,8 +579,8 @@ export function relaySseWithFailedTail( const refusalCode = terminalBoundary.upstreamRefusalCode(); const refusalMessage = terminalBoundary.upstreamError(); controller.enqueue(refusalCode !== undefined && refusalMessage !== undefined - ? refusalFailedTailFrame(encoder, refusalMessage, refusalCode) - : failedTailFrame(encoder, err)); + ? refusalFailedTailFrame(encoder, refusalMessage, refusalCode, opts?.maskCredential) + : failedTailFrame(encoder, err, opts?.maskCredential)); } controller.close(); } catch { /* client already torn down */ } @@ -828,9 +921,15 @@ export function responseWithDeferredRequestLog( } if (!response.body || !contentType.includes("text/event-stream")) { if (response.body && (contentType.includes("application/json") || response.status >= 400)) { + // Some delivery paths already parsed the bounded body while constructing the client JSON. + // Retain EOF/cancel logging there, but do not materialize and parse the same near-limit + // response a second time merely for metadata the first pass already published. + const preinspectedJson = contentType.includes("application/json") + && preinspectedJsonResponses.has(response); const body = createBoundedResponseLogBody(response.body, { json: contentType.includes("application/json"), - inspect: text => inspectResponseLogJson(logCtx, text), + ...(preinspectedJson ? { maxInspectionBytes: 0 } : {}), + inspect: preinspectedJson ? () => undefined : text => inspectResponseLogJson(logCtx, text), finalize: reason => { // Preserve wire status; request history follows the adjacent SSE // convention for a client cancellation or upstream read failure. @@ -890,6 +989,16 @@ export function markNativePassthroughSseResponse(response: Response): Response { return response; } +/** Mark bounded JSON whose request-log metadata was already derived from its source events. */ +export function markPreinspectedJsonResponse(response: Response): Response { + preinspectedJsonResponses.add(response); + return response; +} + +export function isPreinspectedJsonResponse(response: Response): boolean { + return preinspectedJsonResponses.has(response); +} + export function isNativePassthroughSseResponse(response: Response): boolean { return nativePassthroughSseResponses.has(response); } @@ -995,6 +1104,21 @@ export type SseInspectorHandlers = { onTerminal?: (status: ResponsesTerminalStatus, httpStatusOverride?: number) => void; logCtx?: RequestLogContext; onCompletedResponse?: (response: { id?: unknown; output?: unknown; status?: unknown }) => void; + /** A reconstructed snapshot for any valid terminal kind; unlike persistence, includes failures. */ + onTerminalResponse?: ( + status: ResponsesTerminalStatus, + response: { id?: unknown; output?: unknown; status?: unknown }, + ) => void; + /** + * Opt-in reconstruction policy for a caller that must materialize the whole terminal response. + * Persistence inspectors omit this and retain their smaller historical caps/merge semantics. + */ + terminalReconstruction?: { + maxItems: number; + maxSourceBytes: number; + requireCompleteOutputIndices?: boolean; + mergeSparseTerminalOutput?: boolean; + }; /** * Every parsed SSE payload, delivered BEFORE any onCompletedResponse derived from that same * payload. A caller that must decide on the whole turn -- not just its terminal snapshot -- @@ -1008,8 +1132,10 @@ export type SseInspectorHandlers = { * An inspector that only hears about parsed events cannot tell "nothing has been emitted" * from "something was emitted that I could not read", and a replay decision needs that * difference: an unreadable payload is a payload the caller may already have seen. + * The payload lets bounded collectors distinguish the valid `[DONE]` sentinel from malformed + * JSON without adding a second SSE parser; existing observers may ignore the argument. */ - onOpaquePayload?: () => void; + onOpaquePayload?: (payload: string | null) => void; onFirstOutput?: () => void; /** * Provider-scoped compatibility: persist the completed snapshot under the @@ -1021,6 +1147,13 @@ export type SseInspectorHandlers = { type CompletedOutputItem = { item: unknown; sourceBytes: number }; +function outputBearingSseEvent(event: { type?: unknown; output_index?: unknown } | null): boolean { + if (!event || typeof event.type !== "string") return false; + if (Object.hasOwn(event, "output_index")) return true; + return /^response\.(?:output_item|content_part|output_text|refusal|reasoning_[^.]+|function_call_arguments|custom_tool_call_input|image_generation_call|code_interpreter_call|file_search_call|web_search_call|computer_call|tool_search_call)\./ + .test(event.type); +} + function delimiterLengthAt( index: number, length: number, @@ -1071,12 +1204,24 @@ export function createSseInspector(handlers: SseInspectorHandlers): SseInspector let candidateBytes = 0; let discardingOversizedFrame = false; const reportFirstOutput = createFirstOutputReporter(handlers.onFirstOutput); - // Allocate reconstruction state only for persistence-capable inspectors. - const completedItemsByOutputIndex = handlers.onCompletedResponse + const reconstructionPolicy = handlers.terminalReconstruction; + if (reconstructionPolicy + && (!Number.isSafeInteger(reconstructionPolicy.maxItems) || reconstructionPolicy.maxItems <= 0 + || !Number.isSafeInteger(reconstructionPolicy.maxSourceBytes) || reconstructionPolicy.maxSourceBytes <= 0)) { + throw new RangeError("terminal reconstruction limits must be positive safe integers"); + } + const maxCompletedItems = reconstructionPolicy?.maxItems ?? MAX_COMPLETED_OUTPUT_ITEMS; + const maxCompletedItemSourceBytes = reconstructionPolicy?.maxSourceBytes + ?? MAX_COMPLETED_OUTPUT_ITEM_SOURCE_BYTES; + // Allocate reconstruction state only for snapshot-capable inspectors. + const completedItemsByOutputIndex = handlers.onCompletedResponse || handlers.onTerminalResponse ? new Map() : null; let aggregateItemBytes = 0; let reconstructionTainted = false; + const observedOutputIndices = reconstructionPolicy?.requireCompleteOutputIndices + ? new Set() + : null; let firstResponseId: string | undefined; const clearFrameState = (): void => { @@ -1088,6 +1233,7 @@ export function createSseInspector(handlers: SseInspectorHandlers): SseInspector const clearCompletedItems = (): void => { completedItemsByOutputIndex?.clear(); + observedOutputIndices?.clear(); aggregateItemBytes = 0; reconstructionTainted = false; }; @@ -1151,18 +1297,26 @@ export function createSseInspector(handlers: SseInspectorHandlers): SseInspector const retainCompletedItem = (index: number, item: unknown, sourceBytes: number): void => { const previous = completedItemsByOutputIndex!.get(index); if (previous) { + if (reconstructionPolicy?.mergeSparseTerminalOutput === true) { + // Repeated identical done frames are harmless, but a later frame must never rewrite an + // already-completed index. The strict buffered path treats that contradiction as taint. + if (nativeResponseFingerprint(previous.item) !== nativeResponseFingerprint(item)) { + reconstructionTainted = true; + } + return; + } aggregateItemBytes -= previous.sourceBytes; completedItemsByOutputIndex!.delete(index); } - if (sourceBytes > MAX_COMPLETED_OUTPUT_ITEM_SOURCE_BYTES) { + if (sourceBytes > maxCompletedItemSourceBytes) { reconstructionTainted = true; inspectionCounters.itemCapEvictions += 1; return; } completedItemsByOutputIndex!.set(index, { item, sourceBytes }); aggregateItemBytes += sourceBytes; - while (completedItemsByOutputIndex!.size > MAX_COMPLETED_OUTPUT_ITEMS - || aggregateItemBytes > MAX_COMPLETED_OUTPUT_ITEM_SOURCE_BYTES) { + while (completedItemsByOutputIndex!.size > maxCompletedItems + || aggregateItemBytes > maxCompletedItemSourceBytes) { let highestIndex = -1; for (const retainedIndex of completedItemsByOutputIndex!.keys()) { if (retainedIndex > highestIndex) highestIndex = retainedIndex; @@ -1201,38 +1355,30 @@ export function createSseInspector(handlers: SseInspectorHandlers): SseInspector // The other half of the same observation. A payload that did not parse still reached the // caller, so a consumer deciding whether anything has been emitted has to hear about it. if (handlers.onOpaquePayload && parsed === undefined) { - try { handlers.onOpaquePayload(); } catch { /* inspection must never throw into the pump */ } + try { handlers.onOpaquePayload(payload); } catch { /* inspection must never throw into the pump */ } } reportFirstOutput.parsed(parsed); const status = terminalStatusFromParsed(parsed); - const policyTerminal = status === "failed" - && isPolicyRewriteType(parsed) - && cyberPolicyTerminalError(parsed) !== undefined; - if (status) sawTerminal = true; - if (!reported && handlers.onTerminal && status) { - try { - reported = true; - if (handlers.logCtx) { - handlers.logCtx.transportPhase = "terminal_sse"; - handlers.logCtx.terminalSource = "upstream"; - } - handlers.onTerminal(status, policyTerminal ? 400 : undefined); - } finally { - if (status === "failed" || status === "incomplete") clearCompletedItems(); + type ParsedSseEvent = { type?: unknown; output_index?: unknown; item?: unknown; response?: unknown }; + const parsedEvent = parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? parsed as ParsedSseEvent + : null; + if (observedOutputIndices && outputBearingSseEvent(parsedEvent)) { + const outputIndex = parsedEvent?.output_index; + if (Number.isInteger(outputIndex) && (outputIndex as number) >= 0 + && (outputIndex as number) < maxCompletedItems) { + observedOutputIndices.add(outputIndex as number); + } else { + reconstructionTainted = true; } - } else if (status === "failed" || status === "incomplete") { - clearCompletedItems(); } - if (handlers.onCompletedResponse) { - type ParsedSseEvent = { type?: unknown; output_index?: unknown; item?: unknown; response?: unknown }; - const parsedEvent = parsed && typeof parsed === "object" && !Array.isArray(parsed) - ? parsed as ParsedSseEvent - : null; + let reconstructedTerminalResponse: { id?: unknown; output?: unknown; status?: unknown } | null = null; + if (completedItemsByOutputIndex) { const responseRecord = parsedEvent && typeof parsedEvent.response === "object" && parsedEvent.response !== null && !Array.isArray(parsedEvent.response) - ? parsedEvent.response as { id?: unknown } + ? parsedEvent.response as { id?: unknown; output?: unknown; status?: unknown } : null; if (handlers.pinCompletedResponseIdToFirstSeen && responseRecord @@ -1244,46 +1390,86 @@ export function createSseInspector(handlers: SseInspectorHandlers): SseInspector && doneItem !== undefined && Number.isInteger(parsedEvent.output_index) && (parsedEvent.output_index as number) >= 0 + && (!observedOutputIndices || (parsedEvent.output_index as number) < maxCompletedItems) && typeof doneItem === "object" && doneItem !== null && !Array.isArray(doneItem) && typeof (doneItem as { type?: unknown }).type === "string") { retainCompletedItem(parsedEvent.output_index as number, doneItem, sourceBytes); } - - let response = completedResponseFromParsedEvent(parsedEvent); - if (response) { + if (status && responseRecord) { + reconstructedTerminalResponse = responseRecord; if (handlers.pinCompletedResponseIdToFirstSeen && firstResponseId !== undefined - && response.id !== firstResponseId) { - response = { ...response, id: firstResponseId }; + && reconstructedTerminalResponse.id !== firstResponseId) { + reconstructedTerminalResponse = { ...reconstructedTerminalResponse, id: firstResponseId }; } - // Authoritative output is a NON-EMPTY ARRAY only. Anything else - // (missing, null, scalar, object) keeps the historical backfill - // behavior so a malformed terminal cannot reach rememberResponseState - // and destroy continuation state (review C1-2). - const hasAuthoritativeOutput = Array.isArray(response.output) - && response.output.length > 0; - if (!hasAuthoritativeOutput && reconstructionTainted) { - clearCompletedItems(); - return; + const explicitTerminalOutput = reconstructedTerminalResponse.output; + const terminalOutput = Array.isArray(explicitTerminalOutput) + ? explicitTerminalOutput + : null; + if (explicitTerminalOutput !== undefined && explicitTerminalOutput !== null && !terminalOutput) { + reconstructionTainted = true; } - if (!hasAuthoritativeOutput && completedItemsByOutputIndex!.size > 0) { - response = { - ...response, - output: [...completedItemsByOutputIndex!.entries()] + if (terminalOutput && terminalOutput.length > maxCompletedItems) reconstructionTainted = true; + const hasAuthoritativeOutput = terminalOutput !== null && terminalOutput.length > 0; + if (reconstructionPolicy?.mergeSparseTerminalOutput === true) { + if (terminalOutput?.some(item => !item || typeof item !== "object" || Array.isArray(item) + || typeof (item as { type?: unknown }).type !== "string")) reconstructionTainted = true; + let output: Record[] = []; + if (!reconstructionTainted) { + try { + output = nativeResponseOutput( + new Map([...completedItemsByOutputIndex.entries()].map(([index, retained]) => [ + index, + retained.item as Record, + ])), + terminalOutput, + ); + } catch { + reconstructionTainted = true; + } + } + if (output.length > maxCompletedItems) reconstructionTainted = true; + for (const index of observedOutputIndices ?? []) { + if (index >= output.length) reconstructionTainted = true; + } + reconstructedTerminalResponse = reconstructionTainted + ? null + : { ...reconstructedTerminalResponse, output }; + } else if (!hasAuthoritativeOutput && reconstructionTainted) { + reconstructedTerminalResponse = null; + } else if (!hasAuthoritativeOutput && completedItemsByOutputIndex.size > 0) { + reconstructedTerminalResponse = { + ...reconstructedTerminalResponse, + output: [...completedItemsByOutputIndex.entries()] .sort(([left], [right]) => left - right) .map(([, retained]) => retained.item), }; } - try { - handlers.onCompletedResponse(response); - } finally { - clearCompletedItems(); + } + } + const policyTerminal = status === "failed" + && isPolicyRewriteType(parsed) + && cyberPolicyTerminalError(parsed) !== undefined; + if (status) sawTerminal = true; + try { + if (!reported && handlers.onTerminal && status) { + reported = true; + if (handlers.logCtx) { + handlers.logCtx.transportPhase = "terminal_sse"; + handlers.logCtx.terminalSource = "upstream"; } - } else if (parsedEvent?.type === "response.completed") { - clearCompletedItems(); + handlers.onTerminal(status, policyTerminal ? 400 : undefined); + } + if (status === "completed" && reconstructedTerminalResponse && handlers.onCompletedResponse) { + handlers.onCompletedResponse(reconstructedTerminalResponse); } + if (status && reconstructedTerminalResponse && handlers.onTerminalResponse) { + handlers.onTerminalResponse(status, reconstructedTerminalResponse); + } + } finally { + if (status) clearCompletedItems(); } }; @@ -1294,7 +1480,7 @@ export function createSseInspector(handlers: SseInspectorHandlers): SseInspector } const sourceBytes = candidateBytes; const frame = takeCandidate(); - if (reported && !handlers.onCompletedResponse) return; + if (reported && !handlers.onCompletedResponse && !handlers.onTerminalResponse) return; const decoded = decoder!.decode(frame); scanPayload(sseDataPayload(decoded), sourceBytes); }; diff --git a/src/server/responses/adapter-delivery.ts b/src/server/responses/adapter-delivery.ts index fc24e39fc3e..48c30dd87ed 100644 --- a/src/server/responses/adapter-delivery.ts +++ b/src/server/responses/adapter-delivery.ts @@ -23,6 +23,7 @@ import { resolveStallTimeoutMs } from "../../stall-timeout"; import { clientEncoderForDelivery, deliverClientEncodedResponse } from "../inference/client-encoder-delivery"; import { noteKiroServedSuccess } from "../../providers/kiro-usage"; import { persistKiroAccountState } from "../../providers/kiro-account-state-disk"; +import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers"; /** One responsibility of the Responses request pipeline; state owners are explicit. */ export async function deliverAdapterResponse( @@ -30,6 +31,7 @@ export async function deliverAdapterResponse( requestState: Pick< PreparedResponsesRequest, | "parsed" + | "route" | "translatorBudget" | "toolBridgeMaps" | "rememberKiroDeliveredFinalAnswer" @@ -51,6 +53,7 @@ export async function deliverAdapterResponse( const { logCtx, options, config } = requestContext; const { parsed, + route, translatorBudget, toolBridgeMaps, rememberKiroDeliveredFinalAnswer, @@ -71,6 +74,7 @@ export async function deliverAdapterResponse( } = responseEffects; const { routedCompaction } = sidecarState; const bodyInactivityMs = resolveStallTimeoutMs(config.stallTimeoutSec, { localUpstream }); + const upstreamRequestsStream = parsed.stream || isCanonicalOpenAiForwardProvider(route.provider); if (parsed.stream) { @@ -199,15 +203,29 @@ export async function deliverAdapterResponse( }); } - if (transportState.activeAdapter.parseResponse) { + if (transportState.activeAdapter.parseResponse + || (upstreamRequestsStream && transportState.activeAdapter.parseStream)) { let events: AdapterEvent[]; try { - const initialEvents = await readResponseBodyWithInactivity( - upstreamResponse, - upstream.signal, - bodyInactivityMs, - response => transportState.activeAdapter.parseResponse!(response, translatorBudget, logCtx.activeTierMetadata), - ); + const initialEvents: AdapterEvent[] = []; + if (upstreamRequestsStream && transportState.activeAdapter.parseStream) { + // The canonical ChatGPT adapter coerces its actual upstream request to SSE even for a + // JSON client. Routed compaction still owns the non-streaming client response, so fold + // that SSE through the adapter instead of handing it to parseResponse as JSON. + for await (const event of readResponseStreamWithInactivity( + upstreamResponse, + upstream.signal, + bodyInactivityMs, + response => transportState.activeAdapter.parseStream(response, translatorBudget, logCtx.activeTierMetadata), + )) initialEvents.push(event); + } else { + initialEvents.push(...await readResponseBodyWithInactivity( + upstreamResponse, + upstream.signal, + bodyInactivityMs, + response => transportState.activeAdapter.parseResponse!(response, translatorBudget, logCtx.activeTierMetadata), + )); + } for (const event of initialEvents) options.onCompactionRecoveryAdapterEvent?.(event); let guardedEvents: AdapterEvent[]; if (terminalGuardEnabled) { diff --git a/src/server/responses/buffered-sse-json.ts b/src/server/responses/buffered-sse-json.ts new file mode 100644 index 00000000000..a4b71128200 --- /dev/null +++ b/src/server/responses/buffered-sse-json.ts @@ -0,0 +1,345 @@ +import type { ResponsesTerminalStatus } from "../../bridge"; +import { + createSseInspector, + createSseTerminalOutputBoundary, + SseAggregateLimitError, + terminalStatusFromParsed, + type CodexSafetyBufferingFilterOptions, + type SseInspector, +} from "../relay"; +import { SseFrameCountLimitError, SseFrameTooLargeError } from "../sse-frame-buffer"; +import { + MAX_UPSTREAM_JSON_BODY_BYTES, + UPSTREAM_JSON_BODY_INACTIVITY_TIMEOUT_MS, + UPSTREAM_JSON_BODY_TOTAL_TIMEOUT_MS, +} from "./core-lifetime"; + +/** Larger than persistence inspection because this path must return the whole bounded response. */ +export const MAX_BUFFERED_RESPONSES_OUTPUT_ITEMS = 10_000; +export const MAX_BUFFERED_RESPONSES_RECONSTRUCTION_BYTES = MAX_UPSTREAM_JSON_BODY_BYTES; +export const MAX_BUFFERED_RESPONSES_SSE_FRAMES = 100_000; +/** + * A buffered caller cannot consume partial output, so keep one independent lifetime ceiling even + * when the operator disables the ordinary silence clock. Fifteen minutes is intentionally much + * larger than the public default stall budget while still releasing an abandoned body eventually. + */ +export const BUFFERED_RESPONSES_TOTAL_TIMEOUT_MS = 15 * 60_000; +const BUFFERED_SSE_PROCESSING_SLICE_BYTES = 64 * 1024; + +export type BufferedResponsesTerminal = { + status: ResponsesTerminalStatus; + response: Record; +}; + +export type BufferedResponsesSseFailure = { + ok: false; + kind: "aborted" | "malformed" | "missing_terminal" | "oversized" | "read_error" | "timeout"; + error?: unknown; + /** Bounded diagnostic captured from a bare upstream `error` event. */ + upstreamError?: string; + /** Fatal provider refusal verdict that must not be flattened into a retryable transport error. */ + upstreamRefusalCode?: string; + /** Bounded structured class from the same bare error envelope as `upstreamError`. */ + upstreamErrorType?: string; + /** Bounded structured code; unknown codes remain transport failures instead of message guesses. */ + upstreamErrorCode?: string; +}; + +export type BufferedResponsesSseResult = { + ok: true; + /** Present only when the caller needs to replay the bounded transcript through rewrites. */ + bytes?: Uint8Array; + terminal: BufferedResponsesTerminal; +} | BufferedResponsesSseFailure; + +type BufferedReadOptions = { + maxBytes?: number; + maxFrames?: number; + totalTimeoutMs?: number; + /** Absolute whole-turn deadline shared by every validation pass. */ + deadlineAt?: number; + inactivityTimeoutMs?: number; + firstByteTimeoutMs?: number; +}; + +export type BufferedResponsesReadOptions = BufferedReadOptions & { deadlineAt: number }; + +/** + * Resolve the read clocks for canonical SSE that must be folded into client JSON. + * + * A positive stall budget owns both time-to-first-byte and inter-chunk silence. A disabled budget + * must not become an immediate timeout, so both clocks fall back to the independent total ceiling. + * The ceiling remains separate because a JSON caller has no partial body to keep alive forever. + */ +export function bufferedResponsesReadOptions( + stallTimeoutMs: number, + totalTimeoutMs = BUFFERED_RESPONSES_TOTAL_TIMEOUT_MS, + startedAt = Date.now(), +): BufferedResponsesReadOptions { + if (!Number.isFinite(stallTimeoutMs)) throw new RangeError("stallTimeoutMs must be finite"); + if (!Number.isSafeInteger(totalTimeoutMs) || totalTimeoutMs <= 0) { + throw new RangeError("totalTimeoutMs must be a positive safe integer"); + } + if (!Number.isSafeInteger(startedAt) || startedAt < 0 || startedAt > Number.MAX_SAFE_INTEGER - totalTimeoutMs) { + throw new RangeError("startedAt must produce a safe deadline"); + } + const silenceTimeoutMs = stallTimeoutMs > 0 ? Math.ceil(stallTimeoutMs) : totalTimeoutMs; + return { + totalTimeoutMs, + deadlineAt: startedAt + totalTimeoutMs, + firstByteTimeoutMs: silenceTimeoutMs, + inactivityTimeoutMs: silenceTimeoutMs, + }; +} + +type Uint8ReadResult = Awaited["read"]>>; + +class BufferedSseTimeoutError extends Error { + constructor(deadline: "total" | "inactivity") { + super(`buffered Responses SSE ${deadline} timeout`); + this.name = "BufferedSseTimeoutError"; + } +} + +function createBufferedTerminalInspector(maxSourceBytes: number): { + inspector: SseInspector; + result: () => { ok: true; terminal: BufferedResponsesTerminal } | BufferedResponsesSseFailure; +} { + let malformed = false; + let terminalCount = 0; + let status: ResponsesTerminalStatus | undefined; + let terminalResponse: Record | undefined; + const inspector = createSseInspector({ + terminalReconstruction: { + maxItems: MAX_BUFFERED_RESPONSES_OUTPUT_ITEMS, + maxSourceBytes, + requireCompleteOutputIndices: true, + mergeSparseTerminalOutput: true, + }, + onParsedPayload(payload) { + const candidateStatus = terminalStatusFromParsed(payload); + if (!candidateStatus) return; + terminalCount += 1; + status ??= candidateStatus; + }, + onOpaquePayload(payload) { + if (payload !== "[DONE]") malformed = true; + }, + onTerminalResponse(candidateStatus, response) { + if (candidateStatus === status && terminalResponse === undefined) { + if (response.status !== undefined && response.status !== candidateStatus) { + malformed = true; + return; + } + if (response.output !== undefined && !Array.isArray(response.output)) { + malformed = true; + return; + } + terminalResponse = { + ...response, + status: candidateStatus, + output: response.output ?? [], + } as Record; + } + }, + }); + return { + inspector, + result: () => { + if (malformed) return { ok: false, kind: "malformed" }; + if (terminalCount !== 1 || status === undefined) return { ok: false, kind: "missing_terminal" }; + if (!terminalResponse || terminalResponse.status !== status || !Array.isArray(terminalResponse.output)) { + return { ok: false, kind: "malformed" }; + } + return { ok: true, terminal: { status, response: terminalResponse } }; + }, + }; +} + +/** Strict synchronous seam for terminal/reconstruction tests; production feeds bytes incrementally. */ +export function inspectBufferedResponsesTerminal(text: string): + | { ok: true; terminal: BufferedResponsesTerminal } + | BufferedResponsesSseFailure { + const state = createBufferedTerminalInspector(MAX_BUFFERED_RESPONSES_RECONSTRUCTION_BYTES); + try { + state.inspector.feed(new TextEncoder().encode(text)); + state.inspector.finish(); + return state.result(); + } finally { + state.inspector.dispose(); + } +} + +function cancelReader(reader: ReadableStreamDefaultReader, reason?: unknown): void { + try { void reader.cancel(reason).catch(() => undefined); } catch { /* non-conforming stream */ } +} + +async function deadlineRead( + reader: ReadableStreamDefaultReader, + signal: AbortSignal | undefined, + deadlineAt: number, + deadline: "total" | "inactivity", +): Promise { + if (signal?.aborted) throw signal.reason; + return await new Promise((resolve, reject) => { + let settled = false; + let timer: ReturnType | undefined; + const finish = (callback: () => void): void => { + if (settled) return; + settled = true; + if (timer !== undefined) clearTimeout(timer); + signal?.removeEventListener("abort", onAbort); + callback(); + }; + const onAbort = () => finish(() => reject(signal?.reason)); + timer = setTimeout( + () => finish(() => reject(new BufferedSseTimeoutError(deadline))), + Math.max(0, deadlineAt - Date.now()), + ); + signal?.addEventListener("abort", onAbort, { once: true }); + if (signal?.aborted) onAbort(); + void reader.read().then( + value => finish(() => resolve(value)), + error => finish(() => reject(error)), + ); + }); +} + +/** + * Consume through the first terminal, which must validate, with bounded framing and deadlines. + * `retainTranscript:false` feeds the strict assembler directly and never materializes the second, + * client-facing SSE transcript. Side effects remain the delivery owner's responsibility. + */ +export async function collectBufferedResponsesSse( + body: ReadableStream, + upstream: AbortController, + options: { + signal?: AbortSignal; + terminalBoundary?: CodexSafetyBufferingFilterOptions; + read?: BufferedReadOptions; + retainTranscript?: boolean; + } = {}, +): Promise { + const maxBytes = options.read?.maxBytes ?? MAX_UPSTREAM_JSON_BODY_BYTES; + if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0) { + throw new RangeError("maxBytes must be a positive safe integer"); + } + const maxFrames = options.read?.maxFrames ?? MAX_BUFFERED_RESPONSES_SSE_FRAMES; + const totalTimeoutMs = options.read?.totalTimeoutMs ?? UPSTREAM_JSON_BODY_TOTAL_TIMEOUT_MS; + const inactivityTimeoutMs = options.read?.inactivityTimeoutMs ?? UPSTREAM_JSON_BODY_INACTIVITY_TIMEOUT_MS; + const firstByteTimeoutMs = options.read?.firstByteTimeoutMs ?? totalTimeoutMs; + const absoluteDeadlineAt = options.read?.deadlineAt; + if (absoluteDeadlineAt !== undefined && (!Number.isSafeInteger(absoluteDeadlineAt) || absoluteDeadlineAt < 0)) { + throw new RangeError("deadlineAt must be a non-negative safe integer"); + } + const retainTranscript = options.retainTranscript === true; + const reader = body.getReader(); + const boundary = createSseTerminalOutputBoundary({ + ...options.terminalBoundary, + maxInputBytes: maxBytes, + maxOutputBytes: maxBytes, + maxFrames, + }); + const state = createBufferedTerminalInspector(Math.min( + maxBytes, + MAX_BUFFERED_RESPONSES_RECONSTRUCTION_BYTES, + )); + const utf8 = new TextDecoder("utf-8", { fatal: true }); + let retained = retainTranscript ? new Uint8Array(Math.min(maxBytes, 64 * 1024)) : undefined; + let retainedBytes = 0; + const startedAt = Date.now(); + const totalDeadlineAt = absoluteDeadlineAt ?? startedAt + totalTimeoutMs; + let inactivityDeadlineAt = startedAt + firstByteTimeoutMs; + + const withBoundaryError = (failure: T): T => { + const upstreamError = boundary.upstreamError(); + const upstreamRefusalCode = boundary.upstreamRefusalCode(); + const upstreamErrorType = boundary.upstreamErrorType(); + const upstreamErrorCode = boundary.upstreamErrorCode(); + return { + ...failure, + ...(upstreamError === undefined ? {} : { upstreamError }), + ...(upstreamRefusalCode === undefined ? {} : { upstreamRefusalCode }), + ...(upstreamErrorType === undefined ? {} : { upstreamErrorType }), + ...(upstreamErrorCode === undefined ? {} : { upstreamErrorCode }), + }; + }; + + const inspect = (chunk: Uint8Array): void => { + if (chunk.byteLength === 0) return; + // Validate incrementally without retaining a second decoded transcript. The shared inspector + // independently owns SSE framing and JSON reconstruction over the same bytes. + utf8.decode(chunk, { stream: true }); + state.inspector.feed(chunk); + if (!retained) return; + if (chunk.byteLength > maxBytes - retainedBytes) throw new SseAggregateLimitError(maxBytes); + const required = retainedBytes + chunk.byteLength; + if (required > retained.byteLength) { + const grown = new Uint8Array(Math.min(maxBytes, Math.max(required, retained.byteLength * 2))); + grown.set(retained.subarray(0, retainedBytes)); + retained = grown; + } + retained.set(chunk, retainedBytes); + retainedBytes = required; + }; + + try { + for (;;) { + const deadlineAt = Math.min(totalDeadlineAt, inactivityDeadlineAt); + const deadline = totalDeadlineAt <= inactivityDeadlineAt ? "total" : "inactivity"; + const { done, value } = await deadlineRead(reader, options.signal, deadlineAt, deadline); + if (options.signal?.aborted) throw options.signal.reason; + if (done) { + inspect(boundary.finish()); + break; + } + if (!value || value.byteLength === 0) continue; + inactivityDeadlineAt = Date.now() + inactivityTimeoutMs; + // Slice the reader's chunk before framing. This keeps one oversized network read from + // producing a transcript-sized array of frames/output buffers and gives the total deadline + // and aggregate frame cap a checkpoint between bounded pieces of synchronous work. + for (let offset = 0; offset < value.byteLength && !boundary.terminalSeen();) { + if (Date.now() >= totalDeadlineAt) throw new BufferedSseTimeoutError("total"); + const end = Math.min(value.byteLength, offset + BUFFERED_SSE_PROCESSING_SLICE_BYTES); + inspect(boundary.feed(value.subarray(offset, end))); + offset = end; + } + if (boundary.terminalSeen()) { + cancelReader(reader, "Responses terminal event received"); + break; + } + } + utf8.decode(); + state.inspector.finish(); + if (options.signal?.aborted) throw options.signal.reason; + const terminal = state.result(); + if (!terminal.ok) { + upstream.abort(new Error(`buffered Responses SSE ${terminal.kind}`)); + return withBoundaryError(terminal); + } + return { + ...terminal, + ...(retained ? { bytes: retained.subarray(0, retainedBytes) } : {}), + }; + } catch (error) { + upstream.abort(error); + cancelReader(reader, error); + return withBoundaryError({ + ok: false, + kind: options.signal?.aborted + ? "aborted" + : error instanceof BufferedSseTimeoutError + ? "timeout" + : error instanceof SseAggregateLimitError + || error instanceof SseFrameTooLargeError + || error instanceof SseFrameCountLimitError + ? "oversized" + : "read_error", + error, + }); + } finally { + boundary.dispose(); + state.inspector.dispose(); + try { reader.releaseLock(); } catch { /* a cancelled pending read may still own it briefly */ } + } +} diff --git a/src/server/responses/core-codex-account.ts b/src/server/responses/core-codex-account.ts index 47ee892258a..1f6f2992581 100644 --- a/src/server/responses/core-codex-account.ts +++ b/src/server/responses/core-codex-account.ts @@ -395,6 +395,8 @@ export interface CodexPoolAccountRetryArgs { connectMs: number; passthroughEstimate?: number; stream: boolean; + /** Keep a buffered canonical client on HTTP/SSE after moving to another Pool account. */ + httpOnly?: boolean; onResponse?: ( response: Response, authCtx: CodexAuthContext, @@ -535,7 +537,7 @@ export async function retryCodexPoolOnAlternateAccount( ): Promise { const { callerAuthHeaders, config, route, parsed, logCtx, options, firstAuthCtx, firstResponse, - outcomeStatus, upstream, connectMs, passthroughEstimate, stream, + outcomeStatus, upstream, connectMs, passthroughEstimate, stream, httpOnly, } = args; const inboundWire = options.inboundWire ?? "responses"; const entitlementResolver = options.resolveCodexModelEntitlements ?? resolveCodexModelEntitlements; @@ -877,6 +879,7 @@ export async function retryCodexPoolOnAlternateAccount( connectMs, stream, providerFetch(route.provider, options.codexWsRuntimeIdentity, { + httpOnly, providerName: route.providerName, modelId: route.modelId, onCodexWsQuota: codexWsQuotaObserver(retryAuthCtx, route.provider, route.modelId), diff --git a/src/server/responses/core-lifetime.ts b/src/server/responses/core-lifetime.ts index e8cfcf4224c..c6b5653ea28 100644 --- a/src/server/responses/core-lifetime.ts +++ b/src/server/responses/core-lifetime.ts @@ -4,6 +4,8 @@ import { markNativePassthroughSseResponse, isEagerRelaySseResponse, markEagerRelaySseResponse, + isPreinspectedJsonResponse, + markPreinspectedJsonResponse, } from "../relay"; // runTurn adapters own an event queue and perform their combo preflight before @@ -77,6 +79,7 @@ export function finalizeOwnedTranslatorBudget(response: Response, budget: Transl if (isEagerRelaySseResponse(response)) { markEagerRelaySseResponse(finalizedResponse); } + if (isPreinspectedJsonResponse(response)) markPreinspectedJsonResponse(finalizedResponse); return finalizedResponse; } @@ -99,6 +102,7 @@ export function finalizeAccountLease(response: Response, release: () => void): R const wrapped = new Response(body, { status: response.status, statusText: response.statusText, headers: response.headers }); if (isNativePassthroughSseResponse(response)) markNativePassthroughSseResponse(wrapped); if (isEagerRelaySseResponse(response)) markEagerRelaySseResponse(wrapped); + if (isPreinspectedJsonResponse(response)) markPreinspectedJsonResponse(wrapped); return wrapped; } diff --git a/src/server/responses/passthrough-delivery.ts b/src/server/responses/passthrough-delivery.ts index 4c039f0d55c..c3d55b1f445 100644 --- a/src/server/responses/passthrough-delivery.ts +++ b/src/server/responses/passthrough-delivery.ts @@ -1,4 +1,5 @@ import { isNativeControlResponse } from "./native-response-control"; +import { createOutboundCredentialMask, createTerminalErrorRedactionBlockRewrite } from "./terminal-error-redaction"; import type { ResponsesRequestContext, ResponsesAdmissionState } from "./core-options"; import type { PreparedResponsesRequest } from "./request-prepare"; import type { ResponsesTransport } from "./request-transport"; @@ -10,6 +11,7 @@ import { createSseInspector, markEagerRelaySseResponse, markNativePassthroughSseResponse, + markPreinspectedJsonResponse, consumeForInspection, consumeForResponseLogMetadata, relaySseWithFailedTail, @@ -32,7 +34,7 @@ import { recordSubagentQuotaFailureForThreadSpawn } from "../../codex/subagent-m import { recordCodexUpstreamOutcome } from "../../codex/routing"; import { codexProbeLeaseId, codexProbeQuotaScope, codexTransientProbeGrant, releaseCodexAuthContextProbeLease } from "../../codex/auth-context"; import { consumeComboFailure } from "./core-combo-failure"; -import { readDisplaySafeErrorText } from "./core-errors"; +import { clientCancelledResponse, readDisplaySafeErrorText } from "./core-errors"; import { streamingContextOverflowResponse, jsonContextOverflowResponse } from "./context-overflow"; import { formatPassthroughUpstreamError } from "./passthrough-error"; import { rewriteUpstreamPolicyRefusal } from "./policy-refusal"; @@ -131,13 +133,94 @@ import { readBoundedResponseBody } from "../../lib/bounded-body"; import { idleDeadline } from "../../lib/abort"; import { resolveStallTimeoutMs } from "../../stall-timeout"; import { formatErrorResponse } from "../../bridge"; -import { inspectResponseLogJson } from "../request-log"; +import { + httpStatusFromTerminalError, + inspectResponseLogJson, + inspectResponseLogSsePayloadParsed, +} from "../request-log"; import { restoreRoutedCustomCallsInJson } from "../../responses/custom-tool-compat"; import { restoreRoutedToolSearchCallsInJson } from "../../responses/tool-search-compat"; import { responsesJsonToSseStream } from "../responses-json-events"; +import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers"; +import { + bufferedResponsesReadOptions, + collectBufferedResponsesSse, + type BufferedResponsesSseFailure, + type BufferedResponsesSseResult, +} from "./buffered-sse-json"; const PLAINTEXT_V2_SSE_PREFIX_LIMIT = 4096; +/** Replay retained SSE in small views so the rewrite decoder never receives one 32 MiB chunk. */ +function replayBufferedSse(bytes: Uint8Array): ReadableStream { + let offset = 0; + return new ReadableStream({ + pull(controller) { + if (offset >= bytes.byteLength) { + controller.close(); + return; + } + const end = Math.min(bytes.byteLength, offset + 64 * 1024); + controller.enqueue(bytes.subarray(offset, end)); + offset = end; + }, + }); +} + +const BUFFERED_CONTEXT_ERROR_CODES = new Set([ + "context_length_exceeded", "context_window_exceeded", "input_too_long", +]); +const BUFFERED_QUOTA_ERROR_CODES = new Set([ + "usage_limit_exceeded", "usage_limit_reached", "rate_limit_exceeded", "insufficient_quota", +]); +const BUFFERED_OVERLOAD_ERROR_CODES = new Set(["server_is_overloaded"]); + +function bufferedErrorDefaults(status: number): { type: string; code: string } { + switch (status) { + case 400: return { type: "invalid_request_error", code: "invalid_request_error" }; + case 401: return { type: "authentication_error", code: "invalid_api_key" }; + case 403: return { type: "permission_error", code: "permission_denied" }; + case 429: return { type: "rate_limit_error", code: "rate_limit_exceeded" }; + case 499: return { type: "client_closed_request", code: "client_closed_request" }; + case 503: return { type: "server_error", code: "server_is_overloaded" }; + default: return { type: "upstream_error", code: "upstream_server_error" }; + } +} + +/** Preserve only explicit, recognized bare-error classes; diagnostic wording never sets status. */ +function bufferedBareErrorPresentation( + failure: BufferedResponsesSseFailure, + message: string, +): { status: number; error: { type: string; code: string; message: string } } { + if (failure.upstreamRefusalCode !== undefined) { + return { + status: 400, + error: { type: "invalid_request_error", code: failure.upstreamRefusalCode, message }, + }; + } + const explicitCode = failure.upstreamErrorCode; + const status = explicitCode !== undefined + ? BUFFERED_CONTEXT_ERROR_CODES.has(explicitCode) ? 400 + : BUFFERED_QUOTA_ERROR_CODES.has(explicitCode) ? 429 + : BUFFERED_OVERLOAD_ERROR_CODES.has(explicitCode) ? 503 + : httpStatusFromTerminalError({ code: explicitCode }) + : httpStatusFromTerminalError({ type: failure.upstreamErrorType }); + const defaults = bufferedErrorDefaults(status); + if (status === 502) return { status, error: { ...defaults, message } }; + const explicitType = failure.upstreamErrorType; + const typeMatchesStatus = explicitType !== undefined + && (httpStatusFromTerminalError({ type: explicitType }) === status + || (explicitType === "server_error" && status === 503)); + return { + status, + error: { + type: typeMatchesStatus ? explicitType! : defaults.type, + code: explicitCode ?? defaults.code, + message, + }, + }; +} + /** Prefix-probe budget: bounds one silent gap and the whole probe alike. */ interface PlaintextV2SseProbeOptions { timeoutMs: number; @@ -348,10 +431,12 @@ export async function deliverPassthroughResponse( const { requestBindings } = transportState; let upstreamResponse = nativeExchange.upstreamResponse; + const canonicalBufferedJson = clientRequestedStream !== true + && isCanonicalOpenAiForwardProvider(route.provider); const originalContentType = upstreamResponse.headers.get("content-type"); if (isUsageDebugEnabled() && originalContentType) logCtx.usageDebugContentType = originalContentType; if (responseEffects.plaintextV2AgentMessageToolNames.size > 0 - && upstreamResponse.ok && upstreamResponse.body && parsed.stream + && upstreamResponse.ok && upstreamResponse.body && (parsed.stream || canonicalBufferedJson) && !originalContentType?.toLowerCase().includes("text/event-stream") && !originalContentType?.toLowerCase().includes("application/json") && !isCodexWsUpstreamResponse(upstreamResponse) @@ -372,7 +457,11 @@ export async function deliverPassthroughResponse( // reach this fallback only after their first Responses event is confirmed. const passthroughCt = headers.get("content-type")?.toLowerCase(); const isEventStream = passthroughCt?.includes("text/event-stream") - || (responseEffects.plaintextV2AgentMessageToolNames.size === 0 && upstreamResponse.ok && !!upstreamResponse.body && !passthroughCt && parsed.stream); + || (responseEffects.plaintextV2AgentMessageToolNames.size === 0 + && upstreamResponse.ok + && !!upstreamResponse.body + && !passthroughCt + && (parsed.stream || canonicalBufferedJson)); const recordTerminalOutcome = codexForwardTerminalOutcomeRecorder( config, admissionState.authCtx, @@ -522,10 +611,13 @@ export async function deliverPassthroughResponse( const grokUpstreamEchoEnabled = isXaiResponsesDestination(route.provider) && responsesRequestMayReplayToolOutput(parsed._rawBody); if (isEventStream && upstreamResponse.body) { - // For streamed passthrough, a successful terminal response means non-error upstream status - // before relay starts. Waiting for SSE completion would retain request state across the whole - // stream; a later body failure does not undo that this destination accepted and served the turn. - commitReasoningReplayServingRoute(nativeExchange.request.headers); + // Streaming clients commit the serving route once the upstream accepted the turn; retaining + // request state until their body ends would change the existing relay contract. The canonical + // non-streaming fold below is different: nothing is published yet, so it defers this mutation + // until both the raw and rewritten terminal snapshots validate. + if (!canonicalBufferedJson) { + commitReasoningReplayServingRoute(nativeExchange.request.headers); + } const terminalRepairPolicy = route.staticPolicy.model.responsesTerminalRepair; // #3761: opt-in hosted-web-search bridge. Codex always declares the hosted web_search tool, // and this branch relays that declaration on the assumption the destination executes it. @@ -665,7 +757,9 @@ export async function deliverPassthroughResponse( // are not the Responses wire shapes the snapshot must mirror. // Only validated client blocks may publish plaintext continuation state. // Raw inspection precedes rewriting on eager relays, so it cannot own this write. - const plaintextInspector = !grokUpstreamEchoEnabled && responseEffects.plaintextV2AgentMessageToolNames.size > 0 + const plaintextInspector = !canonicalBufferedJson + && !grokUpstreamEchoEnabled + && responseEffects.plaintextV2AgentMessageToolNames.size > 0 ? createSseInspector({ onCompletedResponse: rememberPassthroughResponseChecked }) : undefined; const plaintextEncoder = plaintextInspector ? new TextEncoder() : undefined; @@ -675,6 +769,7 @@ export async function deliverPassthroughResponse( return [block]; }, { dispose: () => plaintextInspector.dispose() }) : undefined; + const maskCredential = createOutboundCredentialMask(nativeExchange.request.headers); const blockRewrites = [ payloadRewrites.length > 0 ? payloadRewriteAsBlockRewrite(composeSsePayloadRewrites(...payloadRewrites)) @@ -742,12 +837,211 @@ export async function deliverPassthroughResponse( rememberPassthroughResponse ? rememberPassthroughResponseChecked : undefined, ) : undefined, + createTerminalErrorRedactionBlockRewrite(nativeExchange.request.headers, maskCredential), rememberPlaintextBlock, ].filter((rewrite): rewrite is NonNullable => rewrite !== undefined); const clientBlockRewrite = blockRewrites.length > 0 ? composeSseBlockRewrites(...blockRewrites) : undefined; const needsClientRewrite = clientBlockRewrite !== undefined; + + if (canonicalBufferedJson) { + const signal = options.abortSignal ?? req.signal; + const bufferedRead = bufferedResponsesReadOptions(resolveStallTimeoutMs( + config.stallTimeoutSec, + { localUpstream: nativeExchange.localUpstream }, + )); + const failBufferedTurn = ( + message: string, + failure?: BufferedResponsesSseFailure, + ): Response => { + upstream.abort(new Error(message)); + const upstreamError = failure?.upstreamError; + const upstreamRefusalCode = failure?.upstreamRefusalCode; + const rawPresentation = upstreamError === undefined || failure === undefined + ? undefined + : bufferedBareErrorPresentation(failure, upstreamError); + const presentation = rawPresentation === undefined ? undefined : { + status: rawPresentation.status, + error: { + type: maskCredential(rawPresentation.error.type), + code: maskCredential(rawPresentation.error.code), + message: maskCredential(rawPresentation.error.message), + }, + }; + if (upstreamError !== undefined) { + // The streaming relay turns a bare `error` event into a terminal failure. Mirror that + // verdict for JSON callers before recording/formatting it, or a provider refusal is + // misreported as a retryable generic 502 and clients can replay a rejected turn. + const error = presentation!.error; + const terminalPayload = { + type: "response.failed", + response: { status: "failed", error, last_error: error }, + }; + // Classification comes only from the bounded structured type/code above. A bare + // transport reset can contain misleading copy such as "invalid api key"; letting the + // log's message fallback reinterpret it would retire a healthy credential. + logCtx.terminalHttpStatus = presentation!.status; + noteInspectedPayload(terminalPayload); + inspectResponseLogSsePayloadParsed( + logCtx, + JSON.stringify(terminalPayload), + terminalPayload, + ); + } + const failureStatus = logCtx.terminalHttpStatus ?? 502; + if (recordTerminalOutcomes) { + logCtx.transportPhase = "mid_stream"; + logCtx.terminalSource = "synthetic"; + if (logCtx.activeAttempt) logCtx.activeAttempt.streamAborted = true; + terminalRecorder?.("failed", failureStatus); + options.onNativePassthroughTerminal?.("failed"); + } + if (upstreamError !== undefined) { + const failureHeaders = new Headers(headers); + failureHeaders.set("content-type", "application/json"); + failureHeaders.delete("content-length"); + failureHeaders.delete("content-encoding"); + if (upstreamRefusalCode !== undefined) failureHeaders.delete("retry-after"); + const error = presentation!.error; + return formatPassthroughUpstreamError( + failureStatus, + JSON.stringify({ error, ...(upstreamRefusalCode === undefined ? {} : { retryable: false }) }), + { + headers: failureHeaders, + suppressRetryAfter: upstreamRefusalCode !== undefined, + }, + ); + } + return formatErrorResponse(502, "upstream_error", message); + }; + let raw: BufferedResponsesSseResult | undefined = await collectBufferedResponsesSse( + passthroughSseBody, + upstream, + { signal, terminalBoundary: codexSafetyBufferingOptions, read: bufferedRead, retainTranscript: true }, + ); + if (!raw.ok) { + if (raw.kind === "aborted" || signal.aborted) { + responseEffects.responseCompletionCancelled = true; + options.onNativePassthroughCancel?.(); + return clientCancelledResponse(); + } + const message = raw.kind === "oversized" + ? "upstream SSE response exceeded the safe body limit" + : raw.kind === "timeout" + ? "upstream SSE response stalled before completing" + : raw.kind === "malformed" || raw.kind === "missing_terminal" + ? "upstream SSE response ended without one valid terminal response" + : "upstream SSE response failed before a valid terminal response"; + return failBufferedTurn(message, raw); + } + if (!raw.bytes) return failBufferedTurn("upstream SSE transcript was not retained for rewriting"); + let rawBytes: Uint8Array | undefined = raw.bytes; + raw = undefined; // Release the raw reconstruction graph before building the client copy. + + // Reuse the ordinary client rewrite chain against the bounded transcript. The raw pass + // above validates before any continuation or outcome side effect; this second validation + // means a rewrite failure cannot publish a turn the non-streaming caller never received. + let bufferedClientBody: ReadableStream | undefined = clientBlockRewrite + ? relaySseWithBlockRewrite( + replayBufferedSse(rawBytes), + clientBlockRewrite, + translatorBudget, + ) + : replayBufferedSse(rawBytes); + const client = await collectBufferedResponsesSse(bufferedClientBody, upstream, { + signal, + read: bufferedRead, + retainTranscript: false, + }); + bufferedClientBody = undefined; + if (!client.ok) { + if (client.kind === "aborted" || signal.aborted) { + responseEffects.responseCompletionCancelled = true; + options.onNativePassthroughCancel?.(); + return clientCancelledResponse(); + } + const message = client.kind === "oversized" + ? "rewritten SSE response exceeded the safe body limit" + : client.kind === "timeout" + ? "rewritten SSE response stalled before completing" + : "rewritten SSE response did not contain one valid terminal response"; + return failBufferedTurn(message, client); + } + if (Date.now() >= bufferedRead.deadlineAt) { + return failBufferedTurn("buffered Responses turn exceeded the safe total deadline"); + } + + const cancelAfterValidation = (): Response => { + responseEffects.responseCompletionCancelled = true; + options.onNativePassthroughCancel?.(); + return clientCancelledResponse(); + }; + if (signal.aborted) return cancelAfterValidation(); + + // Effects intentionally run only after both bounded validations. They inspect the same + // upstream-facing transcript as the streaming relay and fire once; continuation storage + // receives the final client-visible response once after all restorations succeeded. + const reportNativeTerminal = recordTerminalOutcomes + ? (status: ResponsesTerminalStatus, httpStatusOverride?: number) => { + if (signal.aborted) return; + terminalRecorder?.(status, httpStatusOverride); + if (status === "failed" || status === "incomplete") { + const quotaFailureMessage = [httpStatusOverride, logCtx.terminalHttpStatus] + .find(value => value === 429 || value === 402); + if (!isFixedCodexAccount(admissionState.authCtx) && quotaFailureMessage !== undefined) { + recordSubagentQuotaFailureForThreadSpawn( + req.headers, + subagentQuotaFailureModel, + quotaFailureMessage, + config, + requestState.subagentFallbackAccountId, + ); + } + } + options.onNativePassthroughTerminal?.(status); + } + : undefined; + const effectInspector = createSseInspector({ + onTerminal: reportNativeTerminal, + logCtx, + onParsedPayload: noteInspectedPayload, + onFirstOutput: options.onFirstOutput, + pinCompletedResponseIdToFirstSeen: githubCopilotRepairEnabled, + }); + try { + // The deferred effects pass must not turn a bounded 32 MiB transcript into one long + // event-loop monopoly. Keep decoder input small and yield between MiB groups; the + // aggregate frame cap was already enforced by both validation passes. + for (let offset = 0; offset < rawBytes.byteLength; offset += 64 * 1024) { + if (signal.aborted) return cancelAfterValidation(); + effectInspector.feed(rawBytes.subarray(offset, Math.min(rawBytes.byteLength, offset + 64 * 1024))); + if (offset > 0 && offset % (1024 * 1024) === 0) { + await new Promise(resolve => setTimeout(resolve, 0)); + if (signal.aborted) return cancelAfterValidation(); + } + } + if (signal.aborted) return cancelAfterValidation(); + effectInspector.finish(); + } finally { + effectInspector.dispose(); + } + if (signal.aborted) return cancelAfterValidation(); + commitReasoningReplayServingRoute(nativeExchange.request.headers); + rawBytes = undefined; + if (client.terminal.status === "completed") { + rememberPassthroughResponseChecked(client.terminal.response); + } + + const jsonHeaders = sanitizePassthroughHeaders(headers, codexSafetyBufferingOptions); + jsonHeaders.set("content-type", "application/json"); + return markPreinspectedJsonResponse(new Response(JSON.stringify(client.terminal.response), { + status: upstreamResponse.status, + statusText: upstreamResponse.statusText, + headers: jsonHeaders, + })); + } + const relayPlatform = relayPlatformForTests ?? process.platform; // #864: win32 rewrite traffic must never enter the tee()+JS-pull chain // (Bun#32111 JS-sink segfault — text frames pass, the terminal block is @@ -829,6 +1123,7 @@ export async function deliverPassthroughResponse( }, { clientGoneSignal: options.abortSignal, terminalBoundary: codexSafetyBufferingOptions, + maskCredential, ...(inlineEagerRewrite ? { rewriteBudget: translatorBudget } : {}), ...(logCtx.upstreamError === undefined ? {} : { upstreamError: logCtx.upstreamError }), }); @@ -922,7 +1217,7 @@ export async function deliverPassthroughResponse( responseEffects.responseCompletionCancelled = true; clientGone.abort(reason); }, - { upstreamError: logCtx.upstreamError, terminalBoundary: codexSafetyBufferingOptions }, + { upstreamError: logCtx.upstreamError, terminalBoundary: codexSafetyBufferingOptions, maskCredential }, ); return markNativePassthroughSseResponse(new Response(clientBody, { status: upstreamResponse.status, diff --git a/src/server/responses/passthrough-dispatch.ts b/src/server/responses/passthrough-dispatch.ts index ca1f92e473c..ec47d4e6e77 100644 --- a/src/server/responses/passthrough-dispatch.ts +++ b/src/server/responses/passthrough-dispatch.ts @@ -261,6 +261,18 @@ export async function preparePassthroughExchange( const codexSafetyBufferingOptions = isCanonicalOpenAiForwardProvider(route.provider) ? codexSafetyBufferingFilterOptions(config) : undefined; + // The canonical ChatGPT adapter coerces only its final outbound copy to `stream:true` so the + // separately captured client preference can still select JSON delivery. Recovery runs before + // delivery, though, and must classify the bytes the upstream was actually asked to send: + // ChatGPT may omit Content-Type, so consulting `parsed.stream` here would mistake its SSE body + // for non-streaming JSON and skip both opaque-state and safe-reset recovery. + const upstreamRequestsStream = parsed.stream === true + || isCanonicalOpenAiForwardProvider(route.provider); + // A JSON client still needs the canonical destination's HTTP/SSE response so delivery can + // validate and fold it. Letting the forced `stream:true` body select the WebSocket transport + // bypasses manual HTTP redirects and can turn a real 3xx into a connect timeout. + const canonicalBufferedJson = clientRequestedStream !== true + && isCanonicalOpenAiForwardProvider(route.provider); const imageGenCallAliases = route.provider.authMode === "forward" ? new Map() : imageGenToolCallAliases(toolBridgeMaps.toolNsMap, parsed._rawBody, translatorBudget); @@ -961,8 +973,9 @@ export async function preparePassthroughExchange( method: request.method, headers: request.headers, body: request.body, - }, recovery), upstream.signal, connectMs, parsed.stream, + }, recovery), upstream.signal, connectMs, upstreamRequestsStream, providerFetch(route.provider, options.codexWsRuntimeIdentity, { + httpOnly: canonicalBufferedJson, nativeControl: nativeResponseControlEligible(route.provider, options.nativeControl) && options.inboundTransport === "websocket" && !options.comboAttempt && responseEffects.plaintextV2AgentMessageToolNames.size === 0 ? options.nativeControl : undefined, @@ -1065,12 +1078,13 @@ export async function preparePassthroughExchange( method: request.method, headers: request.headers, body: request.body, - }, innerRecovery), upstream.signal, connectMs, parsed.stream, + }, innerRecovery), upstream.signal, connectMs, upstreamRequestsStream, providerFetch(route.provider, options.codexWsRuntimeIdentity, { - nativeControl: nativeResponseControlEligible(route.provider, options.nativeControl) && options.inboundTransport === "websocket" && !options.comboAttempt - && responseEffects.plaintextV2AgentMessageToolNames.size === 0 - ? options.nativeControl : undefined, - dispatchOverride: oauthDispatch(request), + httpOnly: canonicalBufferedJson, + nativeControl: nativeResponseControlEligible(route.provider, options.nativeControl) && options.inboundTransport === "websocket" && !options.comboAttempt + && responseEffects.plaintextV2AgentMessageToolNames.size === 0 + ? options.nativeControl : undefined, + dispatchOverride: oauthDispatch(request), providerName: route.providerName, modelId: route.modelId, onCodexWsQuota: codexWsQuotaObserver(admissionState.authCtx, route.provider, route.modelId), @@ -1173,6 +1187,7 @@ export async function preparePassthroughExchange( // rather than a second one to announce. const oauthReplayExecutor = storedPoolReplayDispatchNotifier( providerFetch(route.provider, options.codexWsRuntimeIdentity, { + httpOnly: canonicalBufferedJson, nativeControl: nativeResponseControlEligible(route.provider, options.nativeControl) && options.inboundTransport === "websocket" && !options.comboAttempt && responseEffects.plaintextV2AgentMessageToolNames.size === 0 ? options.nativeControl : undefined, @@ -1202,7 +1217,7 @@ export async function preparePassthroughExchange( }, recovery), upstream.signal, connectMs, - parsed.stream, + upstreamRequestsStream, oauthReplayExecutor, route.provider.authMode === "forward", ).then(adoptObservedResponse); @@ -1320,12 +1335,13 @@ export async function preparePassthroughExchange( method: request.method, headers: request.headers, body: request.body, - }, recovery), upstream.signal, connectMs, parsed.stream, + }, recovery), upstream.signal, connectMs, upstreamRequestsStream, providerFetch(route.provider, options.codexWsRuntimeIdentity, { - nativeControl: nativeResponseControlEligible(route.provider, options.nativeControl) && options.inboundTransport === "websocket" && !options.comboAttempt - && responseEffects.plaintextV2AgentMessageToolNames.size === 0 - ? options.nativeControl : undefined, - dispatchOverride: oauthDispatch(request), + httpOnly: canonicalBufferedJson, + nativeControl: nativeResponseControlEligible(route.provider, options.nativeControl) && options.inboundTransport === "websocket" && !options.comboAttempt + && responseEffects.plaintextV2AgentMessageToolNames.size === 0 + ? options.nativeControl : undefined, + dispatchOverride: oauthDispatch(request), providerName: route.providerName, modelId: route.modelId, onCodexWsQuota: codexWsQuotaObserver(admissionState.authCtx, route.provider, route.modelId), @@ -1454,12 +1470,13 @@ export async function preparePassthroughExchange( method: request.method, headers: request.headers, body: request.body, - }, recovery), upstream.signal, connectMs, parsed.stream, + }, recovery), upstream.signal, connectMs, upstreamRequestsStream, providerFetch(route.provider, options.codexWsRuntimeIdentity, { - nativeControl: nativeResponseControlEligible(route.provider, options.nativeControl) && options.inboundTransport === "websocket" && !options.comboAttempt - && responseEffects.plaintextV2AgentMessageToolNames.size === 0 - ? options.nativeControl : undefined, - dispatchOverride: oauthDispatch(request), + httpOnly: canonicalBufferedJson, + nativeControl: nativeResponseControlEligible(route.provider, options.nativeControl) && options.inboundTransport === "websocket" && !options.comboAttempt + && responseEffects.plaintextV2AgentMessageToolNames.size === 0 + ? options.nativeControl : undefined, + dispatchOverride: oauthDispatch(request), providerName: route.providerName, modelId: route.modelId, onCodexWsQuota: codexWsQuotaObserver(admissionState.authCtx, route.provider, route.modelId), @@ -1571,7 +1588,8 @@ export async function preparePassthroughExchange( upstream, connectMs, passthroughEstimate, - stream: parsed.stream, + stream: upstreamRequestsStream, + httpOnly: canonicalBufferedJson, onResponse: (response, retryAuthCtx, retryRequest) => { adoptCodexWsStage(response); captureAffinityResponse( @@ -1621,7 +1639,7 @@ export async function preparePassthroughExchange( const recoveryContentType = upstreamResponse.headers.get("content-type")?.toLowerCase() ?? ""; const streamedFunctionOutputCandidate = upstreamResponse.ok && !!upstreamResponse.body - && (recoveryContentType.includes("text/event-stream") || (!recoveryContentType && parsed.stream)) + && (recoveryContentType.includes("text/event-stream") || (!recoveryContentType && upstreamRequestsStream)) && !opaqueBlobRecoveryGuard.attempted && !configuredTransientSendBudgetExhausted() && outboundResponsesBodyCarriesEncryptedFunctionOutput(request.body); @@ -1639,7 +1657,7 @@ export async function preparePassthroughExchange( if (decryptRejection) preflightLog.upstreamError = ENCRYPTED_FUNCTION_OUTPUT_REJECTION; return decryptRejection; }, { - allowMissingContentType: !recoveryContentType && parsed.stream, + allowMissingContentType: !recoveryContentType && upstreamRequestsStream, replayReadErrors: true, }); if (options.abortSignal?.aborted) return transportFailureResponse(options.abortSignal.reason); @@ -1789,7 +1807,7 @@ export async function preparePassthroughExchange( && !(options.nativeControl && options.inboundTransport === "websocket") && ambiguousResend() !== undefined && remainingTransientSendBudget(transientSendAttempts()) > 0 - && (streamRecoveryContentType.includes("text/event-stream") || (!streamRecoveryContentType && parsed.stream)); + && (streamRecoveryContentType.includes("text/event-stream") || (!streamRecoveryContentType && upstreamRequestsStream)); if (protocolRecoveryCandidate) { upstreamResponse = deferProtocolSafeResetRecovery( upstreamResponse, @@ -1808,11 +1826,11 @@ export async function preparePassthroughExchange( authorize: () => authorizeResendForRecovery(stage, "connection-reset", ambiguousResend()).allowed, acceptResponse: candidate => { const type = candidate.headers.get("content-type")?.toLowerCase() ?? ""; - return type.includes("text/event-stream") || (!type && parsed.stream); + return type.includes("text/event-stream") || (!type && upstreamRequestsStream); }, }, ), - { allowMissingContentType: !streamRecoveryContentType && parsed.stream }, + { allowMissingContentType: !streamRecoveryContentType && upstreamRequestsStream }, ); } break; diff --git a/src/server/responses/passthrough-error.ts b/src/server/responses/passthrough-error.ts index 7a993255ee3..15deda0b9c4 100644 --- a/src/server/responses/passthrough-error.ts +++ b/src/server/responses/passthrough-error.ts @@ -79,6 +79,8 @@ export function formatPassthroughUpstreamError( statusText?: string; headers?: Headers; now?: number; + /** Caller holds an authoritative fatal verdict; neither inherited nor inferred waits apply. */ + suppressRetryAfter?: boolean; /** * Provenance from the caller that still holds the response: this body is a refusal this * proxy synthesized. The body check below is the fallback for a re-wrapped body, and it @@ -96,7 +98,7 @@ export function formatPassthroughUpstreamError( const replayRefusal = options?.replayRefusal === true || isReplayRefusalBody(trimmed); // Two different reasons to answer with no wait at all, handled the same way: a hard policy // block will not become servable, and a refusal we made was never a rate limit. - const suppressRetryAfter = cyberPolicyFailure || replayRefusal; + const suppressRetryAfter = options?.suppressRetryAfter === true || cyberPolicyFailure || replayRefusal; const resolved = suppressRetryAfter ? undefined : resolveClientRetryAfter({ diff --git a/src/server/responses/terminal-error-redaction.ts b/src/server/responses/terminal-error-redaction.ts new file mode 100644 index 00000000000..f8b26d3aedc --- /dev/null +++ b/src/server/responses/terminal-error-redaction.ts @@ -0,0 +1,53 @@ +import { REDACTED_SECRET, SENSITIVE_KEY_PATTERN, redactSecrets } from "../../lib/redact"; +import { replaceSseDataPayload, sseDataPayload, type SseBlockRewrite } from "../sse-payload-rewrite"; + +/** Mask the selected outbound credential even when upstream echoes only its raw value. */ +export function createOutboundCredentialMask(outboundHeaders: Record): (text: string) => string { + const knownSecrets = Object.entries(outboundHeaders) + .filter(([name]) => SENSITIVE_KEY_PATTERN.test(name)) + .flatMap(([name, value]) => { + const credential = /^(?:authorization|proxy-authorization)$/i.test(name) + ? value.replace(/^\S+\s+/, "") + : value; + return credential ? [credential] : []; + }) + .sort((a, b) => b.length - a.length); + return (text) => knownSecrets.reduce((safe, secret) => safe.replaceAll(secret, REDACTED_SECRET), text); +} + +/** Mask upstream diagnostics before either SSE delivery or buffered JSON reconstruction. */ +export function createTerminalErrorRedactionBlockRewrite( + outboundHeaders: Record, + maskCredential = createOutboundCredentialMask(outboundHeaders), +): SseBlockRewrite { + const redactDiagnostic = (value: unknown): unknown => { + const safe = redactSecrets(value); + const maskKnown = (entry: unknown): unknown => { + if (typeof entry === "string") return maskCredential(entry); + if (Array.isArray(entry)) return entry.map(maskKnown); + if (entry && typeof entry === "object") { + return Object.fromEntries(Object.entries(entry).map(([key, item]) => [key, maskKnown(item)])); + } + return entry; + }; + return maskKnown(safe); + }; + return (block) => { + const terminalFrame = /^event:[ \t]*response\.(?:failed|incomplete)[ \t]*\r?$/m.test(block); + const payload = sseDataPayload(block); + if (payload === null) return [terminalFrame ? maskCredential(block) : block]; + let event: Record; + try { + event = JSON.parse(payload) as Record; + } catch { + return [terminalFrame ? maskCredential(block) : block]; + } + if (event?.type !== "response.failed" && event?.type !== "response.incomplete") { + return [terminalFrame ? maskCredential(block) : block]; + } + const safeEvent = redactDiagnostic(event); + const rewritten = JSON.stringify(safeEvent); + // SSE comments and extension fields can also carry upstream text. + return [maskCredential(rewritten === payload ? block : replaceSseDataPayload(block, rewritten))]; + }; +} diff --git a/structure/decisions/ADR-6162-responses-http-sse.md b/structure/decisions/ADR-6162-responses-http-sse.md new file mode 100644 index 00000000000..1ca6842071a --- /dev/null +++ b/structure/decisions/ADR-6162-responses-http-sse.md @@ -0,0 +1,12 @@ +# ADR-6162 — decision recorded under "Responses HTTP/SSE" + +- Contract owner: [transports/responses.md](../transports/responses.md#responses-httpsse) + +## Decision record + +- 목적과 의도: Let ordinary OpenAI-compatible clients use non-streaming Responses JSON through a canonical Codex account without weakening terminal correctness. +- 기존 구현 및 제약 조건: The public Responses surface permits omitted or false `stream`, while the canonical ChatGPT Codex endpoint accepts only SSE; passthrough rewrites, continuation publication, usage logging, and account outcomes must remain single-owner effects. +- 검토한 주요 대안: Reject non-streaming clients; teach each integration to request SSE; rebuild JSON independently from deltas; or require SSE upstream and fold the bounded, rewritten terminal snapshot. +- 선택한 방식: Preserve the client preference separately, classify the actual canonical send as streaming for retry/recovery and adapter parsing, require the first terminal and every observed output index to validate under explicit frame/transcript/item-count caps, merge sparse terminal and done items by stable identity/order, then return the complete response object as JSON. Routed compaction remains a JSON client contract but folds the coerced SSE through the adapter before synthesizing its compaction item. The configured stall budget governs first-byte and inter-chunk silence; disabling it falls back to an independent 15-minute buffered-turn ceiling rather than an immediate timeout. A bare upstream `error` retains its bounded message and fatal refusal code, and is recorded/formatted with the same terminal verdict as streaming delivery. +- 다른 대안 대신 이 방식을 선택한 이유: Reusing the existing inspector and rewrite pipeline preserves tool, reasoning, refusal, annotation, usage, model, and continuation semantics that a second delta assembler would inevitably duplicate and drift. +- 장점, 단점 및 영향: Compatible non-streaming clients work without changing `store`; malformed, truncated, stalled, oversized, contradictory, open-index, or rewrite-failed streams return an error rather than partial success. Fatal upstream refusals preserve their code instead of becoming retryable generic 502s. Serving-state/cache/outcome effects wait for raw and rewritten validation. The path retains one raw transcript up to 32 MiB, admits at most 100,000 SSE frames and 10,000 completed items, processes reader chunks in 64 KiB slices, streams the rewritten pass into its assembler, releases raw reconstruction before JSON serialization, and marks request-log metadata as already inspected so the client JSON is not retained and parsed again. The independent 15-minute whole-turn ceiling is intentionally more generous than the public five-minute stall default but means a disabled silence clock is not an unbounded buffered request. This has higher latency and bounded-but-larger memory use than SSE delivery. diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index a96f7c56e5e..92b7b28f29b 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -168,7 +168,7 @@ rewrite rules and the routed-id settlement. | `src/oauth/` | OAuth providers, token storage, refresh, and auth-token resolution. Meta Muse device authorization, polling, and key-mint JSON responses share the 64 KiB bounded-body ceiling and the request's deadline; oversized declared or streamed bodies are rejected before JSON parsing. The login callback listener binds a per-provider FIXED loopback port, so consecutive logins reuse the same number; every response it sends ends its connection (`Connection: close`, including non-callback paths such as a stray `/favicon.ico` 404). Stopping the listener does not close an established socket, so without that a pooled client would deliver the next login's callback to the retired flow, which rejects the unknown state as a CSRF mismatch while the live flow waits. Command Code manual callback JSON remains opaque to the shared `code#state` parser and is state-validated by its provider parser. A raw Command Code paste with an explicit `#state` suffix must match the flow state on the direct prompt as well. Kiro add-account identity prefers same-session `whoami` over a leftover SQLite state profile, and never persists the Builder ID service profile ARN as `accountId`. | | `src/combos/request.ts` | Clones each selected combo target request and applies the existing target capability ladder: adaptive unknown targets and explicit empty ladders receive no unsupported reasoning/thinking controls, while known ladders retain per-target resolution. | -| `src/adapters/openai-responses.ts` | Native OpenAI/ChatGPT Responses passthrough. | +| `src/adapters/openai-responses.ts`, `src/adapters/openai-responses/` | Native OpenAI/ChatGPT Responses passthrough. The canonical ChatGPT adapter forces its upstream-only `stream: true` requirement without changing caller `store`; downstream JSON negotiation remains owned by the [Responses HTTP/SSE contract](transports/responses.md#responses-httpsse). | | `src/responses/muse-tool-name-alias.ts` | Host-gated Meta Muse 64-char tool-name alias/restore used by the Responses passthrough. | | `src/adapters/openai-chat.ts`, `src/adapters/openai-chat/` | OpenAI-compatible Chat Completions bridge, split into leaves (`wire.ts`, `messages.ts`, `response-events.ts`, `passthrough.ts`, `parallel-tool-calls.ts`, `reasoning-wire.ts`, `serialized-tool-call-content.ts`, `tool-call-validation.ts`, `tool-call-id-remint.ts`, `tool-schema.ts`, `errors.ts`). `parallel-tool-calls.ts` owns the `parallel_tool_calls` wire value for both the translated and native builders, so the three provider states — configured opt-out, configured opt-in, and the unset default that forwards only a caller's explicit `false` — cannot drift between them. `reasoning-wire.ts` applies explicit gateway-object and tool-bearing effort-omission declarations to both builders; absent declarations leave native raw forwarding unchanged. Its client delivery shapes in `src/chat/outbound.ts` and `src/server/chat-native-sse.ts` relay the upstream `service_tier` echo on non-stream, folded-stream, and synthesized-SSE bodies, never inventing the key when the upstream omits it. | | `src/adapters/anthropic.ts` | Anthropic Messages bridge. A `refusal` or `content_filter` stop reason yields an explicit `incomplete` event with `retryable: false` rather than `done` with that stopReason (#4312); `max_tokens` remains `done`. It is the wire that defines `tools[*].strict` and `tools[*].allowed_callers`, so a rebuilt declaration carries both: an explicit `strict: true` and any `allowed_callers` the caller declared. An absent `strict` stays absent, because the Messages inbound records it as `false` and a `false` on the wire would read as an opt-out nobody asked for. Anthropic Fast uses the native `anthropic-speed` FastWire: a set decision sends `speed: "fast"` with `fast-mode-2026-02-01` in one case-insensitively merged, deduplicated `anthropic-beta` header that preserves OAuth betas. Stream and buffered `usage.speed` echoes confirm fast or downgrade to standard; no echo leaves the request assumed. `tests/adapters/anthropic/anthropic-fast-speed.test.ts` pins the wire and echoes. Anthropic Fast is opt-in: the registry marks both Anthropic entries `fastOptIn`, and `src/providers/fast-opt-in.ts` (`providerFastSwitchOff`) keeps Fast off until `providers..fastEnabled` is `true`. An off switch is provider capability `false`, applied in the FastPolicy authority (`service-tier.ts`), `resolveModelPolicy`, and router registry enrichment, so no model-level Fast toggle, `--fast` row, or proxy-generated `speed` field is produced. Native Claude Messages passthrough still forwards a `speed` field the caller sends itself, outside the proxy Fast policy. `tests/adapters/anthropic/anthropic-fast-opt-in.test.ts` pins the default, the switch, and the management PATCH/GET. | diff --git a/structure/transports/responses-wire-shapes.md b/structure/transports/responses-wire-shapes.md index 31f2610de7d..7f03faf0306 100644 --- a/structure/transports/responses-wire-shapes.md +++ b/structure/transports/responses-wire-shapes.md @@ -396,26 +396,15 @@ Native passthrough SSE has TWO shapes, selected per request in inspection side-effect set (shared `createSseInspector` factory in `relay.ts`) including the #44 late-terminal semantics. -Both client readers also retain a bounded, redacted message from a bare upstream -`error` event. If EOF arrives without a real Responses terminal, they synthesize -one `response.failed` with that message instead of replacing it with `adapter_eof`. -That synthesized terminal also carries the upstream's own verdict. Codex classifies -a `response.failed` by `error.code` alone and retries every code outside its fatal -set, so a refusal stamped `upstream_server_error` reached the client as a retryable -disconnect and drove a reconnect loop (#5176). The readers now read a refusal code -and the message from the same candidate precedence, taking the first code present so -a refusal nested below a transient one cannot overrule it, and fall back to -recognized refusal copy only when the event carried no code at all. A refusal code -with no message still produces a terminal, and a read that fails after a refusal was -captured reports the refusal rather than a generic reset. Request-log accounting is -unchanged: a row that ends on a refusal still records the transport-level status. -The delivering reader owns this evidence; an asynchronous tee inspection branch -cannot reliably supply it before EOF. Inspection independently applies the same -bare-error rule when EOF arrives, so account health records failure instead of -clearing avoidance as if the turn had succeeded. Existing real terminals and -caller cancellation retain precedence on both branches. Native recovery preflight -also preserves a rejected body reader and its bounded prefix for the normal -mid-stream failure path; it does not turn that rejection into a decrypt retry. +Both client readers retain a bounded, redacted message and the first structured refusal code from a bare upstream `error`. +At EOF without a real terminal they synthesize `response.failed` rather than `adapter_eof`; a code without a message still +produces a terminal. Codex retries codes outside its fatal set, so code and message follow the same candidate precedence; +recognized refusal copy is used only when the event has no code. A read failure after refusal reports that refusal (#5176). +The shared outbound rewrite masks diagnostics on real failed and incomplete terminals before SSE or buffered JSON delivery, +while preserving status and output; failed turns are not retained as continuation state. Buffered JSON masks selected credentials in synthetic bare-error fields before log inspection or client formatting; request logs keep transport status. +The delivering reader owns refusal evidence before EOF; asynchronous tee inspection cannot reliably supply it. +Inspection still applies the bare-error rule at EOF for account health. Real terminals and caller cancellation take precedence. +Native recovery preflight keeps the rejected body reader and bounded prefix for normal mid-stream failure, without decrypt retry. Native Responses may rebuild once when encrypted function/custom-tool output or agent-message content receives the exact known decrypt rejection before output @@ -440,6 +429,17 @@ The two-shape contract is mirror-commented in `src/server/index.ts`; the real and the platform matrix lives in `tests/lib/bun-stream-caps.test.ts`. Keep all three in lockstep with any passthrough-policy change. +A non-streaming canonical ChatGPT client still uses the destination's SSE-only upstream path. +The [Responses HTTP/SSE owner](responses.md#responses-httpsse) validates the first terminal +and strictly covered output indices before publishing JSON or serving state; see [ADR-6162](../decisions/ADR-6162-responses-http-sse.md). +Deferred inspection checks cancellation after each yield and commits serving-route state only +after the final abort check; a disconnect returns 499 without publishing that state or a terminal. +This buffered path makes no tee/eager choice. Failed and incomplete terminals mask selected outbound +credentials across the full event, including nested output and metadata, before JSON or SSE delivery; +synthetic stream failures use the same credential mask in both streaming relays. + +> Decision record: [ADR-6162](../decisions/ADR-6162-responses-http-sse.md) + Canonical ChatGPT forward streaming has one transport-specific exception. A stable Bun runtime at or above 1.4.0 may use Codex's upstream `responses_websockets` transport; bundled Bun 1.3.14, prereleases, and diff --git a/structure/transports/responses.md b/structure/transports/responses.md index bbd5923ce18..46ea7fa69c2 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -424,7 +424,7 @@ is composed from the following owners in `src/server/responses/`; none is a gene | `request-spend.ts` | This request's entries in the durable spend ledger: one per physical send, settled from the terminal usage. | | `passthrough-execution.ts` | Native host-lease transfer and the enclosing dispatch/delivery `finally`. | | `passthrough-dispatch.ts` | Native request preparation, upstream sends and pre-commit recovery. | -| `passthrough-delivery.ts` | Native HTTP/SSE/JSON delivery, rewrite/inspection, terminal accounting, and xAI tool-envelope filtering before continuation storage. | +| `passthrough-delivery.ts`, `terminal-error-redaction.ts` | Native HTTP/SSE/JSON delivery, rewrite/inspection, terminal accounting, terminal diagnostic redaction before client delivery, and xAI tool-envelope filtering before continuation storage. | | `policy-refusal.ts` | Rewrites an allowlisted non-combo HTTP 403 model refusal (`isUpstreamPolicyRefusal` in `src/lib/errors.ts`) from an xAI destination only (`isXaiResponsesDestination`: api.x.ai or the Grok CLI proxy, on either wire) to an HTTP 200 Responses `incomplete` / `content_filter` payload, JSON or SSE, for both `adapter-dispatch.ts` and `passthrough-delivery.ts`. A streamed rewrite takes the turn admission lease and releases it when the body finishes, so the refusal stays inside active-turn accounting. Combo attempts keep the original 403 so failover classifies it as a hop. | | `sidecar-execution.ts` | Image/video versus web-search execution and their shared rotation hook. | | `completion-policy.ts`, `run-turn-execution.ts` | Empty-completion eligibility and adapter-owned event turns. | diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 96dc540b2a4..74dab858946 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -8,6 +8,7 @@ "low-quota-protection.test.ts": "codex-integration", "provider-antigravity-quota-retry.test.ts": "providers", "responses-compaction-recovery.test.ts": "responses", + "responses-canonical-nonstream.test.ts": "responses", "compaction-recovery-settings.test.ts": "config", "responses-compaction-recovery-policy.test.ts": "responses", "plugin-loader.test.ts": "lib", diff --git a/tests/helpers/codex-pool-retry.ts b/tests/helpers/codex-pool-retry.ts index 0702ff72ce3..3b8dddc3512 100644 --- a/tests/helpers/codex-pool-retry.ts +++ b/tests/helpers/codex-pool-retry.ts @@ -40,6 +40,7 @@ export function createPoolRetryHarness(context: { callerBearer?: boolean; headers?: Record; extraBody?: Record; + redirect?: RequestRedirect; }) => Promise; restoreFetch: () => void; server: ReturnType; @@ -188,6 +189,7 @@ export function createPoolRetryHarness(context: { callerBearer = true, headers = {}, extraBody = {}, + redirect, } = {}) => originalGlobalFetch(new URL(path, server.url), { method: "POST", headers: { @@ -197,6 +199,7 @@ export function createPoolRetryHarness(context: { }, body: JSON.stringify({ model, input: path.endsWith("/compact") ? [] : "hello", stream, ...extraBody }), signal, + redirect, }), }; } diff --git a/tests/helpers/responses-core-source.ts b/tests/helpers/responses-core-source.ts index 6a9ec1d4ad7..50632431819 100644 --- a/tests/helpers/responses-core-source.ts +++ b/tests/helpers/responses-core-source.ts @@ -44,6 +44,8 @@ export const RESPONSES_CORE_MODULES = [ "passthrough-dispatch.ts", "reset-replay.ts", "passthrough-delivery.ts", + "buffered-sse-json.ts", + "terminal-error-redaction.ts", "sidecar-execution.ts", "completion-policy.ts", "run-turn-execution.ts", diff --git a/tests/responses/passthrough-abort.test.ts b/tests/responses/passthrough-abort.test.ts index 5d5818bd6c3..0e264ee6e07 100644 --- a/tests/responses/passthrough-abort.test.ts +++ b/tests/responses/passthrough-abort.test.ts @@ -98,7 +98,7 @@ describe("passthrough relayWithAbort (RC2, passthrough path)", () => { expect(sseBranch).toContain("rewriteBlocks: clientBlockRewrite"); // Elsewhere the failed-tail relay converts mid-stream resets into a clean response.failed. expect(sseBranch).toMatch( - /relaySseWithFailedTail\(\s*rewrittenBody,\s*upstream,\s*reason\s*=>\s*\{\s*responseEffects\.responseCompletionCancelled\s*=\s*true;\s*clientGone\.abort\(reason\);\s*\},\s*\{\s*upstreamError:\s*logCtx\.upstreamError,\s*terminalBoundary:\s*codexSafetyBufferingOptions\s*\},\s*\)/, + /relaySseWithFailedTail\(\s*rewrittenBody,\s*upstream,\s*reason\s*=>\s*\{\s*responseEffects\.responseCompletionCancelled\s*=\s*true;\s*clientGone\.abort\(reason\);\s*\},\s*\{\s*upstreamError:\s*logCtx\.upstreamError,\s*terminalBoundary:\s*codexSafetyBufferingOptions,\s*maskCredential\s*\},\s*\)/, ); expect(sseBranch).toContain("new Response(clientBody"); expect(sseBranch).toContain("markNativePassthroughSseResponse"); diff --git a/tests/responses/responses-canonical-nonstream.test.ts b/tests/responses/responses-canonical-nonstream.test.ts new file mode 100644 index 00000000000..913454a29e7 --- /dev/null +++ b/tests/responses/responses-canonical-nonstream.test.ts @@ -0,0 +1,830 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { createResponsesPassthroughAdapter } from "../../src/adapters/openai-responses"; +import { getDefaultConfig } from "../../src/config"; +import { CODEX_FORWARD_BASE_URL } from "../../src/providers/openai-tiers"; +import { handleResponses } from "../../src/server/responses"; +import { expandPreviousResponseInput } from "../../src/responses/state"; +import { setRelayPlatformForTests } from "../../src/server/responses/passthrough-delivery"; +import { + BUFFERED_RESPONSES_TOTAL_TIMEOUT_MS, + bufferedResponsesReadOptions, + collectBufferedResponsesSse, + inspectBufferedResponsesTerminal, +} from "../../src/server/responses/buffered-sse-json"; +import type { HandleResponsesOptions } from "../../src/server/responses/core"; +import type { OcxConfig, OcxParsedRequest, OcxProviderConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { withTestTranslatorBudget } from "../helpers/translator-budget"; +import { repoPath } from "../helpers/repo-root"; + +const provider: OcxProviderConfig = { + adapter: "openai-responses", + baseUrl: CODEX_FORWARD_BASE_URL, + authMode: "forward", + codexAccountMode: "direct", + upstreamWebsocket: false, +}; + +const config: OcxConfig = { + ...getDefaultConfig(), + port: 0, + defaultProvider: "openai", + providers: { openai: provider }, +}; + +const originalFetch = globalThis.fetch; +let releaseSpendHome: (() => void) | undefined; + +afterEach(() => { + globalThis.fetch = originalFetch; + releaseSpendHome?.(); + releaseSpendHome = undefined; + setRelayPlatformForTests(undefined); +}); + +function requestBody(stream: boolean, store: boolean): Record { + return { + model: "openai/gpt-5.6-sol", + input: [{ role: "user", content: [{ type: "input_text", text: "ping" }] }], + stream, + store, + }; +} + +const ENCRYPTED_FUNCTION_OUTPUT = `${Buffer.concat([ + Buffer.from([0x80]), + Buffer.alloc(8), + Buffer.alloc(16), + Buffer.alloc(16), + Buffer.alloc(32), +]).toString("base64url")}==`; + +function encryptedFunctionOutputBody(): Record { + return { + model: "openai/gpt-5.6-sol", + stream: false, + store: false, + input: [ + { type: "function_call", call_id: "call_6162", name: "lookup", arguments: "{}" }, + { + type: "function_call_output", + call_id: "call_6162", + output: [ + { type: "encrypted_content", encrypted_content: ENCRYPTED_FUNCTION_OUTPUT }, + { type: "input_text", text: "visible result" }, + ], + }, + { role: "user", content: [{ type: "input_text", text: "continue" }] }, + ], + }; +} + +function call( + body: Record, + options: HandleResponsesOptions = {}, + requestConfig: OcxConfig = config, +): Promise { + releaseSpendHome = acquireOwnedSpendHome(); + return handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { + "content-type": "application/json", + authorization: "Bearer fixture-forward-token", + }, + body: JSON.stringify(body), + }), requestConfig, { model: "", provider: "" }, options); +} + +function sseEvent(type: string, payload: Record): string { + return `event: ${type}\ndata: ${JSON.stringify({ type, ...payload })}\n\n`; +} + +describe("canonical ChatGPT transport for non-streaming Responses callers (#6162)", () => { + test.each([false, true])("forces upstream SSE without changing explicit store:%s", store => { + const adapter = withTestTranslatorBudget(createResponsesPassthroughAdapter(provider)); + const parsed = { + modelId: "gpt-5.6-sol", + context: { messages: [] }, + stream: false, + options: {}, + _rawBody: { model: "gpt-5.6-sol", input: "ping", stream: false, store }, + } as OcxParsedRequest; + const built = adapter.buildRequest(parsed, { headers: new Headers() }); + const outbound = JSON.parse(built.body) as Record; + expect(outbound.stream).toBe(true); + expect(outbound.store).toBe(store); + built.releaseBodyObservation?.(); + }); + + test("forces upstream SSE when the client omitted stream", () => { + const adapter = withTestTranslatorBudget(createResponsesPassthroughAdapter(provider)); + const built = adapter.buildRequest({ + modelId: "gpt-5.6-sol", + context: { messages: [] }, + stream: false, + options: {}, + _rawBody: { model: "gpt-5.6-sol", input: "ping", store: false }, + } as OcxParsedRequest, { headers: new Headers() }); + expect(JSON.parse(built.body)).toMatchObject({ stream: true, store: false }); + built.releaseBodyObservation?.(); + }); + + test("does not coerce a noncanonical forward gateway", () => { + const adapter = withTestTranslatorBudget(createResponsesPassthroughAdapter({ + ...provider, + baseUrl: "https://forward.example.test/v1", + })); + const built = adapter.buildRequest({ + modelId: "routed-model", + context: { messages: [] }, + stream: false, + options: {}, + _rawBody: { model: "routed-model", input: "ping", stream: false, store: true }, + } as OcxParsedRequest, { headers: new Headers() }); + expect(JSON.parse(built.body)).toMatchObject({ stream: false, store: true }); + built.releaseBodyObservation?.(); + }); + + test("missing Content-Type still returns complete JSON with structured output and accounting", async () => { + const terminal = { + id: "resp_nonstream", + object: "response", + created_at: 1_800_000_000, + status: "completed", + model: "gpt-5.6-sol", + store: true, + output: [ + { + type: "reasoning", id: "rs_1", status: "completed", + summary: [{ type: "summary_text", text: "brief rationale" }], + content: [{ type: "reasoning_text", text: "private rationale" }], + encrypted_content: "opaque-reasoning", + }, + { + type: "message", id: "msg_1", status: "completed", role: "assistant", + content: [ + { type: "output_text", text: "answer", annotations: [{ type: "url_citation", url: "https://example.test", title: "source", start_index: 0, end_index: 6 }] }, + { type: "refusal", refusal: "declined detail" }, + ], + }, + { type: "function_call", id: "fc_1", status: "completed", call_id: "call_1", name: "lookup", arguments: "{\"q\":\"x\"}" }, + { type: "custom_tool_call", id: "ctc_1", status: "completed", call_id: "call_2", name: "patch", input: "*** Begin Patch" }, + ], + usage: { + input_tokens: 17, + output_tokens: 9, + total_tokens: 26, + input_tokens_details: { cached_tokens: 3 }, + output_tokens_details: { reasoning_tokens: 4 }, + subscription: { window: "fixture" }, + }, + }; + let outbound: Record | undefined; + globalThis.fetch = (async (_input, init) => { + outbound = JSON.parse(String(init?.body)) as Record; + return new Response(new TextEncoder().encode([ + sseEvent("response.created", { response: { ...terminal, status: "in_progress", output: [] } }), + ...terminal.output.map((item, output_index) => sseEvent("response.output_item.done", { output_index, item })), + // Canonical streams may leave the terminal output sparse; the shared inspector must + // reconstruct every structured item from output_item.done without losing terminal fields. + sseEvent("response.completed", { response: { ...terminal, output: [] } }), + "data: [DONE]\n\n", + ].join("")), { headers: { "x-codex-turn-id": "turn-6162" } }); + }) as typeof fetch; + const terminals: string[] = []; + const completedModels: string[] = []; + + const response = await call(requestBody(false, true), { + onNativePassthroughTerminal: status => terminals.push(status), + onResponseComplete: model => completedModels.push(model), + }); + + expect(outbound?.stream).toBe(true); + expect(outbound?.store).toBe(true); + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toContain("application/json"); + expect(response.headers.get("x-codex-turn-id")).toBe("turn-6162"); + expect(await response.json()).toEqual(terminal); + expect(terminals).toEqual(["completed"]); + expect(completedModels).toEqual(["gpt-5.6-sol"]); + }); + + test("omitted stream is negotiated as JSON end-to-end while upstream receives true", async () => { + let outbound: Record | undefined; + globalThis.fetch = (async (_input, init) => { + outbound = JSON.parse(String(init?.body)) as Record; + return new Response(sseEvent("response.completed", { response: { + id: "resp_omitted", status: "completed", model: "gpt-5.6-sol", output: [], + } }), { headers: { "content-type": "text/event-stream" } }); + }) as typeof fetch; + const body = requestBody(false, false); + delete body.stream; + + const response = await call(body); + + expect(outbound?.stream).toBe(true); + expect(response.headers.get("content-type")).toContain("application/json"); + expect(await response.json()).toMatchObject({ id: "resp_omitted", status: "completed", output: [] }); + }); + + test.each([false, true])("redacts echoed pooled bearer in failed terminal for stream:%s", async stream => { + const bearer = "fixture-pooled-bearer-123456789"; + const failed = { + id: `resp_failed_secret_${stream}`, + status: "failed", + output: [{ + type: "message", id: "msg_secret_echo", status: "completed", role: "assistant", + content: [{ type: "output_text", text: "fixture-forward-token", annotations: [] }], + }], + error: { type: "server_error", code: "upstream_error", message: `upstream saw Bearer ${bearer}` }, + last_error: { + message: `Authorization: Bearer ${bearer}`, + detail: "raw echo fixture-forward-token", + }, + metadata: { diagnostic: "selected fixture-forward-token" }, + }; + globalThis.fetch = (async () => new Response(`: fixture-forward-token\n${sseEvent("response.failed", { + response: failed, detail: "outer fixture-forward-token", + })}`, { + headers: { "content-type": "text/event-stream" }, + })) as typeof fetch; + + const response = await call(requestBody(stream, true)); + const text = await response.text(); + expect(response.status).toBe(200); + expect(text).not.toContain(bearer); + expect(text).not.toContain("fixture-forward-token"); + expect(text).toContain("[REDACTED]"); + if (stream) { + expect(text).toContain("event: response.failed"); + expect(text).toContain("[REDACTED]"); + expect(text).not.toContain(": fixture-forward-token"); + } else { + const json = JSON.parse(text) as typeof failed; + expect(json).toMatchObject({ id: failed.id, status: "failed", error: { + type: "server_error", code: "upstream_error", message: "upstream saw Bearer [REDACTED]", + } }); + expect(json.last_error.message).toContain("[REDACTED]"); + } + const next = { model: "openai/gpt-5.6-sol", previous_response_id: failed.id, input: "retry" }; + expect(expandPreviousResponseInput(next)).toEqual(next); + }); + + test("missing Content-Type keeps canonical non-stream opaque-state recovery on the SSE path", async () => { + const outbound: Array> = []; + globalThis.fetch = (async (_input, init) => { + outbound.push(JSON.parse(String(init?.body)) as Record); + if (outbound.length === 1) { + return new Response(new TextEncoder().encode(sseEvent("response.failed", { response: { + id: "resp_ciphertext_rejected", + status: "failed", + output: [], + error: { + message: "Encrypted function output content could not be decrypted or decoded.", + type: "server_error", + code: null, + }, + } }))); + } + return new Response(new TextEncoder().encode(sseEvent("response.completed", { response: { + id: "resp_ciphertext_recovered", + status: "completed", + model: "gpt-5.6-sol", + output: [], + } }))); + }) as typeof fetch; + + const response = await call(encryptedFunctionOutputBody()); + const json = await response.json() as Record; + + expect(response.status).toBe(200); + expect(json).toMatchObject({ id: "resp_ciphertext_recovered", status: "completed" }); + expect(outbound).toHaveLength(2); + expect(outbound.map(body => body.stream)).toEqual([true, true]); + expect(JSON.stringify(outbound[0])).toContain("encrypted_content"); + expect(JSON.stringify(outbound[1])).not.toContain("encrypted_content"); + expect(JSON.stringify(outbound[1])).toContain("[encrypted content omitted]"); + }); + + test("missing Content-Type keeps canonical non-stream pre-output reset recovery on the SSE path", async () => { + const outbound: Array> = []; + const created = new TextEncoder().encode(sseEvent("response.created", { response: { + id: "resp_reset_first", + status: "in_progress", + output: [], + } })); + globalThis.fetch = (async (_input, init) => { + outbound.push(JSON.parse(String(init?.body)) as Record); + if (outbound.length === 1) { + let emitted = false; + return new Response(new ReadableStream({ + pull(controller) { + if (!emitted) { + emitted = true; + controller.enqueue(created); + return; + } + controller.error(Object.assign(new Error("fixture body reset before output"), { code: "ECONNRESET" })); + }, + })); + } + return new Response(new TextEncoder().encode(sseEvent("response.completed", { response: { + id: "resp_reset_recovered", + status: "completed", + model: "gpt-5.6-sol", + output: [], + } }))); + }) as typeof fetch; + + const resetConfig: OcxConfig = { + ...config, + providers: { openai: { ...provider, retryOnReset: {} } }, + }; + const response = await call(requestBody(false, false), {}, resetConfig); + const json = await response.json() as Record; + + expect(response.status, JSON.stringify(json)).toBe(200); + expect(json).toMatchObject({ id: "resp_reset_recovered", status: "completed" }); + expect(outbound).toHaveLength(2); + expect(outbound.map(body => body.stream)).toEqual([true, true]); + }); + + test.each([ + ["clean EOF", sseEvent("response.created", { response: { id: "resp_eof", status: "in_progress", output: [] } })], + ["malformed payload", [ + "data: {not-json}\n\n", + sseEvent("response.completed", { response: { id: "resp_bad", status: "completed", output: [] } }), + ].join("")], + ])("fails closed on %s instead of returning partial HTTP 200 JSON", async (_name, body) => { + let outbound: Record | undefined; + globalThis.fetch = (async (_input, init) => { + outbound = JSON.parse(String(init?.body)) as Record; + return new Response(body, { headers: { "content-type": "text/event-stream" } }); + }) as typeof fetch; + const terminals: string[] = []; + + const response = await call(requestBody(false, false), { + onNativePassthroughTerminal: status => terminals.push(status), + }); + const error = await response.json() as { error?: { type?: string; message?: string } }; + + expect(outbound?.stream).toBe(true); + expect(outbound?.store).toBe(false); + expect(response.status).toBe(502); + expect(error.error?.type).toBe("server_error"); + expect(error.error?.message).toContain("valid terminal response"); + expect(terminals).toEqual(["failed"]); + }); + + test("client abort returns 499 without terminal or completion effects", async () => { + const abort = new AbortController(); + let cancelled = 0; + globalThis.fetch = (async () => { + setTimeout(() => abort.abort(new Error("fixture client gone")), 0); + return new Response(new ReadableStream({ + pull: () => new Promise(() => {}), + cancel: () => { cancelled += 1; }, + }), { headers: { "content-type": "text/event-stream" } }); + }) as typeof fetch; + const terminals: string[] = []; + const completedModels: string[] = []; + let nativeCancels = 0; + + const response = await call(requestBody(false, false), { + abortSignal: abort.signal, + onNativePassthroughTerminal: status => terminals.push(status), + onResponseComplete: model => completedModels.push(model), + onNativePassthroughCancel: () => { nativeCancels += 1; }, + }); + + expect(response.status).toBe(499); + expect(terminals).toEqual([]); + expect(completedModels).toEqual([]); + expect(nativeCancels).toBe(1); + expect(cancelled).toBe(1); + }); + + test("client abort during deferred replay yields returns 499 without publishing completion", async () => { + const abort = new AbortController(); + const item = { + type: "message", id: "msg_disconnect", status: "completed", role: "assistant", + content: [{ type: "output_text", text: "done", annotations: [] }], + }; + const transcript = [ + ...Array.from({ length: 14_000 }, () => sseEvent("response.output_text.delta", { + output_index: 0, item_id: "msg_disconnect", delta: "x", + })), + sseEvent("response.output_item.done", { output_index: 0, item }), + sseEvent("response.completed", { response: { + id: "resp_disconnected", status: "completed", model: "gpt-5.6-sol", output: [item], + } }), + ].join(""); + expect(new TextEncoder().encode(transcript).byteLength).toBeGreaterThan(1024 * 1024); + globalThis.fetch = (async () => new Response(transcript, { + headers: { "content-type": "text/event-stream" }, + })) as typeof fetch; + const terminals: string[] = []; + const completedModels: string[] = []; + let nativeCancels = 0; + let firstOutputs = 0; + + const response = await call(requestBody(false, true), { + abortSignal: abort.signal, + onFirstOutput: () => { + firstOutputs += 1; + abort.abort(new Error("fixture client gone")); + }, + onNativePassthroughTerminal: status => terminals.push(status), + onResponseComplete: model => completedModels.push(model), + onNativePassthroughCancel: () => { nativeCancels += 1; }, + }); + + expect(firstOutputs).toBe(1); + expect(abort.signal.aborted).toBe(true); + expect(response.status).toBe(499); + expect(terminals).toEqual([]); + expect(completedModels).toEqual([]); + expect(nativeCancels).toBe(1); + }); + + test.each(["darwin", "win32"] as const)("redacts selected bearer in synthetic %s streaming failure", async platform => { + setRelayPlatformForTests(platform); + globalThis.fetch = (async () => new Response(new ReadableStream({ + pull(controller) { controller.error(new Error("reset after fixture-forward-token")); }, + }), { headers: { "content-type": "text/event-stream" } })) as typeof fetch; + const response = await call(requestBody(true, false)); + const text = await response.text(); + expect(text).toContain("response.failed"); + expect(text).toContain("[REDACTED]"); + expect(text).not.toContain("fixture-forward-token"); + }); + + test("canonical buffered serving-state commit is ordered after both validations", () => { + const source = readFileSync(repoPath("src/server/responses/passthrough-delivery.ts"), "utf8"); + const branch = source.indexOf("if (canonicalBufferedJson) {"); + const rawValidation = source.indexOf("if (!raw.ok)", branch); + const clientValidation = source.indexOf("if (!client.ok)", rawValidation); + const deferredCommit = source.indexOf("commitReasoningReplayServingRoute(nativeExchange.request.headers);", clientValidation); + const finalAbortCheck = source.lastIndexOf("if (signal.aborted) return cancelAfterValidation();", deferredCommit); + expect(branch).toBeGreaterThan(-1); + expect(rawValidation).toBeGreaterThan(branch); + expect(clientValidation).toBeGreaterThan(rawValidation); + expect(deferredCommit).toBeGreaterThan(clientValidation); + expect(finalAbortCheck).toBeGreaterThan(source.indexOf("effectInspector.finish();", clientValidation)); + expect(deferredCommit).toBeGreaterThan(finalAbortCheck); + expect(source.slice(branch, deferredCommit)).not.toContain("commitReasoningReplayServingRoute("); + }); + + test("leaves stream:true callers on the SSE relay", async () => { + let outbound: Record | undefined; + globalThis.fetch = (async (_input, init) => { + outbound = JSON.parse(String(init?.body)) as Record; + return new Response(sseEvent("response.completed", { + response: { id: "resp_stream", status: "completed", model: "gpt-5.6-sol", output: [] }, + }), { headers: { "content-type": "text/event-stream" } }); + }) as typeof fetch; + + const response = await call(requestBody(true, false)); + expect(outbound?.stream).toBe(true); + expect(response.headers.get("content-type")).toContain("text/event-stream"); + expect(await response.text()).toContain("response.completed"); + }); + + test.each([ + ["read error", () => new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(sseEvent("response.created", { + response: { id: "resp_read", status: "in_progress", output: [] }, + }))); + controller.error(new Error("fixture reset")); + }, + })], + ["oversized frame", () => new Response( + `data: ${JSON.stringify({ type: "response.output_text.delta", delta: "x".repeat(4 * 1024 * 1024) })}\n\n`, + ).body!], + ])("bounded collector rejects a %s before terminal publication", async (_name, source) => { + const upstream = new AbortController(); + const result = await collectBufferedResponsesSse(source(), upstream); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.kind).toBe(_name === "oversized frame" ? "oversized" : "read_error"); + }); + + test("bounded collector applies its first-byte deadline to a silent SSE body", async () => { + const source = new ReadableStream({ pull: () => new Promise(() => {}) }); + const result = await collectBufferedResponsesSse(source, new AbortController(), { + read: { firstByteTimeoutMs: 5, inactivityTimeoutMs: 5, totalTimeoutMs: 20 }, + }); + expect(result).toMatchObject({ ok: false, kind: "timeout" }); + }); + + test("disabled stall budget falls back to the independent buffered-turn ceiling", async () => { + const startedAt = 1_000; + expect(bufferedResponsesReadOptions(0, BUFFERED_RESPONSES_TOTAL_TIMEOUT_MS, startedAt)).toEqual({ + deadlineAt: startedAt + BUFFERED_RESPONSES_TOTAL_TIMEOUT_MS, + firstByteTimeoutMs: BUFFERED_RESPONSES_TOTAL_TIMEOUT_MS, + inactivityTimeoutMs: BUFFERED_RESPONSES_TOTAL_TIMEOUT_MS, + totalTimeoutMs: BUFFERED_RESPONSES_TOTAL_TIMEOUT_MS, + }); + expect(bufferedResponsesReadOptions(2_500, BUFFERED_RESPONSES_TOTAL_TIMEOUT_MS, startedAt)).toEqual({ + deadlineAt: startedAt + BUFFERED_RESPONSES_TOTAL_TIMEOUT_MS, + firstByteTimeoutMs: 2_500, + inactivityTimeoutMs: 2_500, + totalTimeoutMs: BUFFERED_RESPONSES_TOTAL_TIMEOUT_MS, + }); + + const body = new ReadableStream({ + start(controller) { + setTimeout(() => { + controller.enqueue(new TextEncoder().encode(sseEvent("response.completed", { + response: { id: "resp_delayed", status: "completed", output: [] }, + }))); + controller.close(); + }, 10); + }, + }); + const result = await collectBufferedResponsesSse(body, new AbortController(), { + read: bufferedResponsesReadOptions(0, 100), + }); + expect(result).toMatchObject({ ok: true, terminal: { status: "completed" } }); + + // A later validation pass must inherit the original absolute deadline instead of receiving + // a fresh totalTimeoutMs window merely because it constructed a new collector. + const expiredSharedRead = bufferedResponsesReadOptions(0, 100, Date.now() - 200); + const expired = await collectBufferedResponsesSse( + new Response(sseEvent("response.completed", { + response: { id: "resp_too_late", status: "completed", output: [] }, + })).body!, + new AbortController(), + { read: expiredSharedRead }, + ); + expect(expired).toMatchObject({ ok: false, kind: "timeout" }); + }); + + test("bare upstream refusal keeps its message, code, and non-retryable status", async () => { + globalThis.fetch = (async () => new Response(sseEvent("error", { + error: { + type: "invalid_request_error", + code: "invalid_prompt", + message: "The upstream rejected this prompt. Please try again in 120s.", + }, + }), { headers: { "content-type": "text/event-stream", "retry-after": "120" } })) as typeof fetch; + const terminals: string[] = []; + + const response = await call(requestBody(false, false), { + onNativePassthroughTerminal: status => terminals.push(status), + }); + + expect(response.status).toBe(400); + expect(response.headers.get("content-type")).toContain("application/json"); + expect(response.headers.get("retry-after")).toBeNull(); + expect(await response.json()).toEqual({ + error: { + type: "invalid_request_error", + code: "invalid_prompt", + message: "The upstream rejected this prompt. Please try again in 120s.", + }, + retryable: false, + }); + expect(terminals).toEqual(["failed"]); + }); + + test("bare non-refusal error copy cannot relabel a transport failure or account outcome", async () => { + globalThis.fetch = (async () => new Response(sseEvent("error", { + error: { + type: "upstream_error", + code: "upstream_reset", + message: "invalid api key while forwarding the upstream reset", + }, + }), { headers: { "content-type": "text/event-stream" } })) as typeof fetch; + const terminals: string[] = []; + + const response = await call(requestBody(false, false), { + onNativePassthroughTerminal: status => terminals.push(status), + }); + + expect(response.status).toBe(502); + expect(await response.json()).toEqual({ + error: { + type: "upstream_error", + code: "upstream_server_error", + message: "invalid api key while forwarding the upstream reset", + }, + }); + expect(terminals).toEqual(["failed"]); + }); + + test("bare structured rate-limit error preserves its authoritative family", async () => { + globalThis.fetch = (async () => new Response(sseEvent("error", { + code: "rate_limit_exceeded", + message: "Rate limit exceeded.", + }), { headers: { "content-type": "text/event-stream" } })) as typeof fetch; + const terminals: string[] = []; + + const response = await call(requestBody(false, false), { + onNativePassthroughTerminal: status => terminals.push(status), + }); + + expect(response.status).toBe(429); + expect(await response.json()).toEqual({ + error: { + type: "rate_limit_error", + code: "rate_limit_exceeded", + message: "Rate limit exceeded.", + }, + }); + expect(terminals).toEqual(["failed"]); + }); + + test("bare structured overload keeps its server class and 503 status", async () => { + globalThis.fetch = (async () => new Response(sseEvent("error", { + error: { + type: "server_error", + code: "server_is_overloaded", + message: "The upstream is overloaded.", + }, + }), { headers: { "content-type": "text/event-stream" } })) as typeof fetch; + + const response = await call(requestBody(false, false)); + + expect(response.status).toBe(503); + expect(await response.json()).toEqual({ + error: { + type: "server_error", + code: "server_is_overloaded", + message: "The upstream is overloaded.", + }, + }); + }); + + test("incomplete terminals retain already-finished output items and terminal usage", () => { + const partial = { + type: "message", id: "msg_partial", status: "incomplete", role: "assistant", + content: [{ type: "output_text", text: "partial", annotations: [] }], + }; + const inspected = inspectBufferedResponsesTerminal([ + sseEvent("response.output_item.done", { output_index: 0, item: partial }), + sseEvent("response.incomplete", { response: { + id: "resp_partial", + status: "incomplete", + output: [], + incomplete_details: { reason: "max_output_tokens" }, + usage: { input_tokens: 4, output_tokens: 2, total_tokens: 6 }, + } }), + ].join("")); + expect(inspected).toEqual({ + ok: true, + terminal: { + status: "incomplete", + response: { + id: "resp_partial", + status: "incomplete", + output: [partial], + incomplete_details: { reason: "max_output_tokens" }, + usage: { input_tokens: 4, output_tokens: 2, total_tokens: 6 }, + }, + }, + }); + }); + + test("comment-only SSE heartbeats do not taint a valid terminal", () => { + const inspected = inspectBufferedResponsesTerminal([ + ": upstream heartbeat\n\n", + sseEvent("response.completed", { + response: { id: "resp_heartbeat", status: "completed", output: [] }, + }), + ].join("")); + expect(inspected).toEqual({ + ok: true, + terminal: { + status: "completed", + response: { id: "resp_heartbeat", status: "completed", output: [] }, + }, + }); + }); + + test("merges a partial nonempty terminal with later done items without loss", () => { + const first = { type: "message", id: "msg_0", status: "completed", role: "assistant", content: [] }; + const second = { type: "function_call", id: "fc_1", status: "completed", call_id: "call_1", name: "lookup", arguments: "{}" }; + const inspected = inspectBufferedResponsesTerminal([ + sseEvent("response.output_item.done", { output_index: 1, item: second }), + sseEvent("response.completed", { response: { + id: "resp_merge", status: "completed", output: [first], + } }), + ].join("")); + expect(inspected).toMatchObject({ ok: true, terminal: { response: { output: [first, second] } } }); + }); + + test("merges an identity-matched sparse terminal item without shifting output indices", () => { + const first = { type: "reasoning", id: "rs_0", summary: [] }; + const second = { type: "message", id: "msg_1", status: "completed", role: "assistant", content: [] }; + const inspected = inspectBufferedResponsesTerminal([ + sseEvent("response.output_item.done", { output_index: 0, item: first }), + sseEvent("response.output_item.done", { output_index: 1, item: second }), + sseEvent("response.completed", { response: { + id: "resp_sparse", status: "completed", output: [second], + } }), + ].join("")); + expect(inspected).toMatchObject({ ok: true, terminal: { response: { output: [first, second] } } }); + }); + + test("rejects contradictory duplicate done events for one output index", () => { + const first = { type: "message", id: "msg_a", status: "completed", role: "assistant", content: [] }; + const conflicting = { ...first, id: "msg_b" }; + expect(inspectBufferedResponsesTerminal([ + sseEvent("response.output_item.done", { output_index: 0, item: first }), + sseEvent("response.output_item.done", { output_index: 0, item: conflicting }), + sseEvent("response.completed", { response: { id: "resp_duplicate", status: "completed", output: [] } }), + ].join(""))).toMatchObject({ ok: false, kind: "malformed" }); + }); + + test("out-of-order done items are sorted when their indices are contiguous", () => { + const zero = { type: "message", id: "msg_0", status: "completed", role: "assistant", content: [] }; + const one = { type: "reasoning", id: "rs_1", summary: [] }; + const inspected = inspectBufferedResponsesTerminal([ + sseEvent("response.output_item.done", { output_index: 1, item: one }), + sseEvent("response.output_item.done", { output_index: 0, item: zero }), + sseEvent("response.completed", { response: { id: "resp_order", status: "completed", output: [] } }), + ].join("")); + expect(inspected).toMatchObject({ ok: true, terminal: { response: { output: [zero, one] } } }); + }); + + test.each([ + ["index gap", sseEvent("response.output_item.done", { + output_index: 1, + item: { type: "message", id: "msg_1", status: "completed", role: "assistant", content: [] }, + })], + ["configured item cap", sseEvent("response.output_item.done", { + output_index: 10_000, + item: { type: "message", id: "msg_cap", status: "completed", role: "assistant", content: [] }, + })], + ])("fails closed when reconstruction is tainted by %s", (_name, itemFrame) => { + const inspected = inspectBufferedResponsesTerminal(itemFrame + sseEvent("response.completed", { + response: { id: "resp_tainted", status: "completed", output: [] }, + })); + expect(inspected).toMatchObject({ ok: false, kind: "malformed" }); + }); + + test("rejects a terminal whose response status disagrees with its event", () => { + expect(inspectBufferedResponsesTerminal(sseEvent("response.completed", { + response: { id: "resp_mismatch", status: "incomplete", output: [] }, + }))).toMatchObject({ ok: false, kind: "malformed" }); + }); + + test("derives a missing response status from the terminal event", () => { + expect(inspectBufferedResponsesTerminal(sseEvent("response.failed", { + response: { id: "resp_failed", error: { code: "tool_not_allowed", message: "blocked" } }, + }))).toMatchObject({ + ok: true, + terminal: { status: "failed", response: { status: "failed", output: [] } }, + }); + }); + + test.each(["response.output_text.delta", "response.function_call_arguments.delta"])( + "rejects sparse completion after %s without output_item.done", + type => { + const delta = type === "response.output_text.delta" + ? { output_index: 0, item_id: "msg_0", delta: "partial" } + : { output_index: 0, item_id: "fc_0", call_id: "call_0", delta: "{\"q\":" }; + expect(inspectBufferedResponsesTerminal([ + sseEvent(type, delta), + sseEvent("response.completed", { response: { id: "resp_open", status: "completed", output: [] } }), + ].join(""))).toMatchObject({ ok: false, kind: "malformed" }); + }, + ); + + test("an authoritative terminal item covers an index whose delta had no done event", () => { + const item = { + type: "message", id: "msg_terminal", status: "completed", role: "assistant", + content: [{ type: "output_text", text: "whole", annotations: [] }], + }; + expect(inspectBufferedResponsesTerminal([ + sseEvent("response.output_text.delta", { output_index: 0, item_id: "msg_terminal", delta: "whole" }), + sseEvent("response.completed", { response: { id: "resp_covered", status: "completed", output: [item] } }), + ].join(""))).toMatchObject({ ok: true, terminal: { response: { output: [item] } } }); + }); + + test("aggregate input budget rejects one large multi-frame network chunk before terminal", async () => { + const body = new Response(": keepalive\n\n".repeat(200)).body!; + const result = await collectBufferedResponsesSse(body, new AbortController(), { + read: { maxBytes: 1024, firstByteTimeoutMs: 100, inactivityTimeoutMs: 100, totalTimeoutMs: 100 }, + }); + expect(result).toMatchObject({ ok: false, kind: "oversized" }); + }); + + test("aggregate frame budget rejects tiny-event amplification across reader chunks", async () => { + const bytes = new TextEncoder().encode(": heartbeat\n\n".repeat(4) + + sseEvent("response.completed", { response: { id: "resp_late", status: "completed", output: [] } })); + const body = new ReadableStream({ + start(controller) { + controller.enqueue(bytes.subarray(0, 28)); + controller.enqueue(bytes.subarray(28)); + controller.close(); + }, + }); + const result = await collectBufferedResponsesSse(body, new AbortController(), { + read: { maxFrames: 3, firstByteTimeoutMs: 100, inactivityTimeoutMs: 100, totalTimeoutMs: 100 }, + }); + expect(result).toMatchObject({ ok: false, kind: "oversized" }); + }); +}); diff --git a/tests/responses/responses-pool-401-refresh.test.ts b/tests/responses/responses-pool-401-refresh.test.ts index 2fc4409c130..c6a6f0450bc 100644 --- a/tests/responses/responses-pool-401-refresh.test.ts +++ b/tests/responses/responses-pool-401-refresh.test.ts @@ -8,6 +8,7 @@ import { codexPoolAffinityKey } from "../../src/codex/auth-context"; import { clearCodexUpstreamHealth, clearThreadAccountMap, + peekConversationStateIssuer, resolveCodexAccountForThreadDetailed, } from "../../src/codex/routing"; import { handleResponses, handleResponsesCompact } from "../../src/server/responses"; @@ -20,6 +21,7 @@ import { import { clearResponseStateForTests, clearResponseStateMemoryForTests, + expandPreviousResponseInput, responseContinuationRetainedStoreSnapshot, runPendingResponseStatePersistForTests, } from "../../src/responses/state"; @@ -339,6 +341,92 @@ afterEach(() => { }); describe("ordinary pool 401 refresh and replay (#2887)", () => { + test.each([false, true])("failed terminal cannot echo the selected pool credential or retain it for replay (stream:%s)", async stream => { + const harness = installHarness({ + responseForSend(authorization) { + return new Response( + `event: response.failed\ndata: ${JSON.stringify({ + type: "response.failed", + response: { + id: "resp_failed_pool_echo", status: "failed", + error: { type: "server_error", code: "upstream_error", message: `rejected ${authorization}` }, + last_error: { detail: "raw token rejected-access" }, + metadata: { diagnostic: "raw token rejected-access" }, + output: [{ + type: "message", id: "msg_pool_echo", status: "completed", role: "assistant", + content: [{ type: "output_text", text: "rejected-access", annotations: [] }], + }], + }, + detail: "raw token rejected-access", + })}\n\n`, + { headers: { "content-type": "text/event-stream" } }, + ); + }, + }); + const response = await handleResponses( + request("/v1/responses", { stream }), config(), { model: "", provider: "" } as RequestLogContext, + ); + const body = await response.text(); + expect(harness.sends).toEqual(["Bearer rejected-access"]); + expect(response.status).toBe(200); + if (stream) expect(body).toContain("event: response.failed"); + else expect(JSON.parse(body)).toMatchObject({ id: "resp_failed_pool_echo", status: "failed" }); + expect(body).not.toContain("rejected-access"); + expect(body).toContain("[REDACTED]"); + const next = { model: "gpt-5.5", previous_response_id: "resp_failed_pool_echo", input: "retry" }; + expect(expandPreviousResponseInput(next)).toEqual(next); + }); + + test("bare upstream SSE error masks the selected pool credential in buffered JSON and diagnostics", async () => { + const harness = installHarness({ + responseForSend(authorization) { + return new Response(`event: error\ndata: ${JSON.stringify({ + type: "error", + error: { type: "server_error", code: "upstream_error", message: `rejected ${authorization.slice(7)}` }, + })}\n\n`, { headers: { "content-type": "text/event-stream" } }); + }, + }); + const logCtx = { model: "", provider: "" } as RequestLogContext; + const response = await handleResponses(request("/v1/responses"), config(), logCtx); + const body = await response.text(); + expect(harness.sends).toEqual(["Bearer rejected-access"]); + expect(response.status).toBe(502); + expect(body).not.toContain("rejected-access"); + expect(body).toContain("[REDACTED]"); + expect(JSON.stringify(logCtx)).not.toContain("rejected-access"); + }); + + test("disconnect during buffered replay does not record a pool conversation-state issuer", async () => { + const abort = new AbortController(); + const headers = { "thread-id": "fixture-pool-disconnect", "x-codex-parent-thread-id": "fixture-pool-parent" }; + const affinityKey = codexPoolAffinityKey(new Headers(headers)); + expect(affinityKey).toBeDefined(); + installHarness({ + responseForSend() { + const delta = `event: response.output_text.delta\ndata: ${JSON.stringify({ + type: "response.output_text.delta", output_index: 0, item_id: "msg_pool_disconnect", delta: "x", + })}\n\n`; + const item = { + type: "message", id: "msg_pool_disconnect", status: "completed", role: "assistant", + content: [{ type: "output_text", text: "done", annotations: [] }], + }; + return new Response([ + ...Array.from({ length: 14_000 }, () => delta), + `event: response.output_item.done\ndata: ${JSON.stringify({ type: "response.output_item.done", output_index: 0, item })}\n\n`, + `event: response.completed\ndata: ${JSON.stringify({ type: "response.completed", response: { + id: "resp_pool_disconnect", status: "completed", model: "gpt-5.5", output: [item], + } })}\n\n`, + ].join(""), { headers: { "content-type": "text/event-stream" } }); + }, + }); + const response = await handleResponses( + request("/v1/responses", { headers }), config(), { model: "", provider: "" } as RequestLogContext, + { abortSignal: abort.signal, onFirstOutput: () => setTimeout(() => abort.abort(), 0) }, + ); + expect(response.status).toBe(499); + expect(peekConversationStateIssuer(affinityKey!)).toBeUndefined(); + }); + test("Responses refreshes a time-valid stored credential once and replays the same account", async () => { const harness = installHarness(); const response = await handleResponses( diff --git a/tests/responses/responses-preview-main-read-fence.test.ts b/tests/responses/responses-preview-main-read-fence.test.ts index 08008b6c1ae..f84b52ec615 100644 --- a/tests/responses/responses-preview-main-read-fence.test.ts +++ b/tests/responses/responses-preview-main-read-fence.test.ts @@ -4,11 +4,13 @@ import { mkdtempSync, unlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { saveCodexAccountCredential } from "../../src/codex/account-store"; +import { resetMainCodexAccountIdentityTrackingForTests } from "../../src/codex/account-lifecycle"; import { resolveCodexAuthContext, type CodexAuthContext, } from "../../src/codex/auth-context"; import { getMainAccountToken, MAIN_CODEX_ACCOUNT_ID } from "../../src/codex/main-account"; +import { clearMainAccountInfoCache } from "../../src/codex/main-account-cache"; import { resetCodexModelEntitlementCacheForTests, seedCodexModelEntitlementsForTests, @@ -38,6 +40,7 @@ import type { OcxConfig } from "../../src/types"; import { codexHeaders, encryptedInput, + fakeChatGptJwt, recoverySse, } from "../helpers/agent-task-recovery"; import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; @@ -59,6 +62,9 @@ import { removeTreeWithRetry } from "../helpers/remove-tree"; const NOW = 1_800_000_000_000; const PREFERRED_MODEL = "gpt-5.6-sol"; const FALLBACK_MODEL = "xai/grok-4.5"; +const POOL_ACCESS_TOKEN = fakeChatGptJwt("pool-account", { + exp: Math.floor((NOW + 24 * 60 * 60_000) / 1_000), +}); const originalFetch = globalThis.fetch; const originalNow = Date.now; @@ -108,7 +114,7 @@ function installCredentials(): void { }, })); saveCodexAccountCredential("pool-a", { - accessToken: "pool-access-token", + accessToken: POOL_ACCESS_TOKEN, refreshToken: "pool-refresh-token", expiresAt: NOW + 24 * 60 * 60_000, chatgptAccountId: "pool-account", @@ -166,6 +172,21 @@ function completedResponses(model = PREFERRED_MODEL): Response { }); } +/** Canonical ChatGPT always answers the adapter-forced upstream stream with SSE. */ +function completedCanonicalSse(model = PREFERRED_MODEL): Response { + const response = { + id: "resp_main_read_fence", + object: "response", + status: "completed", + model, + output: [], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + }; + return new Response(`data: ${JSON.stringify({ type: "response.completed", response })}\n\n`, { + headers: { "content-type": "text/event-stream" }, + }); +} + function readableInput(): unknown[] { return [{ type: "message", @@ -213,6 +234,8 @@ beforeEach(() => { clearThreadAccountMap(); clearCodexUpstreamHealth(); clearAccountQuota(); + clearMainAccountInfoCache(); + resetMainCodexAccountIdentityTrackingForTests(); clearComboSelectionState(); clearComboTargetCooldowns(); clearResponseStateMemoryForTests(); @@ -245,6 +268,8 @@ afterEach(() => { clearThreadAccountMap(); clearCodexUpstreamHealth(); clearAccountQuota(); + clearMainAccountInfoCache(); + resetMainCodexAccountIdentityTrackingForTests(); clearComboSelectionState(); clearComboTargetCooldowns(); clearResponseStateForTests(); @@ -269,6 +294,9 @@ describe("preview and final authentication agree on the native-main read fence", const owned = await resolveCodexAuthContext(codexHeaders("caller-account"), config, "pool", { requestScopedMainCredential: true, modelId: PREFERRED_MODEL, + // This unit observes auth fencing, not the detached quota-prime worker. Without the seam, + // the unowned control below can keep reading into the next case after its assertion ends. + primeCodexPoolQuotas: async () => {}, }); expect(owned).toMatchObject({ kind: "pool", accountId: "pool-a" }); expect(authJsonReads).toBe(0); @@ -278,6 +306,7 @@ describe("preview and final authentication agree on the native-main read fence", await resolveCodexAuthContext(new Headers(), config, "pool", { requestScopedMainCredential: true, modelId: PREFERRED_MODEL, + primeCodexPoolQuotas: async () => {}, }); expect(authJsonReads).toBeGreaterThan(0); }); @@ -286,15 +315,18 @@ describe("preview and final authentication agree on the native-main read fence", seedMainDenial(); calibrateMainReadCounter(); const upstreamAuth: Array = []; - globalThis.fetch = (async (_input, init) => { - upstreamAuth.push(new Headers(init?.headers).get("authorization")); - return completedResponses(); + globalThis.fetch = (async (input, init) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + if (url.pathname.endsWith("/responses")) { + upstreamAuth.push(new Headers(init?.headers).get("authorization")); + } + return completedCanonicalSse(); }) as typeof fetch; const response = await postSpawn(providerConfig()); expect(response.status).toBe(200); - expect(upstreamAuth).toEqual(["Bearer pool-access-token"]); + expect(upstreamAuth).toEqual([`Bearer ${POOL_ACCESS_TOKEN}`]); expect(denialCacheMainReadStacks()).toEqual([]); }); @@ -313,15 +345,18 @@ describe("preview and final authentication agree on the native-main read fence", seedMainDenial(); calibrateMainReadCounter(); const upstreamAuth: Array = []; - globalThis.fetch = (async (_input, init) => { - upstreamAuth.push(new Headers(init?.headers).get("authorization")); - return completedResponses(); + globalThis.fetch = (async (input, init) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + if (url.pathname.endsWith("/responses")) { + upstreamAuth.push(new Headers(init?.headers).get("authorization")); + } + return completedCanonicalSse(); }) as typeof fetch; const response = await postSpawn(providerConfig()); expect(response.status).toBe(200); - expect(upstreamAuth).toEqual(["Bearer pool-access-token"]); + expect(upstreamAuth).toEqual([`Bearer ${POOL_ACCESS_TOKEN}`]); expect(authJsonReadStacks).toEqual([]); expect(authJsonReads).toBe(0); }); @@ -329,7 +364,7 @@ describe("preview and final authentication agree on the native-main read fence", test("the initial preview also fences main for recovery blocking and selector drain", async () => { seedMainDenial(); calibrateMainReadCounter(); - globalThis.fetch = (async () => completedResponses()) as typeof fetch; + globalThis.fetch = (async () => completedCanonicalSse()) as typeof fetch; const snapshot = nativeMainStartupGateSnapshot(); blockedHomeId = snapshot.homeId ?? testDir; @@ -463,7 +498,7 @@ describe("preview and final authentication agree on the native-main read fence", upstreamUrls.push(String(input)); upstreamBodies.push(typeof init?.body === "string" ? init.body : ""); upstreamAuth.push(new Headers(init?.headers).get("authorization")); - return completedResponses(); + return completedCanonicalSse(); }) as typeof fetch; let finalAuth: CodexAuthContext | undefined; const logCtx: RequestLogContext = { model: "", provider: "" }; @@ -485,7 +520,7 @@ describe("preview and final authentication agree on the native-main read fence", expect((logCtx as unknown as Record).subagentModelFallbackTo).toBeUndefined(); // The caller's own bearer is what main is served from. Neither stored credential may appear, // and the denial-cache validator -- the one read this file exists to fence -- never ran. - expect(upstreamAuth[0]).not.toBe("Bearer pool-access-token"); + expect(upstreamAuth[0]).not.toBe(`Bearer ${POOL_ACCESS_TOKEN}`); expect(upstreamAuth[0]).not.toBe("Bearer physical-main-token"); expect(denialCacheMainReadStacks()).toEqual([]); }); @@ -518,12 +553,6 @@ describe("preview and final authentication agree on the native-main read fence", expect((logCtx as unknown as Record).subagentModelFallbackTo).toBe(FALLBACK_MODEL); }); - // The two cases below are last on purpose. Both let a request reach native main, and observing - // a main credential writes module state in `main-account-cache.ts` that no reset helper in this - // file clears -- `beforeEach` rebuilds `OPENCODEX_HOME` and the read counters, not that cache. - // Running them earlier made the recovery/drain case above see three reads it does not make on - // its own. Keep read-count assertions ahead of them. - /** * The other half of #4850, and the reason the seam is scoped to * `previewRequestScopedMainCredential` instead of being applied to every preview. A fix that @@ -534,7 +563,7 @@ describe("preview and final authentication agree on the native-main read fence", test("a preview that owns no credential still probes physical main liveness", async () => { seedMainDenial(); calibrateMainReadCounter(); - globalThis.fetch = (async () => completedResponses()) as typeof fetch; + globalThis.fetch = (async () => completedCanonicalSse()) as typeof fetch; const response = await postSpawn(providerConfig(), {}, new Headers()); @@ -570,7 +599,7 @@ describe("preview and final authentication agree on the native-main read fence", globalThis.fetch = (async (_input, init) => { upstreamAuth.push(new Headers(init?.headers).get("authorization")); upstreamBodies.push(typeof init?.body === "string" ? init.body : ""); - return completedResponses(); + return completedCanonicalSse(); }) as typeof fetch; let finalAuth: CodexAuthContext | undefined; const logCtx: RequestLogContext = { model: "", provider: "" }; @@ -589,7 +618,7 @@ describe("preview and final authentication agree on the native-main read fence", expect(upstreamBodies[0]).toContain(`"model":"${PREFERRED_MODEL}"`); expect((logCtx as unknown as Record).subagentModelFallbackTo).toBeUndefined(); // The caller's own bearer is forwarded. Neither stored credential may appear. - expect(upstreamAuth[0]).not.toBe("Bearer pool-access-token"); + expect(upstreamAuth[0]).not.toBe(`Bearer ${POOL_ACCESS_TOKEN}`); expect(upstreamAuth[0]).not.toBe("Bearer physical-main-token"); }); }); diff --git a/tests/responses/sse-failed-tail.test.ts b/tests/responses/sse-failed-tail.test.ts index dfa030dd681..6da1461f5c0 100644 --- a/tests/responses/sse-failed-tail.test.ts +++ b/tests/responses/sse-failed-tail.test.ts @@ -1,9 +1,10 @@ import { describe, expect, test } from "bun:test"; import { relaySseWithFailedTail, relayWithAbort } from "../../src/server"; import { relaySseEagerBounded, type EagerRelayHooks } from "../../src/server/relay-eager"; -import { MAX_TAIL_ERROR_MESSAGE_CHARS } from "../../src/server/relay"; +import { MAX_TAIL_ERROR_MESSAGE_CHARS, upstreamErrorTailFrame } from "../../src/server/relay"; import { TERMINAL_REFUSAL_FALLBACK_MESSAGE } from "../../src/lib/errors"; import { TranslatorBudgetExceededError } from "../../src/lib/translator-budget"; +import { createOutboundCredentialMask } from "../../src/server/responses/terminal-error-redaction"; const encoder = new TextEncoder(); const decoder = new TextDecoder(); @@ -64,6 +65,26 @@ function doneEvents(text: string): string[] { } describe("relaySseWithFailedTail", () => { + test.each(["tee", "eager"] as const)("masks a JSON-escaped selected credential in a %s synthetic failure", async mode => { + const credential = 'fixture"quoted\\token'; + const maskCredential = createOutboundCredentialMask({ authorization: `Bearer ${credential}` }); + const src = sourceStream([], { failAfter: true, error: new Error(`reset after ${credential}`) }); + const out = await drain(mode === "tee" + ? relaySseWithFailedTail(src, new AbortController(), undefined, { maskCredential }) + : relaySseEagerBounded(src, new AbortController(), parityHooks, { maskCredential })); + expect(failedMessage(out)).toContain("[REDACTED]"); + expect(failedMessage(out)).not.toContain(credential); + }); + + test("masks selected credentials in every synthetic refusal field", () => { + const credential = "fixture-selected-credential"; + const maskCredential = createOutboundCredentialMask({ authorization: `Bearer ${credential}` }); + const frame = decoder.decode(upstreamErrorTailFrame( + encoder, `refused ${credential}`, `invalid_${credential}`, maskCredential, + )); + expect(frame).not.toContain(credential); + expect(frame).toContain("[REDACTED]"); + }); test("relays a healthy stream verbatim with no injected frame", async () => { const upstream = new AbortController(); const src = sourceStream(["event: response.completed\n", 'data: {"type":"response.completed"}\n\n', "data: [DONE]\n\n"]); diff --git a/tests/server/server-auth.test.ts b/tests/server/server-auth.test.ts index 55680463391..12ca927bc09 100644 --- a/tests/server/server-auth.test.ts +++ b/tests/server/server-auth.test.ts @@ -3005,6 +3005,35 @@ describe("server local API auth", () => { } }); + test("canonical buffered Pool retry remains HTTP-only and relays an alternate 307", async () => { + const redirectTarget = "https://dead.invalid/alternate"; + const harness = await startPoolRetryHarness(accountId => accountId === "acct-pool-a" + ? new Response(JSON.stringify({ error: { message: "rate limited" } }), { + status: 429, + headers: { "content-type": "application/json", "retry-after": "60" }, + }) + : new Response(null, { status: 307, headers: { location: redirectTarget } })); + const fixtureWebSocket = globalThis.WebSocket; + let upstreamWebSocketAttempts = 0; + globalThis.WebSocket = new Proxy(fixtureWebSocket, { + construct(target, args, newTarget) { + const url = new URL(String(args[0])); + if (url.protocol === "wss:" && url.hostname === "chatgpt.com") upstreamWebSocketAttempts += 1; + return Reflect.construct(target, args, newTarget); + }, + }); + try { + const response = await harness.request({ redirect: "manual" }); + expect(response.status).toBe(307); + expect(response.headers.get("location")).toBe(redirectTarget); + expect(harness.dispatches).toEqual(["acct-pool-a", "acct-pool-b"]); + expect(upstreamWebSocketAttempts).toBe(0); + } finally { + globalThis.WebSocket = fixtureWebSocket; + await stopPoolRetryHarness(harness); + } + }); + test.each([429, 402] as const)( "a pre-stream %i from the only Pool account retries once with the validated caller main", async rejection => { diff --git a/tests/usage/request-log-nonstream.test.ts b/tests/usage/request-log-nonstream.test.ts index ac2608a1ca2..bd6c3759008 100644 --- a/tests/usage/request-log-nonstream.test.ts +++ b/tests/usage/request-log-nonstream.test.ts @@ -1,6 +1,8 @@ import { describe, expect, test } from "bun:test"; -import { responseWithDeferredRequestLog } from "../../src/server/relay"; +import { markPreinspectedJsonResponse, responseWithDeferredRequestLog } from "../../src/server/relay"; import { MAX_RESPONSE_LOG_INSPECTION_BYTES } from "../../src/server/response-log-body"; +import { finalizeAccountLease, finalizeOwnedTranslatorBudget } from "../../src/server/responses/core-lifetime"; +import { createTranslatorBudget } from "../../src/lib/translator-budget"; import type { RequestLogContext, RequestLogEntry } from "../../src/server/request-log"; const encoder = new TextEncoder(); @@ -45,6 +47,31 @@ describe("deferred non-stream request log integration", () => { expect(entries[0]?.status).toBe(200); }); + test("preinspected JSON keeps EOF logging without reparsing the response body", async () => { + const payload = JSON.stringify({ + model: "must-not-replace", + usage: { input_tokens: 99, output_tokens: 99, total_tokens: 198 }, + }); + const logCtx: RequestLogContext = { + model: "requested-model", + provider: "fixture-provider", + resolvedModel: "already-inspected", + usage: { inputTokens: 3, outputTokens: 2, totalTokens: 5 }, + }; + const marked = markPreinspectedJsonResponse(new Response(payload, { + headers: { "content-type": "application/json" }, + })); + const finalized = finalizeAccountLease( + finalizeOwnedTranslatorBudget(marked, createTranslatorBudget()), + () => undefined, + ); + const { result, entries } = tracked(finalized, logCtx); + expect(await result.text()).toBe(payload); + expect(entries).toHaveLength(1); + expect(entries[0]?.resolvedModel).toBe("already-inspected"); + expect(entries[0]?.usage).toEqual({ inputTokens: 3, outputTokens: 2, totalTokens: 5 }); + }); + test("does not overwrite routed model/usage context from oversized JSON", async () => { const payload = JSON.stringify({ model: "do-not-inspect", padding: "x".repeat(MAX_RESPONSE_LOG_INSPECTION_BYTES) }); const { result, entries, logCtx } = tracked(new Response(payload, { headers: { "content-type": "application/json" } })); From b78bfb8f000078741cbe90f9d74d12afd3d0d1dc Mon Sep 17 00:00:00 2001 From: ingwannu Date: Wed, 30 Sep 2026 05:56:31 +0900 Subject: [PATCH 24/26] fix(oauth): recover Anthropic reset cooldowns (#6203) * fix(oauth): recover Anthropic reset cooldowns * docs(accounts): document Anthropic cooldown clear * docs(accounts): localize Anthropic cooldown clear * fix(oauth): fence Anthropic quota flights and cache ownership Retire old joinable usage flights during cooldown recovery, reject superseded failures before negative-cache publication, and retain live ownership for cached quota reads. * fix(quota): fence stale Anthropic token failures by cooldown flight --------- Co-authored-by: Ingwannu Co-authored-by: JUN --- .../000_decision.md | 46 +++ .../fr/reference/cli/providers-accounts.md | 17 +- .../ja/reference/cli/providers-accounts.md | 17 +- .../ko/reference/cli/providers-accounts.md | 16 +- .../docs/reference/cli/providers-accounts.md | 16 +- .../ru/reference/cli/providers-accounts.md | 17 +- .../tr/reference/cli/providers-accounts.md | 17 +- .../zh-cn/reference/cli/providers-accounts.md | 15 +- .../zh-tw/reference/cli/providers-accounts.md | 15 +- scripts/test-layout/layout.json | 1 + src/cli/account-extended.ts | 47 ++- src/oauth/anthropic-routing.ts | 84 +++- src/providers/quota.ts | 92 ++--- src/providers/quota/account-cache.ts | 3 +- .../quota/anthropic-cooldown-recovery.ts | 118 ++++++ src/providers/quota/vendor-probes-oauth.ts | 143 ++++--- structure/providers-and-adapters.md | 5 + .../anthropic-cooldown-recovery.test.ts | 365 ++++++++++++++++++ tests/cli/cli-account-pool-verbs.test.ts | 67 +++- tests/fixtures/test-layout-expected.json | 1 + 20 files changed, 958 insertions(+), 144 deletions(-) create mode 100644 devlog/_fin/260928_anthropic_cooldown_recovery/000_decision.md create mode 100644 src/providers/quota/anthropic-cooldown-recovery.ts create mode 100644 tests/adapters/anthropic/anthropic-cooldown-recovery.test.ts diff --git a/devlog/_fin/260928_anthropic_cooldown_recovery/000_decision.md b/devlog/_fin/260928_anthropic_cooldown_recovery/000_decision.md new file mode 100644 index 00000000000..eb75d9b5ecd --- /dev/null +++ b/devlog/_fin/260928_anthropic_cooldown_recovery/000_decision.md @@ -0,0 +1,46 @@ +# Anthropic cooldown recovery ownership + +## Decision log + +- Purpose: let an authoritative Anthropic usage refresh release a stale reset-derived + cooldown without weakening explicit upstream backoff or clearing another account's state. +- Existing constraints: routing health is process-local, usage probes are asynchronous, and + credentials or a newer 429 can replace the state observed when a probe starts. +- Alternatives considered: clear every cooldown after any successful usage response; clear + only through the operator endpoint; or bind recovery to the observed refusal and credential. +- Decision: a probe captures the exact account credential generation and cooldown generation + before dispatch. Settlement requires the same live credential, the same reset-derived + cooldown, a fresh timestamp, and utilization below 100% for every window that the 429 marked + rejected. Account-level single-flight keys also include an active recovery generation, so a + forced post-429 refresh cannot join work dispatched before the refusal. Any later cooldown + mutation revokes publication ownership as well as settlement ownership. Partial, failed, + exhausted, stale, Retry-After, and default-backoff evidence does not recover anything. +- Why this option: a successful quota HTTP response alone says neither which credential it + measured nor whether a newer refusal arrived while it was in flight. Generation fences make + those ownership claims explicit while preserving the existing manual escape hatch. +- Impact and trade-off: recovered accounts re-enter routing immediately; uncertain evidence + remains fail-closed until expiry or `clear-cooldown`. The extra bookkeeping is process-local + and bounded to one generation plus one health record per account. Superseded probes return an + unavailable result instead of publishing quota that no longer describes the routing state. + +## Data flow + +1. A 429 records its source, rejected quota windows, and a monotonic cooldown generation. +2. A fresh usage probe captures that generation plus the stored credential generation. + When recovery is pending, both the provider-usage flight and the outer account-quota flight + are generation-scoped, so the probe dispatches after the claim instead of joining older work + that might describe pre-refusal state. +3. The usage response is parsed and checked for complete headroom evidence. +4. Publication and settlement both require the observed cooldown generation to remain current. + Settlement deletes only the still-matching reset-derived record. + +The CLI dispatches `openai` to `/api/codex-auth/accounts/clear-cooldown` and `anthropic` to +`/api/oauth/accounts/clear-cooldown`. Anthropic IDs and aliases are resolved through the OAuth +account list before the write; other providers remain rejected because they do not expose this +process-local cooldown owner. + +## Focused verification + +- `tests/providers/anthropic-cooldown-recovery.test.ts` +- `tests/cli/cli-account-pool-verbs.test.ts` +- `tests/adapters/anthropic/anthropic-ratelimit-headers.test.ts` diff --git a/docs-site/src/content/docs/fr/reference/cli/providers-accounts.md b/docs-site/src/content/docs/fr/reference/cli/providers-accounts.md index 3c43a780a24..91b74330991 100644 --- a/docs-site/src/content/docs/fr/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/fr/reference/cli/providers-accounts.md @@ -118,7 +118,7 @@ alias Set or clear an account's display pause Hold an account out of automatic selection. resume Return a paused account to automatic selection. pause-exhausted Pause every account whose quota is spent. -clear-cooldown Drop a cooldown the proxy set after an upstream failure. +clear-cooldown Drop a cooldown the proxy set after an upstream failure. strategy [] Stratégie du pool ; least-loaded est réservé à Kiro. sticky [<1-100>] Requests a bound thread keeps on one account; omit the value to read it. priority [first|earlier|normal|later|last|-100..100|reset] Selection order; omit the value to read it. @@ -222,6 +222,21 @@ La commande CLI suspend ou reprend un compte Anthropic OAuth par id ou alias uni Efface la sélection manuelle du compte Codex sans résoudre d'id de compte, donc fonctionne même lorsqu'un compte s'appelle littéralement `auto`. Pools Codex uniquement ; les autres types de fournisseur n'ont pas de sélection automatique à rétablir. +### `ocx account clear-cooldown [--json]` + +Supprime un délai d'échec local au processus sans modifier les identifiants enregistrés. Utilisez +`openai` pour un compte du pool Codex ou `anthropic` pour un compte OAuth Anthropic ; les autres +fournisseurs sont refusés. Les deux formes acceptent un id de compte ou un alias unique, tandis que +`main` est réservé au pool Codex. + +```bash +ocx account clear-cooldown anthropic +``` + +La commande réussit même sans délai actif, avec `cleared: false` dans le JSON. La suppression d'un +délai Anthropic avance aussi la génération du compte afin qu'une ancienne sonde de quota ne puisse +pas rétablir l'état supprimé ni publier une éligibilité périmée. + ### `ocx account refresh [--json]` Pour le groupe de comptes Codex, utilisez `ocx account refresh openai [--json]`. Cette commande force l'actualisation des quotas de compte et diff --git a/docs-site/src/content/docs/ja/reference/cli/providers-accounts.md b/docs-site/src/content/docs/ja/reference/cli/providers-accounts.md index a97def7a7c6..5f46501bbb9 100644 --- a/docs-site/src/content/docs/ja/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/ja/reference/cli/providers-accounts.md @@ -92,7 +92,7 @@ alias Set or clear an account's display pause Hold an account out of automatic selection. resume Return a paused account to automatic selection. pause-exhausted Pause every account whose quota is spent. -clear-cooldown Drop a cooldown the proxy set after an upstream failure. +clear-cooldown Drop a cooldown the proxy set after an upstream failure. strategy [] Pool placement strategy; omit the value to read it. sticky [<1-100>] Requests a bound thread keeps on one account; omit the value to read it. priority [first|earlier|normal|later|last|-100..100|reset] Selection order; omit the value to read it. @@ -167,6 +167,21 @@ CLI コマンドは Anthropic OAuth アカウントを id または一意の別 アカウント id を解決せずに Codex アカウントの手動選択を解除するため、`auto` という id のアカウントが存在しても機能します。Codex プール専用です。他のプロバイダー種別には復元する自動選択がありません。 +### `ocx account clear-cooldown [--json]` + +保存済み認証情報を変更せず、プロセスローカルな障害 cooldown を解除します。Codex Pool +アカウントには `openai`、Anthropic OAuth アカウントには `anthropic` を使い、その他の +provider は拒否されます。どちらもアカウント id または一意の alias を受け付けますが、 +`main` は Codex Pool 専用です。 + +```bash +ocx account clear-cooldown anthropic +``` + +有効な cooldown がなくても成功し、JSON では `cleared: false` になります。Anthropic の +cooldown を解除するとアカウント generation も進むため、古い quota probe が解除済み状態を +復元したり、古い quota ベースの eligibility を公開したりできません。 + ### `ocx account refresh [--json]` Codex プールの場合は、`ocx account refresh openai [--json]` を使用します。アカウント クォータを強制的に更新し、利用可能な週次/月次のパーセンテージとリセット時間を出力します。不足しているクォータ データは、0% ではなく不明として報告されます。その JSON エンベロープは `{ accounts: AccountRow[] }` で、Codex の各行に `quota` があります。 diff --git a/docs-site/src/content/docs/ko/reference/cli/providers-accounts.md b/docs-site/src/content/docs/ko/reference/cli/providers-accounts.md index b87fb2dd9c6..7f8c672ca7f 100644 --- a/docs-site/src/content/docs/ko/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/ko/reference/cli/providers-accounts.md @@ -187,7 +187,7 @@ alias Set or clear an account's display pause Hold an account out of automatic selection. resume Return a paused account to automatic selection. pause-exhausted Pause every account whose quota is spent. -clear-cooldown Drop a cooldown the proxy set after an upstream failure. +clear-cooldown Drop a cooldown the proxy set after an upstream failure. strategy [] Pool placement strategy; omit the value to read it. sticky [<1-100>] Requests a bound thread keeps on one account; omit the value to read it. priority [first|earlier|normal|later|last|-100..100|reset] Selection order; omit the value to read it. @@ -262,6 +262,20 @@ CLI 명령은 Anthropic OAuth 계정을 id 또는 유일한 alias로 일시 정 계정 id를 해석하지 않고 Codex 계정의 수동 선택을 지우므로 `auto`라는 id의 계정이 있어도 동작합니다. Codex 풀 전용이며 다른 공급자 유형에는 복원할 자동 선택이 없습니다. +### `ocx account clear-cooldown [--json]` + +저장된 자격 증명은 바꾸지 않고 프로세스 로컬 실패 cooldown을 해제합니다. Codex Pool 계정에는 +`openai`, Anthropic OAuth 계정에는 `anthropic`을 사용하며 다른 provider는 거부됩니다. 두 경로 +모두 계정 id 또는 고유 alias를 받지만 `main`은 Codex Pool에서만 사용할 수 있습니다. + +```bash +ocx account clear-cooldown anthropic +``` + +활성 cooldown이 없어도 명령은 성공하고 JSON에는 `cleared: false`가 표시됩니다. Anthropic +cooldown을 해제하면 계정 generation도 전진하므로 이전 quota probe가 해제된 상태를 되살리거나 +오래된 quota 기반 eligibility를 게시할 수 없습니다. + ### `ocx account refresh [--json]` Codex 풀에는 `ocx account refresh openai [--json]`를 사용합니다. 계정 할당량을 강제로 새로 고치고 사용 가능 주간/월간 비율과 재설정 시간을 출력합니다. 할당량 데이터가 없으면 0%가 아니라 알 수 없음으로 보고합니다. JSON 봉투는 `{ accounts: AccountRow[] }`이며, Codex 행마다 `quota`가 붙습니다. diff --git a/docs-site/src/content/docs/reference/cli/providers-accounts.md b/docs-site/src/content/docs/reference/cli/providers-accounts.md index 738fec91484..46a973c1265 100644 --- a/docs-site/src/content/docs/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/reference/cli/providers-accounts.md @@ -251,7 +251,7 @@ alias Set or clear an account's display pause Hold an account out of automatic selection. resume Return a paused account to automatic selection. pause-exhausted Pause every account whose quota is spent. -clear-cooldown Drop a cooldown the proxy set after an upstream failure. +clear-cooldown Drop a cooldown the proxy set after an upstream failure. strategy [] Pool placement strategy; least-loaded is Kiro-only. sticky [<1-100>] Requests a bound thread keeps on one account; omit the value to read it. priority [first|earlier|normal|later|last|-100..100|reset] Selection order; omit the value to read it. @@ -489,6 +489,20 @@ ocx account pause google-antigravity ocx account resume google-antigravity ``` +### `ocx account clear-cooldown [--json]` + +Drops a process-local failure cooldown without changing stored credentials. Use `openai` for a Codex +pool account or `anthropic` for an Anthropic OAuth account; other providers are rejected. Both forms +accept an account id or unique alias, while `main` is specific to the Codex pool. + +```bash +ocx account clear-cooldown anthropic +``` + +The command reports success even when no cooldown is active, with `cleared: false` in JSON. Clearing +an Anthropic cooldown also advances the account generation so an older in-flight quota probe cannot +restore the cleared state or publish stale quota-derived eligibility afterward. + ### `ocx account refresh [--json]` For the Codex pool, use `ocx account refresh openai [--json]`. It force-refreshes account quotas and diff --git a/docs-site/src/content/docs/ru/reference/cli/providers-accounts.md b/docs-site/src/content/docs/ru/reference/cli/providers-accounts.md index 899eb314a8e..899594aa3ab 100644 --- a/docs-site/src/content/docs/ru/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/ru/reference/cli/providers-accounts.md @@ -107,7 +107,7 @@ alias Set or clear an account's display pause Hold an account out of automatic selection. resume Return a paused account to automatic selection. pause-exhausted Pause every account whose quota is spent. -clear-cooldown Drop a cooldown the proxy set after an upstream failure. +clear-cooldown Drop a cooldown the proxy set after an upstream failure. strategy [] Pool placement strategy; omit the value to read it. sticky [<1-100>] Requests a bound thread keeps on one account; omit the value to read it. priority [first|earlier|normal|later|last|-100..100|reset] Selection order; omit the value to read it. @@ -198,6 +198,21 @@ credential'а, это состояние тоже печатается, но к Снимает ручной выбор аккаунта Codex без разрешения id, поэтому работает, даже когда аккаунт буквально называется `auto`. Только для пулов Codex; у других типов провайдеров нет автоматического выбора для восстановления. +### `ocx account clear-cooldown [--json]` + +Снимает локальный для процесса cooldown после сбоя, не меняя сохранённые учётные данные. Используйте +`openai` для аккаунта пула Codex или `anthropic` для OAuth-аккаунта Anthropic; другие провайдеры +отклоняются. Обе формы принимают id аккаунта или уникальный псевдоним, а `main` относится только к +пулу Codex. + +```bash +ocx account clear-cooldown anthropic +``` + +Команда завершается успешно и без активного cooldown, возвращая `cleared: false` в JSON. При снятии +cooldown Anthropic также увеличивается поколение аккаунта, поэтому старый quota probe не сможет +восстановить снятое состояние или опубликовать устаревшую доступность. + ### `ocx account refresh [--json]` Для пула Codex используйте `ocx account refresh openai [--json]`. Команда принудительно diff --git a/docs-site/src/content/docs/tr/reference/cli/providers-accounts.md b/docs-site/src/content/docs/tr/reference/cli/providers-accounts.md index df4f49ab949..8e8bef8b9e0 100644 --- a/docs-site/src/content/docs/tr/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/tr/reference/cli/providers-accounts.md @@ -126,7 +126,7 @@ alias Set or clear an account's display pause Hold an account out of automatic selection. resume Return a paused account to automatic selection. pause-exhausted Pause every account whose quota is spent. -clear-cooldown Drop a cooldown the proxy set after an upstream failure. +clear-cooldown Drop a cooldown the proxy set after an upstream failure. strategy [] Havuz stratejisi; least-loaded yalnızca Kiro içindir. sticky [<1-100>] Requests a bound thread keeps on one account; omit the value to read it. priority [first|earlier|normal|later|last|-100..100|reset] Selection order; omit the value to read it. @@ -243,6 +243,21 @@ CLI komutu Anthropic OAuth hesabını id veya benzersiz takma ad ile duraklatır Bir hesap id'si çözümlemeden Codex hesabının elle seçimini temizler; `auto` adında bir hesap olsa bile çalışır. Yalnızca Codex havuzları içindir; diğer sağlayıcı türlerinde geri yüklenecek otomatik seçim yoktur. +### `ocx account clear-cooldown [--json]` + +Kaydedilmiş kimlik bilgilerini değiştirmeden süreç içi hata cooldown durumunu kaldırır. Codex havuzu +hesabı için `openai`, Anthropic OAuth hesabı için `anthropic` kullanın; diğer sağlayıcılar reddedilir. +Her iki biçim de hesap id'sini veya benzersiz takma adı kabul eder, `main` ise yalnızca Codex havuzuna +özgüdür. + +```bash +ocx account clear-cooldown anthropic +``` + +Etkin cooldown olmasa da komut başarılı olur ve JSON'da `cleared: false` döner. Anthropic cooldown +temizliği hesap generation değerini de ilerletir; böylece eski bir quota probe temizlenen durumu geri +getiremez veya eski quota uygunluğunu yayımlayamaz. + ### `ocx account refresh [--json]` Codex havuzu için `ocx account refresh openai [--json]` kullanın. Hesap diff --git a/docs-site/src/content/docs/zh-cn/reference/cli/providers-accounts.md b/docs-site/src/content/docs/zh-cn/reference/cli/providers-accounts.md index 4c1cc0afeda..6f77503e6b3 100644 --- a/docs-site/src/content/docs/zh-cn/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/zh-cn/reference/cli/providers-accounts.md @@ -97,7 +97,7 @@ alias Set or clear an account's display pause Hold an account out of automatic selection. resume Return a paused account to automatic selection. pause-exhausted Pause every account whose quota is spent. -clear-cooldown Drop a cooldown the proxy set after an upstream failure. +clear-cooldown Drop a cooldown the proxy set after an upstream failure. strategy [] Pool placement strategy; omit the value to read it. sticky [<1-100>] Requests a bound thread keeps on one account; omit the value to read it. priority [first|earlier|normal|later|last|-100..100|reset] Selection order; omit the value to read it. @@ -181,6 +181,19 @@ CLI 命令通过 id 或唯一别名暂停或恢复 Anthropic OAuth 账户。别 在不解析账号 id 的情况下清除 Codex 账号的手动选择，因此即使存在名为 `auto` 的账号也有效。仅适用于 Codex Pool；其他提供商类型没有可恢复的自动选择。 +### `ocx account clear-cooldown [--json]` + +清除进程本地的失败冷却，但不更改已保存的凭据。Codex Pool 账号使用 `openai`，Anthropic +OAuth 账号使用 `anthropic`；其他 provider 会被拒绝。两种形式都接受账号 id 或唯一别名， +而 `main` 仅适用于 Codex Pool。 + +```bash +ocx account clear-cooldown anthropic +``` + +即使没有活动冷却，命令也会成功，JSON 中的 `cleared` 为 `false`。清除 Anthropic 冷却还会 +推进账号 generation，因此旧的 quota probe 无法恢复已清除的状态或发布过期的配额资格。 + ### `ocx account refresh [--json]` 对于 Codex 池，请使用 `ocx account refresh openai [--json]`。它会强制刷新账号配额， diff --git a/docs-site/src/content/docs/zh-tw/reference/cli/providers-accounts.md b/docs-site/src/content/docs/zh-tw/reference/cli/providers-accounts.md index 09ead299f34..e35f067cd8b 100644 --- a/docs-site/src/content/docs/zh-tw/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/zh-tw/reference/cli/providers-accounts.md @@ -76,7 +76,7 @@ priority [first|earlier|normal|later|last|-100..100|r pause 將帳號移出自動選擇。 resume 將暫停的帳號放回自動選擇。 pause-exhausted 暫停所有配額已用盡的帳號。 -clear-cooldown 清除上游失敗後設定的冷卻。 +clear-cooldown 清除上游失敗後設定的冷卻。 strategy [] 帳號池放置策略；省略取值即讀取目前值。 sticky [<1-100>] 已綁定執行緒在同一帳號上保留的請求數；省略取值即讀取目前值。 remove --yes 在存在檢查後移除已儲存的帳號或金鑰。 @@ -162,6 +162,19 @@ ocx account pause google-antigravity ocx account resume google-antigravity ``` +### `ocx account clear-cooldown [--json]` + +清除行程本地的失敗冷卻，但不變更已儲存的憑證。Codex 池帳號使用 `openai`，Anthropic +OAuth 帳號使用 `anthropic`；其他供應商會被拒絕。兩種形式都接受帳號 id 或唯一別名， +而 `main` 僅適用於 Codex 池。 + +```bash +ocx account clear-cooldown anthropic +``` + +即使沒有作用中的冷卻，命令也會成功，JSON 中的 `cleared` 為 `false`。清除 Anthropic +冷卻也會推進帳號 generation，因此舊的 quota probe 無法恢復已清除狀態或發布過期的配額資格。 + ### `ocx account refresh [--json]` 對於 Codex 池，請使用 `ocx account refresh openai [--json]`。它強制重新整理帳號配額並印出可用的週／月百分比與重置時間；缺失的配額資料被回報為未知，而非 0%。其 JSON 封裝為 `{ accounts: AccountRow[] }`，每個 Codex 列上有 `quota`。 diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 80ddcdc8820..794895ba674 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -109,6 +109,7 @@ "anthropic-agentrouter-language-framing.test.ts": "adapters/anthropic", "anthropic-baseurl-override.test.ts": "adapters/anthropic", "anthropic-compatible-stream.test.ts": "adapters/anthropic", + "anthropic-cooldown-recovery.test.ts": "adapters/anthropic", "anthropic-empty-content.test.ts": "adapters/anthropic", "anthropic-eof-tolerance.test.ts": "adapters/anthropic", "anthropic-error-body.test.ts": "adapters/anthropic", diff --git a/src/cli/account-extended.ts b/src/cli/account-extended.ts index bc61bfb3483..cf38d7b032f 100644 --- a/src/cli/account-extended.ts +++ b/src/cli/account-extended.ts @@ -630,16 +630,7 @@ export async function cmdImport(args: string[], deps: AccountDeps): Promise 0 || result.unsupportedCount > 0 ? 1 : 0; } -/** - * Lift a quota cooldown on a Codex account. - * - * This is the user-facing escape from the lockout described in - * `devlog/_plan/260726_cooldown_lockout_hardening`: injected routing makes the proxy the - * only model path for Codex Desktop, so a stuck cooldown reads as "the whole app is dead". - * - * Codex accounts only. API-key pools already reset their own 429 cooldowns through key - * management (`clearKeyCooldowns`), and OAuth providers have no equivalent state here. - */ +/** Lift a process-local cooldown through the management route that owns that pool. */ export async function cmdClearCooldown(args: string[], deps: AccountDeps): Promise { const wantsJson = flag(args, "--json"); const name = args.shift(); @@ -647,16 +638,34 @@ export async function cmdClearCooldown(args: string[], deps: AccountDeps): Promi if (!name || !requestedId || args.length) return usage(); const classified = configAndType(deps, name); if ("error" in classified) return usage(`Error: ${classified.error}`); - if (classified.type !== "codex") { - return usage(`Error: ${name} is not a Codex account pool; cooldown clearing applies to Codex accounts only`); + if (classified.type !== "codex" && !(classified.type === "oauth" && name === "anthropic")) { + return usage(`Error: ${name} has no operator-clearable account cooldown`); } const baseUrl = await resolveBaseUrl(deps); if (!baseUrl) return proxyUnreachable(); - const target = await resolveCodexAccountTarget(deps, baseUrl, requestedId); - if ("networkDown" in target) return proxyUnreachable(target.transportError); - if ("error" in target) return reportCodexAccountTargetError(target); - const id = target.id; - const response = await apiJson(deps, baseUrl, "POST", "/api/codex-auth/accounts/clear-cooldown", { id }); + let id: string; + let response: Awaited>; + if (classified.type === "codex") { + const target = await resolveCodexAccountTarget(deps, baseUrl, requestedId); + if ("networkDown" in target) return proxyUnreachable(target.transportError); + if ("error" in target) return reportCodexAccountTargetError(target); + id = target.id; + response = await apiJson(deps, baseUrl, "POST", "/api/codex-auth/accounts/clear-cooldown", { id }); + } else { + const list = await apiJson(deps, baseUrl, "GET", `/api/oauth/accounts?provider=${encodeURIComponent(name)}`); + if (list.status === 0) return proxyUnreachable(list.transportError); + if (list.status !== 200) return apiError(list.json, `failed to list ${name} OAuth accounts`, list.status); + const target = resolveGenericOAuthAccountTarget( + Array.isArray(list.json.accounts) ? list.json.accounts : [], + requestedId, + ); + if ("error" in target) return usage(`Error: ${target.error}`); + id = target.id; + response = await apiJson(deps, baseUrl, "POST", "/api/oauth/accounts/clear-cooldown", { + provider: name, + accountId: id, + }); + } if (response.status === 0) return proxyUnreachable(response.transportError); if (response.status !== 200) return apiError(response.json, `failed to clear cooldown for ${requestedId}`, response.status); const cleared = response.json?.cleared === true; @@ -776,7 +785,7 @@ export async function cmdPriority(args: string[], deps: AccountDeps): Promise typeof value === "object" && value !== null && typeof (value as { id?: unknown }).id === "string", ); @@ -806,7 +815,7 @@ export async function cmdPause(args: string[], deps: AccountDeps, paused: boolea const list = await apiJson(deps, baseUrl, "GET", `/api/oauth/accounts?provider=${encodeURIComponent(name)}`); if (list.status === 0) return proxyUnreachable(list.transportError); if (list.status !== 200) return apiError(list.json, `failed to list ${name} OAuth accounts`, list.status); - const target = resolveGenericOAuthPauseTarget(Array.isArray(list.json.accounts) ? list.json.accounts : [], requestedId); + const target = resolveGenericOAuthAccountTarget(Array.isArray(list.json.accounts) ? list.json.accounts : [], requestedId); if ("error" in target) return usage(`Error: ${target.error}`); const response = await apiJson(deps, baseUrl, "PUT", "/api/oauth/accounts/pause", { diff --git a/src/oauth/anthropic-routing.ts b/src/oauth/anthropic-routing.ts index 050d51a6097..a65e0adc30f 100644 --- a/src/oauth/anthropic-routing.ts +++ b/src/oauth/anthropic-routing.ts @@ -34,6 +34,12 @@ import type { OcxAccountPoolQuotaWindow, OcxAccountPoolRotationStrategy, OcxConf import { sweepExpiredOnWrite } from "../lib/state-store-sweeper"; import { retainedUtf8Bytes } from "../lib/admission"; import { routeCandidates, type AnthropicRouteDecision } from "./anthropic-model-routes"; +import type { ProviderQuota } from "../providers/quota-types"; +import { + anthropicCooldownGeneration, + clearAnthropicCooldownGenerations, + noteAnthropicCooldownMutation, +} from "../providers/quota/anthropic-cooldown-recovery"; import { subscribeAccountSelections, subscribeOAuthAccountPauseChanges, subscribeOAuthAccountRoutingPolicyChanges } from "../lib/account-selection-events"; import { effectiveAnthropicAccountThreshold } from "./anthropic-account-threshold"; @@ -78,10 +84,15 @@ export class AnthropicAccountCooldownError extends Error { * `retry-after` would report a drained five-hour window as request-rate throttling. */ type AnthropicCooldownSource = "retry-after" | "reset-derived" | "default"; +type AnthropicQuotaWindow = "five-hour" | "weekly"; interface AccountHealth { cooldownUntil: number; cooldownSource: AnthropicCooldownSource; + /** Windows whose rejected status established a reset-derived cooldown. */ + rejectedQuotaWindows?: AnthropicQuotaWindow[]; + /** Monotonic fence so an older quota probe cannot erase a newer refusal. */ + cooldownGeneration: number; } interface AffinityEntry { @@ -157,11 +168,16 @@ function parseRetryAfterMs(value: string | null | undefined, now: number): numbe } /** Only rejected windows constrain recovery; all must reopen, so take the latest reset. */ -function parseRateLimitResetMs(headers: AnthropicRateLimitHeaders | null | undefined, now: number): number | undefined { +function parseRateLimitReset( + headers: AnthropicRateLimitHeaders | null | undefined, + now: number, +): { delayMs: number; rejectedQuotaWindows: AnthropicQuotaWindow[] } | undefined { if (!headers) return undefined; let latest: number | undefined; - for (const window of ["5h", "7d"] as const) { + const rejectedQuotaWindows: AnthropicQuotaWindow[] = []; + for (const [window, quotaWindow] of [["5h", "five-hour"], ["7d", "weekly"]] as const) { if (headers.get(`anthropic-ratelimit-unified-${window}-status`)?.trim() !== "rejected") continue; + rejectedQuotaWindows.push(quotaWindow); const resetSeconds = Number(headers.get(`anthropic-ratelimit-unified-${window}-reset`)?.trim()); if (!Number.isFinite(resetSeconds) || resetSeconds <= 0) continue; const resetAt = resetSeconds * 1000; @@ -169,7 +185,55 @@ function parseRateLimitResetMs(headers: AnthropicRateLimitHeaders | null | undef if (latest === undefined || resetAt > latest) latest = resetAt; } if (latest === undefined) return undefined; - return latest - now; + return { delayMs: latest - now, rejectedQuotaWindows }; +} + +export type AnthropicCooldownRecoveryClaim = Readonly<{ + accountId: string; + cooldownGeneration: number; + claimedAt: number; +}>; + +export type AnthropicCooldownRecoverySettlement = "cleared" | "retained" | "superseded"; + +/** + * Capture the exact reset-derived refusal a fresh usage probe is allowed to recover. + * + * The generation fence matters because the usage request is asynchronous: a newer 429 may + * arrive while it is in flight, and an older answer must never erase that newer refusal. + */ +export function captureAnthropicCooldownRecovery( + accountId: string, + now = Date.now(), +): AnthropicCooldownRecoveryClaim | null { + const entry = upstreamHealth.get(accountId); + if (!entry || entry.cooldownUntil <= now || entry.cooldownSource !== "reset-derived") return null; + return { accountId, cooldownGeneration: entry.cooldownGeneration, claimedAt: now }; +} + +/** + * Clear only the claimed reset-derived cooldown when a fresh, complete usage result proves + * headroom in every window that upstream previously reported as rejected. + */ +export function settleAnthropicCooldownRecovery( + claim: AnthropicCooldownRecoveryClaim, + quota: ProviderQuota, +): AnthropicCooldownRecoverySettlement { + const entry = upstreamHealth.get(claim.accountId); + if (!entry || entry.cooldownSource !== "reset-derived" + || entry.cooldownGeneration !== claim.cooldownGeneration + || anthropicCooldownGeneration(claim.accountId) !== claim.cooldownGeneration + || !Number.isFinite(quota.updatedAt) || quota.updatedAt < claim.claimedAt) return "superseded"; + const windows = entry.rejectedQuotaWindows; + if (!windows?.length) return "retained"; + const recovered = windows.every(window => { + const percent = window === "five-hour" ? quota.fiveHourPercent : quota.weeklyPercent; + return typeof percent === "number" && Number.isFinite(percent) && percent >= 0 && percent < 100; + }); + if (!recovered) return "retained"; + upstreamHealth.delete(claim.accountId); + noteAnthropicCooldownMutation(claim.accountId); + return "cleared"; } export function getAnthropicAccountHealthSnapshot( @@ -180,13 +244,16 @@ export function getAnthropicAccountHealthSnapshot( if (!entry) return null; if (entry.cooldownUntil <= now) { upstreamHealth.delete(accountId); + noteAnthropicCooldownMutation(accountId); return null; } return { cooldownUntil: entry.cooldownUntil, cooldownSource: entry.cooldownSource }; } export function clearAnthropicAccountCooldown(accountId: string): boolean { - return upstreamHealth.delete(accountId); + const cleared = upstreamHealth.delete(accountId); + if (cleared) noteAnthropicCooldownMutation(accountId); + return cleared; } export function sweepExpiredAnthropicRoutingHealth(now = Date.now()): number { @@ -194,6 +261,7 @@ export function sweepExpiredAnthropicRoutingHealth(now = Date.now()): number { for (const [accountId, health] of upstreamHealth) { if (health.cooldownUntil > now) continue; upstreamHealth.delete(accountId); + noteAnthropicCooldownMutation(accountId); removed += 1; } return removed; @@ -202,6 +270,7 @@ export function sweepExpiredAnthropicRoutingHealth(now = Date.now()): number { /** Test / logout helper. */ export function clearAnthropicAccountPoolState(): void { upstreamHealth.clear(); + clearAnthropicCooldownGenerations(); sessionAffinity.clear(); manualPreference = undefined; quorumCache = null; @@ -854,13 +923,16 @@ export function recordAnthropicAccount429( // without that fallback such a refusal cools for the 60s default and the exhausted // account is back in the rotation a minute later. const parsedRetry = parseRetryAfterMs(retryAfterHeader, now); - const resetDerived = parsedRetry === undefined ? parseRateLimitResetMs(rateLimitHeaders, now) : undefined; - const cooldownMs = parsedRetry ?? resetDerived ?? DEFAULT_COOLDOWN_MS; + const resetDerived = parsedRetry === undefined ? parseRateLimitReset(rateLimitHeaders, now) : undefined; + const cooldownMs = parsedRetry ?? resetDerived?.delayMs ?? DEFAULT_COOLDOWN_MS; + const cooldownGeneration = noteAnthropicCooldownMutation(failedAccountId); upstreamHealth.set(failedAccountId, { cooldownUntil: now + cooldownMs, cooldownSource: parsedRetry !== undefined ? "retry-after" : resetDerived !== undefined ? "reset-derived" : "default", + ...(resetDerived ? { rejectedQuotaWindows: resetDerived.rejectedQuotaWindows } : {}), + cooldownGeneration, }); sweepExpiredOnWrite(now); clearAnthropicSessionAffinityForAccount(failedAccountId); diff --git a/src/providers/quota.ts b/src/providers/quota.ts index 07c8cf75a5d..df45b0e494b 100644 --- a/src/providers/quota.ts +++ b/src/providers/quota.ts @@ -1,6 +1,6 @@ import { listCodexAuthAccountsSnapshot } from "../codex/auth-api"; import { resolveEnvValue } from "../config"; -import { getAccountCredential, getAccountCredentialWithStatus, getAccountSet } from "../oauth/store"; +import { getAccountCredential, getAccountSet } from "../oauth/store"; import { apiKeyPoolEntryId } from "./api-keys"; import { captureConfigGeneration, sweepExpiredOnWrite } from "../lib/state-store-sweeper"; import { ACCOUNT_QUOTA_TTL_MS, CACHE_TTL_MS } from "./quota-wire"; @@ -68,7 +68,7 @@ import { fetchCommandCodeQuota, fetchKimiQuota, keyQuotaReaderForProvider } from import { antigravityQuotaDiagnosticIdentity, fetchAntigravityQuota, probeAntigravityUsageQuota } from "./quota/antigravity"; import { persistKiroAccountState } from "./kiro-account-state-disk"; import { kiroProbeCurrent, kiroProbeIdentity } from "./quota/kiro-account-probe"; - +import { AnthropicQuotaProbeOwnershipError, anthropicCooldownFlightKey, assertAnthropicQuotaSendAllowed, probeAnthropicQuotaWithRecovery } from "./quota/anthropic-cooldown-recovery"; export type { ProviderQuota, ProviderQuotaCreditsUsd, ProviderQuotaWindow } from "./quota-types"; export { QUOTA_RESPONSE_MAX_BYTES } from "./quota-wire"; export { @@ -98,7 +98,6 @@ export { export { fetchAntigravityUsageQuota, isCanonicalAntigravityQuotaUrl, setAntigravityAccountQuotaTransportForTests } from "./quota/antigravity"; export { parseOllamaCloudQuota, parseZaiQuotaLimits, providerApiKeyQuotaMode } from "./quota/vendor-probes-key"; export { parseXaiCreditsResponse } from "./quota/vendor-probes-oauth"; - export async function fetchProviderApiKeyQuotas(config: OcxConfig, name: string, forceRefresh = false): Promise { const provider = config.providers[name]; if (!provider || !keyQuotaReaderForProvider(name, provider)) return []; @@ -109,7 +108,6 @@ export async function fetchProviderApiKeyQuotas(config: OcxConfig, name: string, return result ? { kind: "quota", quota: result.quota } : { kind: "unavailable" }; }); } - async function maybeFetchProviderQuota( name: string, provider: OcxProviderConfig, @@ -338,7 +336,6 @@ export async function fetchProviderQuotaReports(config: OcxConfig, forceRefresh if (inflight.get(key) === entry) inflight.delete(key); } } - async function readExplicitAccountQuota(provider: string, accountId: string, configured?: OcxProviderConfig): Promise<{ result: ProviderQuotaProbeResult; identity: string | undefined; @@ -367,7 +364,6 @@ async function readExplicitAccountQuota(provider: string, accountId: string, con } return { result, identity, isCurrent }; } - async function fetchExplicitAccountQuota(provider: string, accountId: string, force: boolean, configured?: OcxProviderConfig): Promise { const key = accountCacheKey(provider, accountId); const identity = explicitQuotaIdentity(provider, accountId, configured); @@ -427,14 +423,19 @@ async function fetchAccountQuota( const writerGeneration = captureConfigGeneration(); const kiroIdentity = provider === "kiro" ? kiroProbeIdentity(accountId) : undefined; const cachedCandidate = accountQuotaCache.get(key); - const cached = provider !== "kiro" || cachedCandidate?.identity === kiroIdentity ? cachedCandidate : undefined; + const cached = (provider !== "kiro" || cachedCandidate?.identity === kiroIdentity) + && (provider !== "anthropic" || cachedCandidate?.isCurrent?.() !== false) ? cachedCandidate : undefined; if (!forceRefresh && cached && Date.now() - cached.ts < ACCOUNT_QUOTA_TTL_MS) { if (provider === "google-antigravity" && cached.quotaFailure && cached.quotaFailureIsCurrent?.() !== true) return { ...cached, quotaFailure: undefined }; return provider === "anthropic" ? { ...cached, quota: normalizeAnthropicQuota(cached.quota, Date.now()) } : cached; } - const joinable = accountQuotaInflight.get(key); + const flightKey = provider === "anthropic" ? anthropicCooldownFlightKey(key, accountId) : key; + const flightCurrent = () => provider !== "anthropic" || anthropicCooldownFlightKey(key, accountId) === flightKey; + const joinable = accountQuotaInflight.get(flightKey); if (joinable) { const joined = await joinable; + const joinedCurrent = joined.isCurrent?.(); + if (joinedCurrent === false || (!flightCurrent() && joinedCurrent !== true)) return { ts: Date.now(), quota: null, unavailable: true }; return provider !== "kiro" || joined.identity === kiroIdentity ? joined : fetchAccountQuota(provider, accountId, true, providerConfig); } @@ -447,15 +448,36 @@ async function fetchAccountQuota( catch { return false; } }; const diagnosticFields = () => quotaFailure && quotaFailureIsCurrent() ? { quotaFailure, quotaFailureIsCurrent } : {}; + const unavailable = (): AccountQuotaCacheEntry => { + if (!flightCurrent()) return { ts: Date.now(), quota: null, unavailable: true }; + const previous = accountQuotaCache.get(key); + const retained = provider === "anthropic" && previous?.isCurrent?.() === false ? undefined : previous; + const entry: AccountQuotaCacheEntry = { + ts: Date.now(), + quota: provider === "anthropic" ? normalizeAnthropicQuota(retained?.quota, Date.now()) : cached?.quota ?? null, + unavailable: true, + ...(provider === "anthropic" && retained?.isCurrent ? { isCurrent: retained.isCurrent } : {}), + ...(provider === "kiro" ? { identity: kiroIdentity } : {}), + ...diagnosticFields(), + }; + if (mayCommitAccountQuotaKey(key, writerGeneration) && (provider !== "kiro" || kiroProbeCurrent(accountId, kiroIdentity))) { + accountQuotaCache.set(key, entry); + if (provider === "kiro") persistKiroAccountState(); + sweepExpiredOnWrite(entry.ts); + } + return entry; + }; try { if (provider === "google-antigravity") diagnosticIdentity = antigravityQuotaDiagnosticIdentity(accountId); let quota: ProviderQuota | null; + let anthropicCurrent: (() => boolean) | undefined; let kiroSnapshot: KiroUsageSnapshot | null = null; if (provider === "kiro") { kiroSnapshot = await fetchKiroUsageSnapshot(await kiroUsageContextForAccount(accountId)); quota = kiroSnapshot?.quota ?? null; } else { const token = await getTokenForAccountQuotaProbe(provider, accountId); + if (!flightCurrent()) throw new AnthropicQuotaProbeOwnershipError("anthropic quota flight is stale"); if (provider === "google-antigravity") { // Per-account Gem/Cla windows (#1082). The project id is part of the stored // credential; without it the probe cannot be made, and that is "unavailable", @@ -467,61 +489,38 @@ async function fetchAccountQuota( quota = result.kind === "available" ? result.quota : null; if (result.kind === "unavailable") quotaFailure = result.failure; } else if (provider === "anthropic") { - const row = getAccountCredentialWithStatus(provider, accountId); - if (!row || row.paused || row.needsReauth || row.credential.access !== token) return { ts: Date.now(), quota: null, unavailable: true }; - quota = await fetchAnthropicUsageQuota(token); + const result = await probeAnthropicQuotaWithRecovery(accountId, token, + fresh => { assertAnthropicQuotaSendAllowed(accountId, token); return fetchAnthropicUsageQuota(token, fresh); }, + () => mayCommitAccountQuotaKey(key, writerGeneration)); + if (result && !result.isCurrent()) throw new AnthropicQuotaProbeOwnershipError("anthropic quota probe lost publication ownership"); + quota = result?.quota ?? null; + anthropicCurrent = result?.isCurrent; } else { return { ts: Date.now(), quota: null, unavailable: true }; } } - if (!quota) { - // Preserve last-good bars while negative-caching the failed probe. - const entry: AccountQuotaCacheEntry = { - ts: Date.now(), - quota: provider === "anthropic" - ? normalizeAnthropicQuota(accountQuotaCache.get(key)?.quota, Date.now()) : cached?.quota ?? null, - unavailable: true, - ...(provider === "kiro" ? { identity: kiroIdentity } : {}), - ...diagnosticFields(), - }; - if (mayCommitAccountQuotaKey(key, writerGeneration) && (provider !== "kiro" || kiroProbeCurrent(accountId, kiroIdentity))) { - accountQuotaCache.set(key, entry); - if (provider === "kiro") persistKiroAccountState(); - sweepExpiredOnWrite(entry.ts); - } - return entry; - } + if (!quota) return unavailable(); const entry: AccountQuotaCacheEntry = { ts: Date.now(), quota: provider === "anthropic" ? normalizeAnthropicQuota(quota, Date.now()) : quota, ...(provider === "kiro" ? { identity: kiroIdentity } : {}), + ...(anthropicCurrent ? { isCurrent: anthropicCurrent } : {}), }; - if (mayCommitAccountQuotaKey(key, writerGeneration) && (provider !== "kiro" || kiroProbeCurrent(accountId, kiroIdentity))) { + if (mayCommitAccountQuotaKey(key, writerGeneration) + && (provider !== "kiro" || kiroProbeCurrent(accountId, kiroIdentity))) { accountQuotaCache.set(key, entry); if (provider === "kiro") { commitKiroAccountUsageState(key, kiroSnapshot, kiroIdentity); persistKiroAccountState(); } sweepExpiredOnWrite(entry.ts); } return entry; - } catch { + } catch (error) { + if (provider === "anthropic" && error instanceof AnthropicQuotaProbeOwnershipError) return { ts: Date.now(), quota: null, unavailable: true }; if (provider === "google-antigravity") quotaFailure = "account_unavailable"; - const entry: AccountQuotaCacheEntry = { - ts: Date.now(), - quota: provider === "anthropic" - ? normalizeAnthropicQuota(accountQuotaCache.get(key)?.quota, Date.now()) : cached?.quota ?? null, - unavailable: true, - ...(provider === "kiro" ? { identity: kiroIdentity } : {}), - ...diagnosticFields(), - }; - if (mayCommitAccountQuotaKey(key, writerGeneration) && (provider !== "kiro" || kiroProbeCurrent(accountId, kiroIdentity))) { - accountQuotaCache.set(key, entry); - if (provider === "kiro") persistKiroAccountState(); - sweepExpiredOnWrite(entry.ts); - } - return entry; + return unavailable(); } })().finally(() => { - if (accountQuotaInflight.get(key) === probe) accountQuotaInflight.delete(key); + if (accountQuotaInflight.get(flightKey) === probe) accountQuotaInflight.delete(flightKey); }); - accountQuotaInflight.set(key, probe); + accountQuotaInflight.set(flightKey, probe); return probe; } /** @@ -545,6 +544,7 @@ export async function fetchProviderAccountQuotas( ...(entry.unavailable && entry.quotaFailure && entry.quotaFailureIsCurrent?.() === true ? { quotaFailure: entry.quotaFailure } : {}), }; if (entry.quotaFailureIsCurrent) Object.defineProperty(result, "quotaFailureIsCurrent", { value: entry.quotaFailureIsCurrent }); + if (provider === "anthropic" && entry.isCurrent) Object.defineProperty(result, "isCurrent", { value: entry.isCurrent }); if (!explicitAccountReader(provider)) return result; const identity = entry.identity; Object.defineProperty(result, "isCurrent", { value: () => { diff --git a/src/providers/quota/account-cache.ts b/src/providers/quota/account-cache.ts index eaa1538567a..841d7c27365 100644 --- a/src/providers/quota/account-cache.ts +++ b/src/providers/quota/account-cache.ts @@ -272,7 +272,8 @@ export function recordAnthropicAccountQuotaFromHeaders( // whole map. Landing before any reader has hydrated would persist this single row and erase // every other provider's saved row. hydrateAccountQuotaCache(); - const previous = accountQuotaCache.get(key); + const candidate = accountQuotaCache.get(key); + const previous = candidate?.isCurrent?.() === false ? undefined : candidate; accountQuotaCache.set(key, { ...previous, // Headers do not prove that the last usage probe succeeded. diff --git a/src/providers/quota/anthropic-cooldown-recovery.ts b/src/providers/quota/anthropic-cooldown-recovery.ts new file mode 100644 index 00000000000..867df5318f8 --- /dev/null +++ b/src/providers/quota/anthropic-cooldown-recovery.ts @@ -0,0 +1,118 @@ +import { credentialGeneration, getAccountCredential, getAccountCredentialWithStatus } from "../../oauth/store"; +import type { ProviderQuota } from "../quota-types"; + +export class AnthropicQuotaProbeOwnershipError extends Error {} + +/** Check live pause and bearer ownership at the last synchronous step before usage dispatch. */ +export function assertAnthropicQuotaSendAllowed(accountId: string, token: string): void { + const row = getAccountCredentialWithStatus("anthropic", accountId); + if (!row || row.paused || row.needsReauth || row.credential.access !== token) { + throw new AnthropicQuotaProbeOwnershipError("anthropic quota account is no longer eligible"); + } +} + +const cooldownGenerationByAccount = new Map(); +let nextCooldownGeneration = 1; +let afterSettlementForTests: (() => void) | undefined; + +export function setAnthropicQuotaAfterSettlementForTests(hook: (() => void) | undefined): void { + afterSettlementForTests = hook; +} + +/** Monotonic per-account fence shared by routing mutations and quota publication. */ +export function anthropicCooldownGeneration(accountId: string): number { + return cooldownGenerationByAccount.get(accountId) ?? 0; +} + +/** Keep same-state quota calls joinable while moving post-429 calls to a fresh flight. */ +export function anthropicCooldownFlightKey(baseKey: string, accountId: string): string { + return `${baseKey}\0anthropic-cooldown:${anthropicCooldownGeneration(accountId)}`; +} + +export function noteAnthropicCooldownMutation(accountId: string): number { + const generation = nextCooldownGeneration++; + cooldownGenerationByAccount.set(accountId, generation); + return generation; +} + +export function clearAnthropicCooldownGenerations(): void { + cooldownGenerationByAccount.clear(); +} + +export type AnthropicCooldownRecoveryProbe = Readonly<{ + requiresFreshDispatch: boolean; + isCurrentCredential(): boolean; + isCurrent(): boolean; + settle(quota: ProviderQuota): "cleared" | "retained" | "superseded"; +}>; + +export type AnthropicQuotaRecoveryResult = Readonly<{ + quota: ProviderQuota; + /** Must be checked synchronously at each publication boundary. */ + isCurrent(): boolean; +}>; + +/** + * Bind a quota probe to both the credential and cooldown generations it observed before + * dispatch. A successful response may otherwise arrive after either credential replacement + * or a newer 429 and incorrectly make unrelated state eligible. + */ +export async function captureAnthropicCooldownRecoveryProbe( + accountId: string, + accessToken: string, +): Promise { + const credential = getAccountCredential("anthropic", accountId); + if (!credential || credential.access !== accessToken) return null; + const generation = credentialGeneration(credential); + // Lazy because anthropic-routing reads the quota cache on normal request routing. + const routing = await import("../../oauth/anthropic-routing"); + const cooldownGeneration = anthropicCooldownGeneration(accountId); + const claim = routing.captureAnthropicCooldownRecovery(accountId); + const isCurrentCredential = () => { + const current = getAccountCredential("anthropic", accountId); + return !!current && credentialGeneration(current) === generation; + }; + return { + requiresFreshDispatch: claim !== null, + isCurrentCredential, + isCurrent: () => isCurrentCredential() + && anthropicCooldownGeneration(accountId) === cooldownGeneration, + settle: quota => claim === null ? "retained" + : routing.settleAnthropicCooldownRecovery(claim, quota), + }; +} + +/** Run one authoritative usage read and publish its recovery effect only while still owned. */ +export async function probeAnthropicQuotaWithRecovery( + accountId: string, + accessToken: string, + read: (requireFreshDispatch: boolean) => Promise, + mayPublish: () => boolean, +): Promise { + const probe = await captureAnthropicCooldownRecoveryProbe(accountId, accessToken); + if (!probe) throw new AnthropicQuotaProbeOwnershipError("anthropic quota probe lost credential ownership"); + let quota: ProviderQuota | null; + try { quota = await read(probe.requiresFreshDispatch); } + catch (error) { + if (!probe.isCurrent() || !mayPublish()) throw new AnthropicQuotaProbeOwnershipError("anthropic quota probe failure is stale"); + throw error; + } + if (!probe.isCurrent() || !mayPublish()) { + throw new AnthropicQuotaProbeOwnershipError("anthropic quota probe result is stale"); + } + if (!quota) return null; + const settlement = probe.settle(quota); + if (settlement === "superseded") { + throw new AnthropicQuotaProbeOwnershipError("anthropic quota probe lost cooldown ownership"); + } + // Clearing the claimed cooldown intentionally advances the fence. Adopt that exact new + // generation; any later observation/429 then invalidates publication before a cache write. + const publicationGeneration = anthropicCooldownGeneration(accountId); + afterSettlementForTests?.(); + return { + quota, + isCurrent: () => probe.isCurrentCredential() + && anthropicCooldownGeneration(accountId) === publicationGeneration + && mayPublish(), + }; +} diff --git a/src/providers/quota/vendor-probes-oauth.ts b/src/providers/quota/vendor-probes-oauth.ts index c4c4d006be1..14e0c3dd254 100644 --- a/src/providers/quota/vendor-probes-oauth.ts +++ b/src/providers/quota/vendor-probes-oauth.ts @@ -1,7 +1,7 @@ import { effectiveCodexAuthAccountId, fetchMainAccountInfoSnapshot, listCodexAuthAccountsSnapshot } from "../../codex/auth-api"; import { MAIN_CODEX_ACCOUNT_ID } from "../../codex/main-account"; import { getValidAccessToken } from "../../oauth"; -import { captureOAuthAccountSelection, getAccountCredential, getAccountCredentialWithStatus, getAccountSet } from "../../oauth/store"; +import { captureOAuthAccountSelection, getAccountCredential, getAccountSet } from "../../oauth/store"; import { hydrateKiroAccountState, persistKiroAccountState } from "../kiro-account-state-disk"; import { kiroProbeCurrent, kiroProbeIdentity } from "./kiro-account-probe"; import { fetchMuseKeyQuotaSnapshot } from "../muse-key-quota"; @@ -39,6 +39,7 @@ import { } from "./account-cache"; import type { OcxConfig, OcxProviderConfig } from "../../types"; import type { ProviderQuota, ProviderQuotaWindow } from "../quota-types"; +import { AnthropicQuotaProbeOwnershipError, assertAnthropicQuotaSendAllowed, probeAnthropicQuotaWithRecovery } from "./anthropic-cooldown-recovery"; const XAI_BILLING_URL = "https://cli-chat-proxy.grok.com/v1/billing"; const XAI_CREDITS_URL = `${XAI_BILLING_URL}?format=credits`; @@ -263,6 +264,56 @@ function parseClaudeLimit(value: unknown): ProviderQuotaWindow | null { /** Claude's OAuth usage endpoint, probed with ONE account's own bearer token. */ const anthropicUsageInflight = new Map>(); +async function readAnthropicUsageQuota(accessToken: string): Promise { + const response = await fetch("https://api.anthropic.com/api/oauth/usage", { + headers: { + Accept: "application/json, text/plain, */*", + "Content-Type": "application/json", + "User-Agent": CLAUDE_CLI_USER_AGENT, + "anthropic-beta": "claude-code-20250219,oauth-2025-04-20,interleaved-thinking-2025-05-14,context-management-2025-06-27,prompt-caching-scope-2026-01-05", + Authorization: `Bearer ${accessToken}`, + }, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + if (!response.ok) return null; + const body = asRecord(await readQuotaJson(response)); + if (!body) return null; + const fiveHour = parseClaudeBucket(body.five_hour); + const sevenDay = parseClaudeBucket(body.seven_day); + const fable = parseClaudeBucket(body.seven_day_fable); + const opus = parseClaudeBucket(body.seven_day_opus); + const sonnet = parseClaudeBucket(body.seven_day_sonnet); + const customWindows: ProviderQuotaWindow[] = []; + if (fable?.percent !== undefined) customWindows.push({ label: "Fable", scope: "model", percent: fable.percent, ...(fable.resetAt !== undefined ? { resetAt: fable.resetAt } : {}) }); + if (opus?.percent !== undefined) customWindows.push({ label: "Opus", scope: "model", percent: opus.percent, ...(opus.resetAt !== undefined ? { resetAt: opus.resetAt } : {}) }); + if (sonnet?.percent !== undefined) customWindows.push({ label: "Sonnet", scope: "model", percent: sonnet.percent, ...(sonnet.resetAt !== undefined ? { resetAt: sonnet.resetAt } : {}) }); + const knownLabels = new Set(customWindows.map(window => window.label.toLowerCase())); + const limits = Array.isArray(body.limits) ? body.limits : []; + for (const rawLimit of limits) { + const limitRecord = asRecord(rawLimit); + // `session` and `weekly_all` mirror the canonical five-hour and weekly + // buckets above; only model-scoped weekly limits add a third window. + if (String(limitRecord?.kind ?? "").trim().toLowerCase() !== "weekly_scoped") continue; + const limit = parseClaudeLimit(rawLimit); + if (!limit || knownLabels.has(limit.label.toLowerCase())) continue; + knownLabels.add(limit.label.toLowerCase()); + customWindows.push(limit); + } + const quota: ProviderQuota = { + // Claude's 5-hour window is a first-class rate limit, same as the Codex login 5h/weekly + // rows: report it in the canonical fields so the dashboard renders it with the standard + // "5-hour limit" label and ordering instead of as a generic extra window. + ...(fiveHour?.percent !== undefined ? { fiveHourPercent: fiveHour.percent } : {}), + ...(fiveHour?.resetAt !== undefined ? { fiveHourResetAt: fiveHour.resetAt } : {}), + ...(sevenDay?.percent !== undefined ? { weeklyPercent: sevenDay.percent } : {}), + ...(sevenDay?.resetAt !== undefined ? { weeklyResetAt: sevenDay.resetAt } : {}), + ...(customWindows.length > 0 ? { customWindows } : {}), + updatedAt: Date.now(), + }; + // Empty / schema-changed payloads must not cache as "success with no bars". + return hasQuotaRows(quota) ? quota : null; +} + /** * Anthropic per-credential usage. * @@ -278,59 +329,16 @@ const anthropicUsageInflight = new Map>(); * model-scoped window tracks entitlement rather than seat size. Populate `plan` only when * upstream returns the tier itself. */ -export async function fetchAnthropicUsageQuota(accessToken: string): Promise { - const joinable = anthropicUsageInflight.get(accessToken); +export async function fetchAnthropicUsageQuota( + accessToken: string, + requireFreshDispatch = false, +): Promise { + // Recovery evidence must be requested after the claimed cooldown. Joining an older request + // can return after the 429 while still describing provider state from before that refusal. + const joinable = requireFreshDispatch ? undefined : anthropicUsageInflight.get(accessToken); if (joinable) return joinable; - const probe = (async (): Promise => { - const response = await fetch("https://api.anthropic.com/api/oauth/usage", { - headers: { - Accept: "application/json, text/plain, */*", - "Content-Type": "application/json", - "User-Agent": CLAUDE_CLI_USER_AGENT, - "anthropic-beta": "claude-code-20250219,oauth-2025-04-20,interleaved-thinking-2025-05-14,context-management-2025-06-27,prompt-caching-scope-2026-01-05", - Authorization: `Bearer ${accessToken}`, - }, - signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), - }); - if (!response.ok) return null; - const body = asRecord(await readQuotaJson(response)); - if (!body) return null; - const fiveHour = parseClaudeBucket(body.five_hour); - const sevenDay = parseClaudeBucket(body.seven_day); - const fable = parseClaudeBucket(body.seven_day_fable); - const opus = parseClaudeBucket(body.seven_day_opus); - const sonnet = parseClaudeBucket(body.seven_day_sonnet); - const customWindows: ProviderQuotaWindow[] = []; - if (fable?.percent !== undefined) customWindows.push({ label: "Fable", scope: "model", percent: fable.percent, ...(fable.resetAt !== undefined ? { resetAt: fable.resetAt } : {}) }); - if (opus?.percent !== undefined) customWindows.push({ label: "Opus", scope: "model", percent: opus.percent, ...(opus.resetAt !== undefined ? { resetAt: opus.resetAt } : {}) }); - if (sonnet?.percent !== undefined) customWindows.push({ label: "Sonnet", scope: "model", percent: sonnet.percent, ...(sonnet.resetAt !== undefined ? { resetAt: sonnet.resetAt } : {}) }); - const knownLabels = new Set(customWindows.map(window => window.label.toLowerCase())); - const limits = Array.isArray(body.limits) ? body.limits : []; - for (const rawLimit of limits) { - const limitRecord = asRecord(rawLimit); - // `session` and `weekly_all` mirror the canonical five-hour and weekly - // buckets above; only model-scoped weekly limits add a third window. - if (String(limitRecord?.kind ?? "").trim().toLowerCase() !== "weekly_scoped") continue; - const limit = parseClaudeLimit(rawLimit); - if (!limit || knownLabels.has(limit.label.toLowerCase())) continue; - knownLabels.add(limit.label.toLowerCase()); - customWindows.push(limit); - } - const quota: ProviderQuota = { - // Claude's 5-hour window is a first-class rate limit, same as the Codex login 5h/weekly - // rows: report it in the canonical fields so the dashboard renders it with the standard - // "5-hour limit" label and ordering instead of as a generic extra window. - ...(fiveHour?.percent !== undefined ? { fiveHourPercent: fiveHour.percent } : {}), - ...(fiveHour?.resetAt !== undefined ? { fiveHourResetAt: fiveHour.resetAt } : {}), - ...(sevenDay?.percent !== undefined ? { weeklyPercent: sevenDay.percent } : {}), - ...(sevenDay?.resetAt !== undefined ? { weeklyResetAt: sevenDay.resetAt } : {}), - ...(customWindows.length > 0 ? { customWindows } : {}), - updatedAt: Date.now(), - }; - // Empty / schema-changed payloads must not cache as "success with no bars". - return hasQuotaRows(quota) ? quota : null; - })().finally(() => { + const probe = readAnthropicUsageQuota(accessToken).finally(() => { if (anthropicUsageInflight.get(accessToken) === probe) anthropicUsageInflight.delete(accessToken); }); anthropicUsageInflight.set(accessToken, probe); @@ -350,25 +358,34 @@ export async function fetchAnthropicQuota(provider: string): Promise boolean) | undefined; + try { + const result = await probeAnthropicQuotaWithRecovery(probedAccountId, accessToken, + fresh => { assertAnthropicQuotaSendAllowed(probedAccountId, accessToken); return fetchAnthropicUsageQuota(accessToken, fresh); }, + () => mayCommitAccountQuotaKey(probedAccountKey, writerGeneration)); + if (result && !result.isCurrent()) return null; + quota = result?.quota ?? null; + anthropicCurrent = result?.isCurrent; + } catch (error) { + if (error instanceof AnthropicQuotaProbeOwnershipError) return null; + throw error; + } if (!quota) return null; // Share the active-account probe with the per-account cache so Providers-page // loads do not double-hit Anthropic's rate-limited usage endpoint. if (probedAccountId && probedAccountKey) { const stillOwnsToken = getAccountCredential("anthropic", probedAccountId)?.access === accessToken; - if (stillOwnsToken && mayCommitAccountQuotaKey(probedAccountKey, writerGeneration)) { - accountQuotaCache.set(probedAccountKey, { ts: Date.now(), quota }); + if (stillOwnsToken && anthropicCurrent?.() && mayCommitAccountQuotaKey(probedAccountKey, writerGeneration)) { + accountQuotaCache.set(probedAccountKey, { ts: Date.now(), quota, isCurrent: anthropicCurrent }); } } - const result = report(provider, "anthropic:oauth-usage", quota); - if (result && probedAccountId) { - accountReportCurrent.set(result, () => getAccountSet("anthropic")?.activeAccountId === probedAccountId); + const quotaReport = report(provider, "anthropic:oauth-usage", quota); + if (quotaReport && anthropicCurrent) { + accountReportCurrent.set(quotaReport, () => anthropicCurrent() && getAccountSet("anthropic")?.activeAccountId === probedAccountId); } - return result; + return quotaReport; } /** diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index 92b7b28f29b..65443f86644 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -76,6 +76,11 @@ OrcaRouter key exchange uses the shared raw-byte reader before returning a durab 64 KiB response ceiling, single 30-second header/body deadline, and cancellation behavior follow the [bounded ingestion contract](transports/inventory.md#bounded-response-ingestion-and-orcarouter-login). +Anthropic model-scoped quota labels in `src/providers/quota/vendor-probes-oauth.ts` publish +only canonical Fable, Opus, or Sonnet labels after removing terminal controls; unknown upstream display names are omitted. +Anthropic usage flights replace older joinable transports when recovery requires a fresh read. `src/providers/quota/anthropic-cooldown-recovery.ts` fences successful, empty, and rejected results by credential and cooldown generation before cache publication. Live account quota entries retain that currentness predicate; routing and account-list readers reject a superseded entry before its TTL expires. Persisted and header-only observations carry no live probe predicate of their own. +Per-account quota flights also retain their starting cooldown generation through token resolution. A stale token failure returns unavailable to its caller without replacing the cache row or its timestamp; a joined flight rechecks ownership before returning. + MiniMax and MiniMax CN Coding Plan quota in `src/providers/quota/vendor-probes-key.ts` uses the region-matched `/v1/api/openplatform/coding_plan/remains` endpoint. It publishes the `general` model's consumed 5-hour percentage and, when active, weekly percentage with their reset times; diff --git a/tests/adapters/anthropic/anthropic-cooldown-recovery.test.ts b/tests/adapters/anthropic/anthropic-cooldown-recovery.test.ts new file mode 100644 index 00000000000..3b7b0d62227 --- /dev/null +++ b/tests/adapters/anthropic/anthropic-cooldown-recovery.test.ts @@ -0,0 +1,365 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + clearAnthropicAccountPoolState, + getAnthropicAccountHealthSnapshot, + rotateAnthropicAccountOn429, +} from "../../../src/oauth/anthropic-routing"; +import { getAccountSet, saveAccountCredential, saveCredential } from "../../../src/oauth/store"; +import { clearAccountQuotaCache, fetchProviderAccountQuotas, getCachedProviderAccountQuota } from "../../../src/providers/quota"; +import { fetchAnthropicUsageQuota } from "../../../src/providers/quota/vendor-probes-oauth"; +import { setAnthropicQuotaAfterSettlementForTests } from "../../../src/providers/quota/anthropic-cooldown-recovery"; +import { accountCacheKey, accountQuotaCache } from "../../../src/providers/quota/account-cache"; +import type { OcxConfig } from "../../../src/types"; +import { removeTreeWithRetry } from "../../helpers/remove-tree"; + +const originalFetch = globalThis.fetch; +const originalHome = process.env.OPENCODEX_HOME; +let home = ""; + +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), "ocx-anthropic-cooldown-recovery-")); + process.env.OPENCODEX_HOME = home; + clearAnthropicAccountPoolState(); + clearAccountQuotaCache(); + setAnthropicQuotaAfterSettlementForTests(undefined); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + clearAnthropicAccountPoolState(); + clearAccountQuotaCache(); + setAnthropicQuotaAfterSettlementForTests(undefined); + if (originalHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = originalHome; + removeTreeWithRetry(home); +}); + +const config = { + port: 0, + defaultProvider: "anthropic", + providers: { anthropic: { adapter: "anthropic", authMode: "oauth", baseUrl: "https://api.anthropic.com" } }, + anthropicAccountPool: { enabled: true }, +} as OcxConfig; + +async function seed(): Promise { + await saveCredential("anthropic", { + access: "access-old", + refresh: "refresh-old", + expires: Date.now() + 60 * 60_000, + accountId: "upstream-account", + email: "recovery@example.test", + }); + return getAccountSet("anthropic")!.activeAccountId; +} + +function rejected(resetAt: number, windows: Array<"5h" | "7d"> = ["5h"]): Headers { + const headers = new Headers(); + for (const window of windows) { + headers.set(`anthropic-ratelimit-unified-${window}-status`, "rejected"); + headers.set(`anthropic-ratelimit-unified-${window}-reset`, String(Math.floor(resetAt / 1_000))); + } + return headers; +} + +function quotaResponse(fiveHour?: number, weekly?: number): Response { + const reset = new Date(Date.now() + 60 * 60_000).toISOString(); + return Response.json({ + ...(fiveHour === undefined ? {} : { five_hour: { utilization: fiveHour, resets_at: reset } }), + ...(weekly === undefined ? {} : { seven_day: { utilization: weekly, resets_at: reset } }), + }); +} + +describe("Anthropic reset-derived cooldown recovery", () => { + test("a fresh credential-matched quota report clears the exact recovered cooldown", async () => { + const id = await seed(); + rotateAnthropicAccountOn429(config, id, null, null, Date.now(), rejected(Date.now() + 60 * 60_000)); + globalThis.fetch = (async () => quotaResponse(20, 30)) as typeof fetch; + + const [row] = await fetchProviderAccountQuotas("anthropic", true); + expect(row?.quota).toMatchObject({ fiveHourPercent: 20, weeklyPercent: 30 }); + expect(getAnthropicAccountHealthSnapshot(id)).toBeNull(); + }); + + test("partial, exhausted, and failed probes retain a reset-derived cooldown", async () => { + const id = await seed(); + const resetAt = Date.now() + 2 * 60 * 60_000; + rotateAnthropicAccountOn429(config, id, null, null, Date.now(), rejected(resetAt, ["5h", "7d"])); + + globalThis.fetch = (async () => quotaResponse(20)) as typeof fetch; + await fetchProviderAccountQuotas("anthropic", true); + expect(getAnthropicAccountHealthSnapshot(id)?.cooldownUntil).toBe(Math.floor(resetAt / 1_000) * 1_000); + + globalThis.fetch = (async () => quotaResponse(20, 100)) as typeof fetch; + await fetchProviderAccountQuotas("anthropic", true); + expect(getAnthropicAccountHealthSnapshot(id)?.cooldownSource).toBe("reset-derived"); + + globalThis.fetch = (async () => new Response("busy", { status: 503 })) as typeof fetch; + const [failed] = await fetchProviderAccountQuotas("anthropic", true); + expect(failed?.unavailable).toBe(true); + expect(getAnthropicAccountHealthSnapshot(id)?.cooldownSource).toBe("reset-derived"); + + globalThis.fetch = (async () => quotaResponse(20, 30)) as typeof fetch; + await fetchProviderAccountQuotas("anthropic", true); + expect(getAnthropicAccountHealthSnapshot(id)).toBeNull(); + }); + + test("successful quota reports do not clear Retry-After or guessed cooldowns", async () => { + const id = await seed(); + globalThis.fetch = (async () => quotaResponse(0, 0)) as typeof fetch; + + rotateAnthropicAccountOn429(config, id, "3600", null, Date.now()); + await fetchProviderAccountQuotas("anthropic", true); + expect(getAnthropicAccountHealthSnapshot(id)?.cooldownSource).toBe("retry-after"); + + clearAnthropicAccountPoolState(); + rotateAnthropicAccountOn429(config, id, null, null, Date.now(), new Headers()); + await fetchProviderAccountQuotas("anthropic", true); + expect(getAnthropicAccountHealthSnapshot(id)?.cooldownSource).toBe("default"); + }); + + test("credential replacement while the probe is in flight retains the cooldown", async () => { + const id = await seed(); + rotateAnthropicAccountOn429(config, id, null, null, Date.now(), rejected(Date.now() + 60 * 60_000)); + let started!: () => void; + const dispatched = new Promise(resolve => { started = resolve; }); + let finish!: (response: Response) => void; + const response = new Promise(resolve => { finish = resolve; }); + globalThis.fetch = (async () => { started(); return response; }) as typeof fetch; + + const pending = fetchProviderAccountQuotas("anthropic", true); + await dispatched; + await saveCredential("anthropic", { + access: "access-new", + refresh: "refresh-new", + expires: Date.now() + 60 * 60_000, + accountId: "upstream-account", + email: "recovery@example.test", + }); + finish(quotaResponse(0, 0)); + const [row] = await pending; + expect(row?.unavailable).toBe(true); + expect(getCachedProviderAccountQuota("anthropic", id)).toBeNull(); + expect(getAnthropicAccountHealthSnapshot(id)?.cooldownSource).toBe("reset-derived"); + }); + + test("an older quota probe cannot erase a newer 429 generation", async () => { + const id = await seed(); + rotateAnthropicAccountOn429(config, id, null, null, Date.now(), rejected(Date.now() + 60 * 60_000)); + let started!: () => void; + const dispatched = new Promise(resolve => { started = resolve; }); + let finish!: (response: Response) => void; + const response = new Promise(resolve => { finish = resolve; }); + globalThis.fetch = (async () => { started(); return response; }) as typeof fetch; + + const pending = fetchProviderAccountQuotas("anthropic", true); + await dispatched; + const newerReset = Date.now() + 2 * 60 * 60_000; + rotateAnthropicAccountOn429(config, id, null, null, Date.now(), rejected(newerReset)); + finish(quotaResponse(0, 0)); + const [stale] = await pending; + expect(stale?.unavailable).toBe(true); + expect(getCachedProviderAccountQuota("anthropic", id)).toBeNull(); + expect(getAnthropicAccountHealthSnapshot(id)?.cooldownUntil).toBe(Math.floor(newerReset / 1_000) * 1_000); + }); + + test("a newer 429 after settlement blocks the older quota publication", async () => { + const id = await seed(); + const firstReset = Date.now() + 60 * 60_000; + const newerReset = Date.now() + 2 * 60 * 60_000; + rotateAnthropicAccountOn429(config, id, null, null, Date.now(), rejected(firstReset)); + setAnthropicQuotaAfterSettlementForTests(() => { + rotateAnthropicAccountOn429(config, id, null, null, Date.now(), rejected(newerReset)); + }); + globalThis.fetch = (async () => quotaResponse(10, 20)) as typeof fetch; + + const [stale] = await fetchProviderAccountQuotas("anthropic", true); + expect(stale?.unavailable).toBe(true); + expect(getCachedProviderAccountQuota("anthropic", id)).toBeNull(); + expect(getAnthropicAccountHealthSnapshot(id)?.cooldownUntil).toBe(Math.floor(newerReset / 1_000) * 1_000); + }); + + test("forced recovery bypasses an account-level quota flight started before the 429", async () => { + const id = await seed(); + let finishOld!: (response: Response) => void; + const oldResponse = new Promise(resolve => { finishOld = resolve; }); + let oldStarted!: () => void; + const dispatched = new Promise(resolve => { oldStarted = resolve; }); + let calls = 0; + globalThis.fetch = (async () => { + calls += 1; + if (calls === 1) { + oldStarted(); + return oldResponse; + } + return quotaResponse(10, 20); + }) as typeof fetch; + + const oldProbe = fetchProviderAccountQuotas("anthropic", true); + await dispatched; + rotateAnthropicAccountOn429(config, id, null, null, Date.now(), rejected(Date.now() + 60 * 60_000)); + + const [recovered] = await fetchProviderAccountQuotas("anthropic", true); + expect(calls).toBe(2); + expect(recovered?.quota).toMatchObject({ fiveHourPercent: 10, weeklyPercent: 20 }); + expect(getAnthropicAccountHealthSnapshot(id)).toBeNull(); + + finishOld(quotaResponse(90, 90)); + const [stale] = await oldProbe; + expect(stale?.unavailable).toBe(true); + expect(getCachedProviderAccountQuota("anthropic", id)).toMatchObject({ + fiveHourPercent: 10, + weeklyPercent: 20, + }); + }); + + test("recovery dispatch does not join a usage request started before the cooldown", async () => { + const id = await seed(); + let finishOld!: (response: Response) => void; + const oldResponse = new Promise(resolve => { finishOld = resolve; }); + let calls = 0; + globalThis.fetch = (async () => { + calls += 1; + return calls === 1 ? oldResponse : quotaResponse(0, 0); + }) as typeof fetch; + const oldProbe = fetchAnthropicUsageQuota("access-old"); + await Promise.resolve(); + rotateAnthropicAccountOn429(config, id, null, null, Date.now(), rejected(Date.now() + 60 * 60_000)); + + await fetchProviderAccountQuotas("anthropic", true); + expect(calls).toBe(2); + expect(getAnthropicAccountHealthSnapshot(id)).toBeNull(); + finishOld(quotaResponse(100, 100)); + await oldProbe; + }); + + test("a post-recovery third flight cannot rejoin the pre-429 usage request", async () => { + const id = await seed(); + let finishOld!: (response: Response) => void; + const oldResponse = new Promise(resolve => { finishOld = resolve; }); + let finishThird!: (response: Response) => void; + const thirdResponse = new Promise(resolve => { finishThird = resolve; }); + let calls = 0; + globalThis.fetch = (async () => { + calls += 1; + return calls === 1 ? oldResponse : calls === 2 ? quotaResponse(10, 20) : thirdResponse; + }) as typeof fetch; + + const oldProbe = fetchProviderAccountQuotas("anthropic", true); + while (calls < 1) await Promise.resolve(); + rotateAnthropicAccountOn429(config, id, null, null, Date.now(), rejected(Date.now() + 60 * 60_000)); + await fetchProviderAccountQuotas("anthropic", true); + const thirdTransport = fetchAnthropicUsageQuota("access-old"); + const thirdProbe = fetchProviderAccountQuotas("anthropic", true); + finishOld(quotaResponse(100, 100)); + await oldProbe; + if (calls === 3) finishThird(quotaResponse(30, 40)); + await thirdTransport; + const [third] = await thirdProbe; + expect(calls).toBe(3); + expect(third?.quota).toMatchObject({ fiveHourPercent: 30, weeklyPercent: 40 }); + expect(getCachedProviderAccountQuota("anthropic", id)).toMatchObject({ fiveHourPercent: 30, weeklyPercent: 40 }); + }); + + for (const failure of ["null", "reject"] as const) { + test(`superseded ${failure} cannot replace recovered cache availability or timestamp`, async () => { + const id = await seed(); + let finishOld!: (response: Response) => void; + let rejectOld!: (reason: Error) => void; + const oldResponse = new Promise((resolve, reject) => { finishOld = resolve; rejectOld = reject; }); + let calls = 0; + globalThis.fetch = (async () => { + calls += 1; + return calls === 1 ? oldResponse : quotaResponse(10, 20); + }) as typeof fetch; + + const oldProbe = fetchProviderAccountQuotas("anthropic", true); + while (calls < 1) await Promise.resolve(); + rotateAnthropicAccountOn429(config, id, null, null, Date.now(), rejected(Date.now() + 60 * 60_000)); + await fetchProviderAccountQuotas("anthropic", true); + const recovered = accountQuotaCache.get(accountCacheKey("anthropic", id)); + expect(recovered?.unavailable).toBeUndefined(); + if (failure === "null") finishOld(new Response("busy", { status: 503 })); + else rejectOld(new Error("timeout")); + await oldProbe; + + expect(accountQuotaCache.get(accountCacheKey("anthropic", id))).toBe(recovered); + expect(getCachedProviderAccountQuota("anthropic", id)).toMatchObject({ fiveHourPercent: 10, weeklyPercent: 20 }); + const [cached] = await fetchProviderAccountQuotas("anthropic"); + expect(cached?.unavailable).toBeUndefined(); + expect(calls).toBe(2); + }); + } + + test("an old token-resolution failure cannot replace a newer recovered quota entry", async () => { + const id = await seed(); + await saveAccountCredential("anthropic", id, { + access: "access-expired", refresh: "refresh-old", expires: Date.now() - 60_000, + accountId: "upstream-account", email: "recovery@example.test", + }); + let failRefresh!: (error: Error) => void; + let refreshStarted!: () => void; + const refreshPending = new Promise((_resolve, reject) => { failRefresh = reject; }); + const started = new Promise(resolve => { refreshStarted = resolve; }); + globalThis.fetch = (async (input: RequestInfo | URL) => { + if (String(input).includes("/v1/oauth/token")) { refreshStarted(); return refreshPending; } + return quotaResponse(10, 20); + }) as typeof fetch; + + const older = fetchProviderAccountQuotas("anthropic", true); + await started; + await saveAccountCredential("anthropic", id, { + access: "access-recovered", refresh: "refresh-new", expires: Date.now() + 60 * 60_000, + accountId: "upstream-account", email: "recovery@example.test", + }); + rotateAnthropicAccountOn429(config, id, null, null, Date.now(), rejected(Date.now() + 60 * 60_000)); + const [newer] = await fetchProviderAccountQuotas("anthropic", true); + expect(newer?.unavailable).toBeUndefined(); + const key = accountCacheKey("anthropic", id); + const recovered = accountQuotaCache.get(key); + expect(recovered?.quota).toMatchObject({ fiveHourPercent: 10, weeklyPercent: 20 }); + + failRefresh(new Error("old refresh failed")); + const [stale] = await older; + expect(stale?.unavailable).toBe(true); + expect(accountQuotaCache.get(key)).toBe(recovered); + expect(accountQuotaCache.get(key)?.ts).toBe(recovered?.ts); + const [cached] = await fetchProviderAccountQuotas("anthropic"); + expect(cached?.unavailable).toBeUndefined(); + }); + + test("credential replacement invalidates a live success on routing and account reads", async () => { + const id = await seed(); + let calls = 0; + globalThis.fetch = (async () => { calls += 1; return quotaResponse(calls === 1 ? 10 : 30, 20); }) as typeof fetch; + const [live] = await fetchProviderAccountQuotas("anthropic", true); + expect(live?.isCurrent?.()).toBe(true); + expect(getCachedProviderAccountQuota("anthropic", id)?.fiveHourPercent).toBe(10); + + await saveCredential("anthropic", { + access: "access-new", refresh: "refresh-new", expires: Date.now() + 60 * 60_000, + accountId: "upstream-account", email: "recovery@example.test", + }); + expect(live?.isCurrent?.()).toBe(false); + expect(getCachedProviderAccountQuota("anthropic", id)).toBeNull(); + const [fresh] = await fetchProviderAccountQuotas("anthropic"); + expect(calls).toBe(2); + expect(fresh?.quota?.fiveHourPercent).toBe(30); + }); + + test("a later cooldown invalidates live cached success before the account TTL", async () => { + const id = await seed(); + let calls = 0; + globalThis.fetch = (async () => { calls += 1; return quotaResponse(calls === 1 ? 10 : 30, 20); }) as typeof fetch; + const [live] = await fetchProviderAccountQuotas("anthropic", true); + expect(live?.isCurrent?.()).toBe(true); + rotateAnthropicAccountOn429(config, id, null, null, Date.now(), rejected(Date.now() + 60 * 60_000)); + expect(live?.isCurrent?.()).toBe(false); + expect(getCachedProviderAccountQuota("anthropic", id)).toBeNull(); + const [fresh] = await fetchProviderAccountQuotas("anthropic"); + expect(calls).toBe(2); + expect(fresh?.quota?.fiveHourPercent).toBe(30); + }); +}); diff --git a/tests/cli/cli-account-pool-verbs.test.ts b/tests/cli/cli-account-pool-verbs.test.ts index a6b2613fbcf..caab488bcf6 100644 --- a/tests/cli/cli-account-pool-verbs.test.ts +++ b/tests/cli/cli-account-pool-verbs.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { cmdPause, cmdPauseExhausted, cmdStrategy, cmdSticky, cmdRoutes } from "../../src/cli/account-extended"; +import { cmdClearCooldown, cmdPause, cmdPauseExhausted, cmdStrategy, cmdSticky, cmdRoutes } from "../../src/cli/account-extended"; import type { AccountDeps } from "../../src/cli/account-api"; /** @@ -184,6 +184,71 @@ describe("ocx account pause / resume", () => { }); }); +describe("ocx account clear-cooldown", () => { + test("Codex keeps its dedicated route and body", async () => { + const calls: Captured[] = []; + const out = capture(); + try { + expect(await cmdClearCooldown(["openai", "acct_1"], deps(() => ({ json: { ok: true, cleared: true } }), calls))).toBe(0); + } finally { out.restore(); } + expect(calls.at(-1)).toEqual({ + method: "POST", + path: "/api/codex-auth/accounts/clear-cooldown", + body: { id: "acct_1" }, + }); + }); + + test("Anthropic resolves an alias and posts to the OAuth cooldown owner", async () => { + const calls: Captured[] = []; + const out = capture(); + const anthropicDeps: AccountDeps = { + baseUrl: "http://127.0.0.1:10100", + loadConfigImpl: () => ({ providers: { anthropic: {} } }) as never, + fetchImpl: (async (url: string | URL | Request, init?: RequestInit) => { + const parsed = new URL(String(url)); + const call: Captured = { + method: init?.method ?? "GET", + path: parsed.pathname + parsed.search, + body: init?.body === undefined ? undefined : JSON.parse(String(init.body)), + }; + calls.push(call); + const json = call.method === "GET" + ? { accounts: [{ id: "anthropic_1", alias: "work" }] } + : { ok: true, cleared: true }; + return Response.json(json); + }) as typeof fetch, + }; + try { + expect(await cmdClearCooldown(["anthropic", "work"], anthropicDeps)).toBe(0); + } finally { out.restore(); } + expect(calls).toEqual([ + { method: "GET", path: "/api/oauth/accounts?provider=anthropic", body: undefined }, + { + method: "POST", + path: "/api/oauth/accounts/clear-cooldown", + body: { provider: "anthropic", accountId: "anthropic_1" }, + }, + ]); + expect(out.lines.join("\n")).toContain("anthropic: cooldown lifted for work"); + }); + + test("unrelated OAuth providers are rejected before any management request", async () => { + const calls: Captured[] = []; + const out = capture(); + let code: number; + try { + code = await cmdClearCooldown(["google-antigravity", "acct_1"], { + baseUrl: "http://127.0.0.1:10100", + loadConfigImpl: () => ({ providers: { "google-antigravity": { authMode: "oauth" } } }) as never, + fetchImpl: (async () => { calls.push({ method: "GET", path: "unexpected", body: undefined }); return Response.json({}); }) as typeof fetch, + }); + } finally { out.restore(); } + expect(code).toBe(1); + expect(calls).toHaveLength(0); + expect(out.errors.join("\n")).toContain("no operator-clearable account cooldown"); + }); +}); + describe("ocx account pause-exhausted", () => { test("reports which accounts were paused", async () => { const calls: Captured[] = []; diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 74dab858946..b755124ee7d 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -117,6 +117,7 @@ "anthropic-agentrouter-language-framing.test.ts": "adapters/anthropic", "anthropic-baseurl-override.test.ts": "adapters/anthropic", "anthropic-compatible-stream.test.ts": "adapters/anthropic", + "anthropic-cooldown-recovery.test.ts": "adapters/anthropic", "anthropic-empty-content.test.ts": "adapters/anthropic", "anthropic-eof-tolerance.test.ts": "adapters/anthropic", "anthropic-error-body.test.ts": "adapters/anthropic", From 540af24384d713603ff96863d811dfccca5392e8 Mon Sep 17 00:00:00 2001 From: JUN Date: Wed, 30 Sep 2026 07:37:15 +0900 Subject: [PATCH 25/26] fix(keyring): target-platform path rules for packaged addon candidates (#6261) * fix(keyring): compute packaged addon candidates with the target platform's path rules * test(keyring): cover target-platform path rules from a foreign host --- src/lib/keyring-native.ts | 5 ++++- tests/ci-workflows/keyring-smoke.test.ts | 7 +++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/lib/keyring-native.ts b/src/lib/keyring-native.ts index 5ade1a003f0..587f3783e61 100644 --- a/src/lib/keyring-native.ts +++ b/src/lib/keyring-native.ts @@ -1,6 +1,6 @@ import { existsSync } from "node:fs"; import { createRequire } from "node:module"; -import { basename, dirname, join, resolve } from "node:path"; +import { posix, win32 } from "node:path"; import { isStandaloneBinary, standaloneRoot } from "./standalone"; export interface KeyringBinding { @@ -74,6 +74,9 @@ export function packagedKeyringCandidates({ if (root === undefined) return []; const asset = runtimeAsset(platform, arch); if (!asset) return []; + // Paths follow the target platform's rules rather than the host's, so a candidate list is the + // same whether it is computed on that platform or simulated from another one. + const { basename, dirname, join, resolve } = platform === "win32" ? win32 : posix; const executableDir = resolve(root); const adjacent = join(executableDir, "keyring", asset.filename); if (platform === "linux") { diff --git a/tests/ci-workflows/keyring-smoke.test.ts b/tests/ci-workflows/keyring-smoke.test.ts index c853c3c0b46..9119752b7ac 100644 --- a/tests/ci-workflows/keyring-smoke.test.ts +++ b/tests/ci-workflows/keyring-smoke.test.ts @@ -199,6 +199,13 @@ describe("packaged keyring native binding", () => { .toEqual(["/opt/ocx/keyring/keyring.linux-x64-gnu.node"]); }); + test("candidate paths follow the target platform's rules on any host", () => { + expect(packagedKeyringCandidates({ root: "C:\\Program Files\\OpenCodex", platform: "win32", arch: "x64" })) + .toEqual(["C:\\Program Files\\OpenCodex\\keyring\\keyring.win32-x64-msvc.node"]); + expect(packagedKeyringCandidates({ root: "/opt/ocx", platform: "linux", arch: "x64" })) + .toEqual(["/opt/ocx/keyring/keyring.linux-x64-gnu.node"]); + }); + test("loads only an existing deterministic packaged path and never consults cwd", () => { const binding = { Entry: class {}, AsyncEntry: class {} } as unknown as KeyringBinding; const calls: string[] = []; From cfa56e34df9e48546c17335dbb2d674e23b60797 Mon Sep 17 00:00:00 2001 From: JUN Date: Wed, 30 Sep 2026 08:27:36 +0900 Subject: [PATCH 26/26] feat(models): GPT-6.1 Sol across providers as the Sol default, and TokenLab per-model wires (#6263) * docs(devlog): plan GPT-6.1 Sol rollout, default swap and TokenLab protocol routing * docs(devlog): record the GPT-6.1 Sol plan audit and JEV deferral * feat(models): add GPT-6.1 Sol across providers and make it the Sol default OpenAI released GPT-6.1 Sol on 2026-09-29 as the upgrade to GPT-6 Sol (Astra and Luna did not move). List it wherever GPT-6 Sol is served: the native Codex row (pinned verbatim from openai/codex models.json after #49318; low..ultra, default low, 272K/872K), the OpenAI API (1.05M / 922K / 128K, low..max), OpenRouter, GitHub Copilot (Responses), CodeBuddy/Kiro/Devin preemptive seeds, and the Bedrock/Cloudflare/Kilo/OpenCode Zen/Vercel metadata rows. Price it at $2 / $0.10 cached / $2.50 write / $10 with the long-context tier. Move every GPT-6 Sol default to GPT-6.1 Sol: the subagent roster default, the configured-native template, docker smoke and docs. Roster migration v3 swaps a bare gpt-6-sol in stored rosters once; GPT-6 Sol stays selectable. * feat(tokenlab): route each model over its declared wire TokenLab publishes per-model request formats (tokenlab.accepted_request_formats on GET /v1/models/{id}). Keep the released Chat preset as the provider-wide wire, and: - send gpt-6-astra, gpt-6.1-sol, gpt-6-sol, gpt-6-luna, grok-4.7, deepseek-v4.1-flash, deepseek-v4-pro, kimi-k3 and glm-5.3 over Responses for Responses inbound (Codex); Chat and Anthropic clients keep Chat, and an explicit modelAdapters entry still wins; - pin claude-* ids to Anthropic Messages (/v1/messages, x-api-key) through the endpoint-bound prefix pin Command Code uses, so a retargeted row is untouched; - leave gemini-3.8-flash and every other model on Chat. No delivery-policy header is sent; the API key policy stays authoritative. * docs(devlog): script the 2.73.0 release bundling RT6 * test(ci): docker smoke expects subagent roster version 3 --- .../260930_gpt_6_1_sol_rollout/000_README.md | 21 ++ .../010_research_digest.md | 53 ++++ .../020_catalog_surfaces.md | 30 +++ .../030_default_swap.md | 14 ++ .../040_tokenlab_protocols.md | 20 ++ .../260930_gpt_6_1_sol_rollout/050_release.md | 39 +++ .../src/content/docs/fr/guides/claude-code.md | 2 +- .../docs/fr/reference/configuration/agents.md | 2 +- .../src/content/docs/guides/claude-code.md | 2 +- .../src/content/docs/guides/providers.md | 14 ++ .../src/content/docs/ja/guides/claude-code.md | 2 +- .../docs/ja/reference/configuration/agents.md | 2 +- .../src/content/docs/ko/guides/claude-code.md | 2 +- .../docs/ko/reference/configuration/agents.md | 2 +- .../src/content/docs/reference/adapters.md | 4 +- .../docs/reference/configuration/agents.md | 11 +- .../docs/reference/configuration/providers.md | 18 +- .../src/content/docs/ru/guides/claude-code.md | 2 +- .../docs/ru/reference/configuration/agents.md | 2 +- .../src/content/docs/tr/guides/claude-code.md | 2 +- .../docs/tr/reference/configuration/agents.md | 2 +- .../content/docs/zh-cn/guides/claude-code.md | 2 +- .../zh-cn/reference/configuration/agents.md | 2 +- .../content/docs/zh-tw/guides/claude-code.md | 2 +- .../zh-tw/reference/configuration/agents.md | 2 +- scripts/ci/docker-smoke.ts | 4 +- scripts/model-metadata.source.json | 231 ++++++++++++++++++ scripts/test-layout/layout.json | 2 + src/adapters/devin/live-models.ts | 5 + src/adapters/kiro/reasoning.ts | 2 + src/codex/catalog/metadata.ts | 6 + src/codex/catalog/native-models.ts | 20 +- src/codex/catalog/pinned-models.ts | 4 +- src/codex/data/roster-pinned-models.json | 179 +++++++++++++- src/codex/model-entitlements.ts | 2 +- src/config/subagent-models.ts | 35 ++- src/generated/model-metadata.ts | 8 +- src/providers/codebuddy-models.ts | 6 + src/providers/kiro-models.ts | 4 + src/providers/registry/entries-core.ts | 4 +- src/providers/registry/entries-extended.ts | 15 +- src/providers/registry/model-seeds.ts | 8 +- src/types/wire.ts | 7 + src/usage/expected-prices.ts | 11 + structure/catalog.md | 8 +- structure/providers-and-adapters.md | 6 + structure/providers/openai-accounts.md | 9 +- structure/subagents.md | 8 +- .../claude-management-api.test.ts | 4 +- tests/codex-integration/codex-catalog.test.ts | 2 +- .../configured-native-models.test.ts | 12 +- .../gpt6-native-rows.test.ts | 6 +- .../codex-integration/gpt61-sol-rows.test.ts | 127 ++++++++++ tests/fixtures/test-layout-expected.json | 2 + .../provider-registry-parity.test.ts | 2 +- tests/providers/tokenlab-protocols.test.ts | 139 +++++++++++ .../routing/subagent-roster-migration.test.ts | 61 +++-- ...erver-startup-reconcile-resilience.test.ts | 6 +- tests/service/service-tier-capability.test.ts | 1 + tests/usage/usage-cost.test.ts | 8 +- 60 files changed, 1125 insertions(+), 83 deletions(-) create mode 100644 devlog/_plan/260930_gpt_6_1_sol_rollout/000_README.md create mode 100644 devlog/_plan/260930_gpt_6_1_sol_rollout/010_research_digest.md create mode 100644 devlog/_plan/260930_gpt_6_1_sol_rollout/020_catalog_surfaces.md create mode 100644 devlog/_plan/260930_gpt_6_1_sol_rollout/030_default_swap.md create mode 100644 devlog/_plan/260930_gpt_6_1_sol_rollout/040_tokenlab_protocols.md create mode 100644 devlog/_plan/260930_gpt_6_1_sol_rollout/050_release.md create mode 100644 tests/codex-integration/gpt61-sol-rows.test.ts create mode 100644 tests/providers/tokenlab-protocols.test.ts diff --git a/devlog/_plan/260930_gpt_6_1_sol_rollout/000_README.md b/devlog/_plan/260930_gpt_6_1_sol_rollout/000_README.md new file mode 100644 index 00000000000..48a97ac491c --- /dev/null +++ b/devlog/_plan/260930_gpt_6_1_sol_rollout/000_README.md @@ -0,0 +1,21 @@ +# 260930 GPT-6.1 Sol rollout, TokenLab protocol follow-up, release + +Status: open. Loop session `01a0ef42-da06-7200-8394-aa35ebbe4ba9`, branch `codex/gpt-6-1-sol-rollout` from `origin/dev` `b78bfb8f00`. + +OpenAI released GPT-6.1 Sol on 2026-09-29 as the successor to GPT-6 Sol. Only Sol moved to 6.1; Astra and Luna stay on GPT-6. This unit adds the model everywhere GPT-6 Sol is served, moves every place where GPT-6 Sol is the *default* to GPT-6.1 Sol (the same move #5640 made from GPT-5.6 to GPT-6), folds in TokenLab's protocol request from mail 546, and ships a release. + +| Doc | Work-phase | Content | +|---|---|---| +| 010_research_digest.md | wp1 | Sourced facts for GPT-6.1 Sol and the TokenLab contract | +| 020_catalog_surfaces.md | wp2 | Diff-level list of provider, catalog, pricing and docs rows | +| 030_default_swap.md | wp2 | Defaults moving from gpt-6-sol to gpt-6.1-sol, roster migration v3 | +| 040_tokenlab_protocols.md | wp3 | Per-model wire routing; TokenLab JEV decision backend deferred to its own unit | +| 050_release.md | wp4 | PR, CI, merge, preview/main promotion, release.yml, npm verification | + +## Audit record (wp1) + +Read-only reviewer on gpt-6-sol (high), 2026-09-30: + +- Round 1 FAIL, four blockers. Three concerned the JEV backend (combo dispatch passes no combo settings, combo normalization/persistence and the GUI editor would drop new fields, credential/URL/outbound guard must switch together). One named roster tests that enumerate models by value. +- Fold: the roster tests are listed in 020; the JEV backend is deferred to its own unit (040 records why and sketches it). +- Round 2 PASS. Non-blocking notes kept for B: the v3 roster migration must run after the v2 step and touch only bare `gpt-6-sol`; TokenLab's anthropic wire sends `x-api-key` to `/v1/messages`, which TokenLab accepts; `kiro-adapter.test.ts` (2047/2050) and `codex-catalog.test.ts` (7974/7985) take in-place edits only. diff --git a/devlog/_plan/260930_gpt_6_1_sol_rollout/010_research_digest.md b/devlog/_plan/260930_gpt_6_1_sol_rollout/010_research_digest.md new file mode 100644 index 00000000000..258a8e74f7e --- /dev/null +++ b/devlog/_plan/260930_gpt_6_1_sol_rollout/010_research_digest.md @@ -0,0 +1,53 @@ +# 010 Research digest + +Observed 2026-09-30 KST. Full report with every URL: Aside run, `/Users/jun/.aside/u/0/artifacts/gpt61sol-research.md` (kept outside the repo). Live probes re-run from this checkout are marked (live). + +## GPT-6.1 Sol, official + +| Fact | Value | Source | +|---|---|---| +| API id | `gpt-6.1-sol`, no dated snapshot | developers.openai.com/api/docs/models/gpt-6.1-sol | +| Released | 2026-09-29 (API, ChatGPT Work, Codex) | openai.com/index/introducing-gpt-6-1-sol/ | +| API context / max input / max output | 1,050,000 / 922,000 / 128,000 | model page | +| API efforts | low, medium (default), high, xhigh, max | model page | +| Price per 1M | $2 input, $0.10 cached, $2.50 cache write, $10 output | pricing.md | +| Long context (>272K prompt) | $4 / $0.20 / $5 / $15 | pricing.md | +| Fast | 2x standard | pricing.md | +| Modalities | text + image in, text out | model page | +| Codex row (live, openai/codex models.json, #49318) | priority 1 (catalog default), efforts low..ultra, default effort low, context 272,000 / 872,000, Fast tier "2x speed, increased usage", minimal client 0.153.0 | codex-rs/models-manager/models.json | +| gpt-6-sol | retained: visibility list, priority 3, no upgrade, not deprecated | same file; deprecations page | +| GPT-6.1 Luna / Astra | do not exist (doc pages 404, absent from pricing, models index and Codex catalog) | model pages | + +Only cached input changed versus GPT-6 Sol ($0.20 -> $0.10). + +## Third-party listing (drives 020) + +| Provider in this repo | Status | Id | +|---|---|---| +| OpenRouter | listed | `openai/gpt-6.1-sol` (1,050,000 ctx, 128,000 out) | +| Vercel AI Gateway | listed | `openai/gpt-6.1-sol` | +| GitHub Copilot | listed, GA rollout (github.blog 2026-09-29) | `gpt-6.1-sol` | +| Kilo | listed (models.dev) | `openai/gpt-6.1-sol` | +| OpenCode Zen | listed (live /zen/v1/models) | `gpt-6.1-sol` | +| Devin | listed (devin.ai blog); id not published | preemptive `gpt-6-1-sol` following its `gpt-5-6-sol` spelling | +| TokenLab | listed (live /v1/models/gpt-6.1-sol, Chat + Responses) | `gpt-6.1-sol` (live discovery) | +| Amazon Bedrock | not in models.dev; openai/codex source references `openai.gpt-6.1-sol` | preemptive | +| Cloudflare AI Gateway | not in catalog; OpenAI passthrough | preemptive | +| Kiro, CodeBuddy | not listed / unreadable; both carry preemptive gpt-6-sol rows today | preemptive, same policy as the Sonnet 5.5 Kiro rows | +| ZenMux | page explicitly "not a registered ZenMux page" | not added (live discovery picks it up) | +| DigitalOcean, Scaleway | not listed; DO naming for 6.1 unknown | not added | + +## TokenLab (mail 546, 2026-09-30; live contract re-checked) + +`GET https://api.tokenlab.sh/v1/models/{id}` returns `tokenlab.accepted_request_formats` (live): + +| Models | Formats | +|---|---| +| gpt-6-astra, gpt-6-sol, gpt-6-luna, gpt-6.1-sol | chat, responses | +| claude-opus-5, claude-opus-5-5, claude-sonnet-5, claude-sonnet-5-5, claude-fable-5, claude-fable-5-1 | chat, anthropic_messages | +| deepseek-v4.1-flash, deepseek-v4-pro, kimi-k3, glm-5.3 | chat, responses, anthropic_messages | +| grok-4.7 | chat, responses | +| gemini-3.8-flash | chat, gemini_generate_content | + +Endpoints from docs.tokenlab.sh/llms.txt: `/v1/chat/completions`, `/v1/responses`, `/v1/messages` (x-api-key or Bearer), `/v1/systemone`. System One takes `{model, state, questions}` with Bearer auth, current model `jev-1.13` — the same body shape `src/combos/jev.ts` already sends to TypeSafe. Vincent asks to keep the user's API-key delivery policy default (no forced `X-TokenLab-Delivery-Policy`). + diff --git a/devlog/_plan/260930_gpt_6_1_sol_rollout/020_catalog_surfaces.md b/devlog/_plan/260930_gpt_6_1_sol_rollout/020_catalog_surfaces.md new file mode 100644 index 00000000000..fc9511b2394 --- /dev/null +++ b/devlog/_plan/260930_gpt_6_1_sol_rollout/020_catalog_surfaces.md @@ -0,0 +1,30 @@ +# 020 Catalog surfaces for gpt-6.1-sol (wp2) + +Rule: add a row wherever a gpt-6-sol row exists and 010 says listed or preemptive; comment preemptive rows `260930 preemptive`. No gpt-6.1-luna / gpt-6.1-astra anywhere. + +| File | Change | +|---|---| +| scripts/model-metadata.source.json | Clone the gpt-6-sol row as gpt-6.1-sol in openai, openai-codex, github-copilot, kilo, openrouter, vercel-ai-gateway, opencode-zen, amazon-bedrock (preemptive), cloudflare-ai-gateway (preemptive); cache_read 0.10 (0.20 on opencode-zen per its published figure); release_date 2026-09-29; knowledge 2026-04-30. ZenMux not added. Then `bun run generate:model-metadata`. | +| src/usage/expected-prices.ts | `GPT61_SOL` cost4 (2 / 10 / 0.10 / 2.50) with long-context tier; rows for openai-apikey (verified), openai (verified-derived), devin, devin-cli (verified-derived); pricing-list id. | +| src/providers/registry/model-seeds.ts | `OPENAI_GPT6_MODELS` gains gpt-6.1-sol (flows to openai-apikey 1,050,000 / 922,000 / 128,000 / low..max and OpenRouter models); OpenRouter context map adds openai/gpt-6.1-sol. | +| src/providers/registry/entries-core.ts | OpenRouter `modelSupportsServiceTier` adds openai/gpt-6.1-sol; BizRouter seed adds openai/gpt-6.1-sol; Devin seed adds gpt-6-1-sol (preemptive). | +| src/providers/registry/entries-extended.ts | Copilot models + `modelWireDefaults` gpt-6.1-sol -> openai-responses. | +| src/providers/codebuddy-models.ts, src/providers/kiro-models.ts, src/adapters/kiro/reasoning.ts, src/adapters/devin/live-models.ts | Mirror the gpt-6-sol rows (preemptive). | +| src/codex/catalog/native-models.ts | `NATIVE_GPT61_SOL_MODEL = "gpt-6.1-sol"` in built-in list, SELF_DESCRIBED set, drain-sentinel set; configured-native template moves to gpt-6.1-sol. | +| src/codex/catalog/metadata.ts | DOCUMENTED_NATIVE_OPENAI_ADDITIONS and context map (NATIVE_GPT6_CONTEXT). | +| src/codex/catalog/effort.ts | Ladder entry low..ultra like Sol. | +| src/codex/data/roster-pinned-models.json | Append the upstream gpt-6.1-sol row verbatim from openai/codex models.json (codex-rs bundle, #49318); pinned-models.ts comment updated. | +| src/codex/model-entitlements.ts | Comment: 6.1 Sol ungated like Sol. | +| docs-site providers.md, structure/catalog.md, structure/providers/openai-accounts.md | Document the row. | + +Tests that enumerate the roster by value and must gain gpt-6.1-sol (audit A1 blocker 4): + +- tests/codex-integration/native-model-toggle.test.ts:95 (built-in native roster) and :127 (flagship list) +- tests/providers/provider-registry-parity.test.ts:314 (exact openai-apikey model list) +- tests/codex-integration/codex-catalog.test.ts:6842 (catalog roster; file is 7974/7985 lines, so edit in place without adding lines) +- tests/providers/github-copilot/github-copilot-wire-defaults.test.ts:33, tests/service/service-tier-capability.test.ts:73 (OpenRouter tier map), tests/routing/subagent-model-fallback.test.ts:276 +- tests/providers/kiro/kiro-adapter.test.ts:1744 is at 2047/2050: edit the existing list in place only. +- tests/codex-integration/configured-native-models.test.ts:93-102: a configured native must now borrow gpt-6.1-sol's pinned capabilities and hash (wp2 audit NEAR-PASS item). catalog-routed-comp-hash.test.ts:75 keeps its gpt-6-sol forward-alias expectation. +- BizRouter is not added: 010 has no evidence it lists gpt-6.1-sol, and live discovery picks it up once it does. + +New focused test `tests/codex-integration/gpt61-sol-rows.test.ts` (registered in scripts/test-layout/layout.json and tests/fixtures/test-layout-expected.json) asserting: native row ladder low..ultra, default effort low, 272,000/872,000 context; openai-apikey 1,050,000/922,000/128,000 and low..max; Copilot Responses wire; expected price 2/10/0.10/2.50; no gpt-6.1-luna/astra id in any registry entry. diff --git a/devlog/_plan/260930_gpt_6_1_sol_rollout/030_default_swap.md b/devlog/_plan/260930_gpt_6_1_sol_rollout/030_default_swap.md new file mode 100644 index 00000000000..3c7267d977a --- /dev/null +++ b/devlog/_plan/260930_gpt_6_1_sol_rollout/030_default_swap.md @@ -0,0 +1,14 @@ +# 030 Default swap gpt-6-sol -> gpt-6.1-sol (wp2) + +Precedent: #5640 (`8adda594fd`) set the subagent default to the GPT-6 trio and upgraded stored rosters from 5.6 once, by version. + +| Surface | Change | +|---|---| +| src/config/subagent-models.ts | `DEFAULT_SUBAGENT_MODELS = [astra, gpt-6.1-sol, luna]`; `SUBAGENT_MODELS_VERSION = 3`; the v3 step replaces bare `gpt-6-sol` with `gpt-6.1-sol` in place (deduped), once. A later deliberate re-pick of gpt-6-sol stays. v0/v1 installs chain through v2 then v3. | +| scripts/ci/docker-smoke.ts | subagentModels default trio. | +| Codex configured-native template | borrows gpt-6.1-sol's row (020). | +| docs-site agents.md (10 locales), guides/claude-code.md (10 locales), structure/subagents.md | Name the new default. | +| Tests | subagent-roster-migration (v2 -> v3, idempotence, re-pick retained, routed ids untouched), claude picker/intercept/management expectations that read the default trio, server startup reconcile. | + +Out of scope: removing gpt-6-sol anywhere; it stays listed and selectable. + diff --git a/devlog/_plan/260930_gpt_6_1_sol_rollout/040_tokenlab_protocols.md b/devlog/_plan/260930_gpt_6_1_sol_rollout/040_tokenlab_protocols.md new file mode 100644 index 00000000000..bb762036c79 --- /dev/null +++ b/devlog/_plan/260930_gpt_6_1_sol_rollout/040_tokenlab_protocols.md @@ -0,0 +1,20 @@ +# 040 TokenLab per-model protocols and JEV backend (wp3) + +## Wires + +Keep the preset's provider-wide adapter `openai-chat` (the released, verified path). Add registry defaults so each model rides its declared native wire: + +- `modelWireDefaults` -> `{ wire: "openai-responses", inbound: ["responses"] }` for gpt-6-astra, gpt-6-sol, gpt-6-luna, gpt-6.1-sol, grok-4.7, deepseek-v4.1-flash, deepseek-v4-pro, kimi-k3, glm-5.3. Scoped to Responses inbound (Codex) like the Alibaba Token Plan precedent (tests/providers/alibaba-token-plan-wire-defaults.test.ts): a Chat or Anthropic client keeps the verified Chat wire with no translation hop. User-overridable: an explicit `modelAdapters` entry of `openai-chat` wins. +- Claude ids -> Anthropic Messages through the existing endpoint-bound prefix pin (`WIRE_ADAPTER_PIN_PREFIXES` in src/types/wire.ts, the Command Code mechanism): `tokenlab: { endpoint: "https://api.tokenlab.sh/v1", prefixes: { "claude-": "anthropic" } }`. The anthropic adapter already normalizes `/v1` to `/v1/messages`. The pin is bound to the canonical endpoint, so a retargeted TokenLab row is untouched. Every live `claude-*` TokenLab id is checked to declare anthropic_messages before the prefix is used. +- Everything else, including gemini-3.8-flash, stays on Chat. Gemini native is deferred until tested, as Vincent proposed. +- No delivery-policy header is added: the user's API-key default stays authoritative. + +Rejected: a provider-wide switch to Responses (sends Claude/Gemini to an endpoint they do not accept); separate provider entries (duplicates models by default); widening `MODEL_ADAPTER_OVERRIDE_ALLOWED` to anthropic (needs the #404 credential threat model; the prefix pin already exists and is endpoint-bound). + +Tests: resolveWireProtocolOverride / resolved policy for each class on the canonical endpoint, user modelAdapters override back to Chat for a Responses default, retargeted base URL keeps Chat, anthropic URL resolves to https://api.tokenlab.sh/v1/messages. + +## JEV decision backend — deferred to its own unit + +Decision after audit A1 (blockers 1-3): not in this release. A per-combo backend has to travel through `src/server/responses/core-combo.ts:536` (the call passes no combo settings), combo normalization and persistence (`src/combos/types.ts:379`, `src/server/management/combo-routes.ts:231`), and the GUI combo editor round trip (`gui/src/combo-workspace-data.ts:266`, `:460`), and credential, fixed URL and canonical outbound guard (`src/combos/jev.ts:568`, `:599`) must switch together. It also sends conversation-derived decision state to a new third party, which deserves its own review and a GUI screenshot. That does not meet this unit's "fits cleanly" bar. + +Follow-up unit sketch: combo fields `jevBackend: "typesafe" | "tokenlab"` (default typesafe) and `jevModel` (`jev-*`), fixed endpoint per backend (`https://api.typesafe.ai/v1/systemone`, `https://api.tokenlab.sh/v1/systemone`, body `{model, state, questions}` confirmed identical in docs.tokenlab.sh/api-reference/systemone/create-decision), credentials never shared between backends, allowlist/timeout/cancellation/fail-open unchanged. diff --git a/devlog/_plan/260930_gpt_6_1_sol_rollout/050_release.md b/devlog/_plan/260930_gpt_6_1_sol_rollout/050_release.md new file mode 100644 index 00000000000..0da14177b60 --- /dev/null +++ b/devlog/_plan/260930_gpt_6_1_sol_rollout/050_release.md @@ -0,0 +1,39 @@ +# 050 Release (wp4) + +1. Isolated verification in a /private/tmp checkout of the exact head with fresh OPENCODEX_HOME / CODEX_HOME: typecheck, focused tests, test:changed, structure:check, privacy:scan, skill:surface:check, file-size ratchet. +2. One PR to dev from codex/gpt-6-1-sol-rollout with the template; wait for exact-head required CI; maintainer merge under MAINTAINERS.md dev policy. +3. Release train per scripts/release.ts and the 2.70.0 precedent: dev version pre-move PR, preview promotion PR, main promotion PR; push-event Cross-platform CI + Service lifecycle green on each promotion SHA; release.yml dispatched with expected-sha for preview then stable. +4. Verify GitHub release assets, npm `latest` / `preview` dist-tags, latest.json. +5. Close the unit: move to devlog/_fin with an outcome doc. + +## 2.73.0 concrete steps (revalidated at wp4 P, 2026-09-30) + +Scope, on the owner's request to bundle the RT6 stabilization chat: 2.73.0 ships everything on `dev` +since v2.72.0 — the RT6 train (13 PRs through #6203, recorded in `devlog/_plan/260930_release_train_6/` +by that chat), #6261 (`540af24384`, Windows keyring test portability), and this PR. That chat was asked +not to run the release train itself. `dev` already reads 2.73.0 (#6243). + +1. PR from `codex/gpt-6-1-sol-rollout` (rebased on `540af24384`). Exact-head PR CI plus a + `lane=all` Cross-platform CI dispatch on the head (the pull_request event skips windows 1-9). + Before merge, an explicit read-only security review of the TokenLab wire change + (MAINTAINERS.md: credential handling): where the API key is sent, which header, endpoint binding + of the Claude pin, no new logging; verdict recorded on the PR. + `scripts/ci/assert-mergeable-review.sh --maintainer-integration `, decision comment, + `gh pr merge --admin --squash --match-head-commit ` -> C. Assert `git show C:package.json` = 2.73.0. +2. Pre-move: `gh workflow run dev-version-bump.yml --ref main -f intended-version=2.73.0 -f mode=pre-move`; + merge its PR after CI -> dev 2.74.0. +3. Preview: branch `codex/promote-preview-2.73.0` from C, `git merge -s ours origin/preview`, + `bun scripts/release-version-sources.ts sync 2.73.0-preview.20260930`; diff vs C = four version sources. + PR to preview, `--admin --merge --match-head-commit` after its PR checks. The maintainer-integration + exception covers only `dev`; promotion merges run on the owner's explicit release authorization + for this unit (2026-09-30, "exec and release" / "배포해줘"), as 2.70.0-2.72.0 did, and each promotion + PR records that authorization. +4. Main: branch `codex/promote-main-2.73.0` from C, `git merge -s ours origin/main`, `git diff --quiet C HEAD`. + PR to main, same merge. +5. Push-event Cross-platform CI + Service lifecycle success on each promotion SHA. +6. `gh workflow run release.yml --ref preview -f version=2.73.0-preview.20260930 -f tag=preview -f dry-run=false -f expected-sha=`, + then `gh workflow run release.yml --ref main -f version=2.73.0 -f tag=latest -f dry-run=false -f expected-sha=
` + (release.yml defaults to a dry run and rejects other refs). Never republish; resume with + `resume-after-npm-publish=true` after an npm-acknowledged failure. +7. Verify npm dist-tags, gitHead, GitHub releases (stable not prerelease, asset count as v2.72.0), latest.json. +8. Record PR moving this unit to `devlog/_fin/`. diff --git a/docs-site/src/content/docs/fr/guides/claude-code.md b/docs-site/src/content/docs/fr/guides/claude-code.md index 6eeed5840aa..cdc60bf5320 100644 --- a/docs-site/src/content/docs/fr/guides/claude-code.md +++ b/docs-site/src/content/docs/fr/guides/claude-code.md @@ -190,7 +190,7 @@ du sélecteur à une route opencodex : ```bash ocx claude desktop bind claude-sonnet-4-6 xai/grok-4.7 -ocx claude desktop bind claude-opus-4-6 native/gpt-6-sol +ocx claude desktop bind claude-opus-4-6 native/gpt-6.1-sol ocx claude desktop unbind claude-opus-4-6 ``` diff --git a/docs-site/src/content/docs/fr/reference/configuration/agents.md b/docs-site/src/content/docs/fr/reference/configuration/agents.md index 7d846a6e1f2..495c22ffdb6 100644 --- a/docs-site/src/content/docs/fr/reference/configuration/agents.md +++ b/docs-site/src/content/docs/fr/reference/configuration/agents.md @@ -11,7 +11,7 @@ Les paramètres des agents déterminent la surface de collaboration Codex annonc | --- | --- | --- | --- | | `multiAgentMode?` | `"v1" \| "default" \| "v2"` | `"default"` | `v1` marque tous les modèles du catalogue comme compatibles v1 ; `v2` les marque tous comme compatibles v2. `default` rétablit les choix imposés en amont (Sol/Terra en v2, Luna en v1) et suit sinon l’indicateur natif `multi_agent_v2`. S’applique aux nouvelles sessions. | | `keepNativeChatGptOnV1?` | `boolean` | `false` | Lorsque `multiAgentMode` vaut `"v2"`, marque les lignes natives ChatGPT (Sol/Terra et les autres modèles du backend ChatGPT) comme v1. Les parents routés restent en v2. Utilisez cette option pour qu'un parent ChatGPT puisse encore lancer Grok ou Claude — les tâches enfants v2 natives sont chiffrées par le service en amont ([#92](https://github.com/lidge-jun/opencodex/issues/92)). Ignoré en `v1` et `default`. | -| `subagentModels?` | `string[]` | `gpt-6-astra`, `gpt-6-sol`, `gpt-6-luna` | Jusqu’à cinq identifiants de modèles natifs non qualifiés, qualifiés par un compte sous la forme `/`, ou routés sous la forme `provider/model`, affichés en tête du sélecteur de sous-agents. Le tableau de bord ne propose que les identifiants natifs non qualifiés et les identifiants routés ; lors de l’enregistrement, il omet les choix exacts qualifiés par un compte. Pour les définir, utilisez `ocx agent subagents set` ou modifiez la configuration. Après la [migration unique vers Astra](/reference/configuration/agents/#astra-roster-upgrade), une liste explicitement vide est conservée. | +| `subagentModels?` | `string[]` | `gpt-6-astra`, `gpt-6.1-sol`, `gpt-6-luna` | Jusqu’à cinq identifiants de modèles natifs non qualifiés, qualifiés par un compte sous la forme `/`, ou routés sous la forme `provider/model`, affichés en tête du sélecteur de sous-agents. Le tableau de bord ne propose que les identifiants natifs non qualifiés et les identifiants routés ; lors de l’enregistrement, il omet les choix exacts qualifiés par un compte. Pour les définir, utilisez `ocx agent subagents set` ou modifiez la configuration. Après la [migration unique vers Astra](/reference/configuration/agents/#astra-roster-upgrade), une liste explicitement vide est conservée. | | `injectionModel?` | `string` | — | Modèle de sous-agent natif ou routé privilégié dans les consignes de délégation v2 produites par le proxy. | | `injectionEffort?` | `string` | — | Niveau d’effort privilégié (de `low` à `ultra`), pertinent uniquement avec `injectionModel`. | | `injectionPrompt?` | `string` | — | Remplace le corps des consignes v2 intégrées. Accepte `{{model}}`, `{{effort}}`, `{{roster}}` et `{{fallback}}`. La présence d’un `injectionModel` suffit pour produire le prompt personnalisé. | diff --git a/docs-site/src/content/docs/guides/claude-code.md b/docs-site/src/content/docs/guides/claude-code.md index 4526e23c91d..1d3394163fe 100644 --- a/docs-site/src/content/docs/guides/claude-code.md +++ b/docs-site/src/content/docs/guides/claude-code.md @@ -255,7 +255,7 @@ every request, so you bind a picker row to an opencodex route instead: ```bash ocx claude desktop bind claude-sonnet-4-6 xai/grok-4.7 -ocx claude desktop bind claude-opus-4-6 native/gpt-6-sol +ocx claude desktop bind claude-opus-4-6 native/gpt-6.1-sol ocx claude desktop unbind claude-opus-4-6 ``` diff --git a/docs-site/src/content/docs/guides/providers.md b/docs-site/src/content/docs/guides/providers.md index 5bbf4735c66..ee2191f61b5 100644 --- a/docs-site/src/content/docs/guides/providers.md +++ b/docs-site/src/content/docs/guides/providers.md @@ -562,6 +562,20 @@ The preset uses [Chat Completions](https://docs.tokenlab.sh/quickstart) and disc `GET /v1/models?category=chat`, keeping only entries that declare `tool-use` capability. Image, video, audio, embedding and decision models are excluded from this chat preset. +Each model then uses the request format TokenLab declares for it +(`tokenlab.accepted_request_formats` on `GET /v1/models/{model}`): + +| Models | Codex (Responses clients) | Chat clients | Claude Code (Anthropic clients) | +| --- | --- | --- | --- | +| `gpt-6-astra`, `gpt-6.1-sol`, `gpt-6-sol`, `gpt-6-luna`, `grok-4.7`, `deepseek-v4.1-flash`, `deepseek-v4-pro`, `kimi-k3`, `glm-5.3` | [Responses](https://docs.tokenlab.sh/api-reference/responses/create-response) | Chat Completions | Chat Completions | +| `claude-*` | [Messages](https://docs.tokenlab.sh/api-reference/messages/create-message) | Messages | Messages | +| Every other model, including `gemini-3.8-flash` | Chat Completions | Chat Completions | Chat Completions | + +To keep a model on Chat Completions, add it to the provider's `modelAdapters`, for example +`"modelAdapters": { "gpt-6.1-sol": "openai-chat" }`. The Claude routing applies only while the +provider points at `https://api.tokenlab.sh/v1`. OpenCodex sends no delivery-policy header, so +your API key's own delivery policy decides how TokenLab serves each request. + The [model catalog](https://docs.tokenlab.sh/api-reference/models/list-models) is public without a key, but a supplied key is validated and scopes results to its model permissions and delivery policy. Use a valid key with a funded workspace for inference. `gpt-5.6-terra` is the seeded diff --git a/docs-site/src/content/docs/ja/guides/claude-code.md b/docs-site/src/content/docs/ja/guides/claude-code.md index 6a074ece037..b657396615f 100644 --- a/docs-site/src/content/docs/ja/guides/claude-code.md +++ b/docs-site/src/content/docs/ja/guides/claude-code.md @@ -161,7 +161,7 @@ Anthropic モデル ID なので、代わりにピッカーの行を opencodex ```bash ocx claude desktop bind claude-sonnet-4-6 xai/grok-4.7 -ocx claude desktop bind claude-opus-4-6 native/gpt-6-sol +ocx claude desktop bind claude-opus-4-6 native/gpt-6.1-sol ocx claude desktop unbind claude-opus-4-6 ``` diff --git a/docs-site/src/content/docs/ja/reference/configuration/agents.md b/docs-site/src/content/docs/ja/reference/configuration/agents.md index 52a4f13bdea..19644ea5da2 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/agents.md +++ b/docs-site/src/content/docs/ja/reference/configuration/agents.md @@ -10,7 +10,7 @@ description: マルチエージェント サーフェス、委任ガイダンス |フィールド |タイプ |デフォルト |意味 | | --- | --- | --- | --- | | `multiAgentMode?` | `"v1" \| "default" \| "v2"` | `"default"` | `v1` はすべてのカタログ モデルを v1 としてスタンプします。 `v2` はすべてのモデルを v2 としてスタンプします。 `default` はアップストリーム ピン (Sol/Terra v2、Luna v1) を復元し、それ以外の場合はネイティブの `multi_agent_v2` フラグに従います。新しいセッションに適用されます。 | -| `subagentModels?` | `string[]` | `gpt-6-astra`、`gpt-6-sol`、`gpt-6-luna` |最大 5 つの bare native id、account-qualified `/` id、または routed `provider/model` id をサブエージェント ピッカーで優先表示します。Subagents ページで選べるのは bare native id と routed id だけで、保存時には exact account-qualified の選択が除外されます。exact の選択には `ocx agent subagents set` を使用するか、設定を直接編集してください。[Astra への一度限りの移行](/reference/configuration/agents/#astra-roster-upgrade)後は、明示的な空リストも保持されます。 | +| `subagentModels?` | `string[]` | `gpt-6-astra`、`gpt-6.1-sol`、`gpt-6-luna` |最大 5 つの bare native id、account-qualified `/` id、または routed `provider/model` id をサブエージェント ピッカーで優先表示します。Subagents ページで選べるのは bare native id と routed id だけで、保存時には exact account-qualified の選択が除外されます。exact の選択には `ocx agent subagents set` を使用するか、設定を直接編集してください。[Astra への一度限りの移行](/reference/configuration/agents/#astra-roster-upgrade)後は、明示的な空リストも保持されます。 | | `injectionModel?` | `string` | — |プロキシ作成の v2 委任ガイダンスで使用される、優先されるネイティブまたはルーティングされたサブエージェント モデル。 | | `injectionEffort?` | `string` | — |優先努力 (`low` ～ `ultra`)。`injectionModel` でのみ意味があります。 | | `injectionPrompt?` | `string` | — | 組み込みの v2 ガイダンス本文を置き換えます。`{{model}}`、`{{effort}}`、`{{roster}}`、`{{fallback}}`をサポートします。`injectionModel` が設定されていればカスタムプロンプトが生成されます。 | diff --git a/docs-site/src/content/docs/ko/guides/claude-code.md b/docs-site/src/content/docs/ko/guides/claude-code.md index a6cff146937..d45ed65f471 100644 --- a/docs-site/src/content/docs/ko/guides/claude-code.md +++ b/docs-site/src/content/docs/ko/guides/claude-code.md @@ -184,7 +184,7 @@ opencodex 라우트에 묶어서 씁니다. ```bash ocx claude desktop bind claude-sonnet-4-6 xai/grok-4.7 -ocx claude desktop bind claude-opus-4-6 native/gpt-6-sol +ocx claude desktop bind claude-opus-4-6 native/gpt-6.1-sol ocx claude desktop unbind claude-opus-4-6 ``` diff --git a/docs-site/src/content/docs/ko/reference/configuration/agents.md b/docs-site/src/content/docs/ko/reference/configuration/agents.md index 768af611d5a..60a0af24f67 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/agents.md +++ b/docs-site/src/content/docs/ko/reference/configuration/agents.md @@ -10,7 +10,7 @@ description: 멀티 에이전트 표면, 위임 안내, 선호 모델, 대체 | 필드 | 형식 | 기본값 | 의미 | | --- | --- | --- | --- | | `multiAgentMode?` | `"v1" \| "default" \| "v2"` | `"default"` | `v1`은 카탈로그의 모든 모델에 v1을 표시하고, `v2`는 모든 모델에 v2를 표시합니다. `default`는 상위 고정값(Sol/Terra는 v2, Luna는 v1)을 복원하고, 그 외에는 네이티브 `multi_agent_v2` 플래그를 따릅니다. 새 세션에 적용됩니다. | -| `subagentModels?` | `string[]` | `gpt-6-astra`, `gpt-6-sol`, `gpt-6-luna` | 최대 다섯 개의 bare native id, account-qualified `/` id 또는 routed `provider/model` id를 서브에이전트 선택기에서 우선 표시합니다. Subagents 페이지는 bare native와 routed id만 제공하며 저장할 때 exact account-qualified 선택을 제외합니다. exact 선택은 `ocx agent subagents set`을 사용하거나 설정을 직접 편집하세요. [Astra 최초 업그레이드](/reference/configuration/agents/#astra-roster-upgrade) 이후에는 빈 목록도 그대로 보존됩니다. | +| `subagentModels?` | `string[]` | `gpt-6-astra`, `gpt-6.1-sol`, `gpt-6-luna` | 최대 다섯 개의 bare native id, account-qualified `/` id 또는 routed `provider/model` id를 서브에이전트 선택기에서 우선 표시합니다. Subagents 페이지는 bare native와 routed id만 제공하며 저장할 때 exact account-qualified 선택을 제외합니다. exact 선택은 `ocx agent subagents set`을 사용하거나 설정을 직접 편집하세요. [Astra 최초 업그레이드](/reference/configuration/agents/#astra-roster-upgrade) 이후에는 빈 목록도 그대로 보존됩니다. | | `injectionModel?` | `string` | — | 프록시가 작성한 v2 위임 안내에서 사용하는 선호 네이티브 또는 라우팅된 서브에이전트 모델입니다. | | `injectionEffort?` | `string` | — | 선호 노력(`low`부터 `ultra`까지)입니다. `injectionModel`이 있을 때만 의미가 있습니다. | | `injectionPrompt?` | `string` | — | 내장 v2 안내 본문을 대체합니다. `{{model}}`, `{{effort}}`, `{{roster}}`, `{{fallback}}`를 지원합니다. `injectionModel`만 설정되어 있어도 사용자 정의 프롬프트가 발동합니다. | diff --git a/docs-site/src/content/docs/reference/adapters.md b/docs-site/src/content/docs/reference/adapters.md index 390a66e0b85..1dae3494998 100644 --- a/docs-site/src/content/docs/reference/adapters.md +++ b/docs-site/src/content/docs/reference/adapters.md @@ -220,7 +220,9 @@ a fresh session ID. Recovery and cached-history replay preserve this classificat The API-key `commandcode` provider uses Chat Completions for most model ids and the Anthropic Messages adapter (`x-api-key`) for `claude-*` ids, which Command Code serves only on `/provider/v1/messages`; the pin applies only while the provider points at that -endpoint. It supports forwarding `prompt_cache_key`; this is separate +endpoint. The `tokenlab` provider uses the same endpoint-bound pin for `claude-*` ids, which +TokenLab declares for Chat and Messages only, on `https://api.tokenlab.sh/v1/messages`. +Command Code supports forwarding `prompt_cache_key`; this is separate from the OAuth adapter's session header and does not guarantee a provider cache hit. The OAuth `command-code` preset streams `/alpha/generate` as NDJSON. MiMo tool-call markup echoed by the gateway as text is removed when it duplicates a real call. Markup diff --git a/docs-site/src/content/docs/reference/configuration/agents.md b/docs-site/src/content/docs/reference/configuration/agents.md index 20165746900..0680b278d31 100644 --- a/docs-site/src/content/docs/reference/configuration/agents.md +++ b/docs-site/src/content/docs/reference/configuration/agents.md @@ -26,14 +26,19 @@ The previous default list therefore becomes Astra, Sol, Terra, Luna, 5.5. An unset legacy list receives the current defaults; an explicit empty legacy list becomes `["gpt-6-astra"]`. Existing Astra entries are not duplicated. -The current default is `gpt-6-astra`, `gpt-6-sol`, `gpt-6-luna`. On the first +The current default is `gpt-6-astra`, `gpt-6.1-sol`, `gpt-6-luna`. On the first start after upgrading, a stored list is cleaned of retired rows: `gpt-5.6-sol` and `gpt-5.6-luna` become `gpt-6-sol` and `gpt-6-luna` in the same position, and every other `gpt-5.5` or `gpt-5.6` model is removed. Ids with a `/` (routed `provider/model` or account-qualified choices) are left as written. A list that held only retired rows receives the current default; an empty list stays empty. -The internal `subagentModelsVersion` marker (currently `2`) makes each step a +GPT-6.1 Sol replaced GPT-6 Sol as the default on September 30, 2026. On the first start +after that upgrade, a bare `gpt-6-sol` in a stored list becomes `gpt-6.1-sol` in the same +position (without a duplicate if 6.1 Sol is already listed). GPT-6 Sol stays available: add it +back afterwards and it stays. + +The internal `subagentModelsVersion` marker (currently `3`) makes each step a one-time upgrade. Afterwards you can reorder, remove Astra, or save an empty list without startup changing your choices again. Disabled models remain disabled. Astra availability still depends on upstream support for your account. @@ -42,7 +47,7 @@ still depends on upstream support for your account. | --- | --- | --- | --- | | `multiAgentMode?` | `"v1" \| "default" \| "v2"` | `"default"` | `v1` stamps every catalog model as v1; `v2` stamps every model as v2. `default` restores upstream pins (Sol/Terra v2, Luna v1) and otherwise follows the native `multi_agent_v2` flag. Applies to new sessions. | | `keepNativeChatGptOnV1?` | `boolean` | `false` | When `multiAgentMode` is `"v2"`, disable the global V2 override, stamp ChatGPT-native rows as v1, and keep routed rows on v2. Codex resolves the global override before catalog pins, so both parts are required for a ChatGPT parent to spawn routed children without backend-encrypted tasks ([#92](https://github.com/lidge-jun/opencodex/issues/92)). Ignored in `v1` and `default`. | -| `subagentModels?` | `string[]` | `gpt-6-astra`, `gpt-6-sol`, `gpt-6-luna` | Up to five bare native, account-qualified `/`, or routed `provider/model` ids featured first in the sub-agent picker. The dashboard offers only bare native and routed ids and omits exact account-qualified choices when it saves; use `ocx agent subagents set` or edit the configuration for exact choices. After the [one-time Astra upgrade](#astra-roster-upgrade), an explicit empty list is preserved. | +| `subagentModels?` | `string[]` | `gpt-6-astra`, `gpt-6.1-sol`, `gpt-6-luna` | Up to five bare native, account-qualified `/`, or routed `provider/model` ids featured first in the sub-agent picker. The dashboard offers only bare native and routed ids and omits exact account-qualified choices when it saves; use `ocx agent subagents set` or edit the configuration for exact choices. After the [one-time Astra upgrade](#astra-roster-upgrade), an explicit empty list is preserved. | | `injectionModel?` | `string` | — | Preferred native or routed sub-agent model used in proxy-authored v2 delegation guidance. | | `injectionEffort?` | `string` | — | Preferred effort (`low` through `ultra`), meaningful only with `injectionModel`. | | `injectionPrompt?` | `string` | — | Replaces the built-in v2 guidance body. Supports `{{model}}`, `{{effort}}`, `{{roster}}`, and `{{fallback}}`. A configured `injectionModel` is sufficient to render the custom prompt. | diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index 745e3536ad0..e4250e11654 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -135,9 +135,21 @@ when your installed Codex catalog predates them. | `gpt-6-sol` | 272,000 | 872,000 | `medium` | `low` through `ultra` | | `gpt-6-luna` | 272,000 | 872,000 | `medium` | `low` through `max` (no `ultra`) | +[GPT-6.1 Sol](https://openai.com/index/introducing-gpt-6-1-sol/) (announced September 29, 2026) +is the upgrade to GPT-6 Sol; Astra and Luna did not get a 6.1 release. It is listed as +`gpt-6.1-sol` (**GPT-6.1-Sol**), ungated like Sol, and its row comes from the Codex catalog, where +it needs `client_version` **0.153.0 or later** and is the Codex default. GPT-6 Sol stays listed. + +| Model | Default context | Opt-in ceiling | Default effort | Reasoning ladder | +| --- | ---: | ---: | --- | --- | +| `gpt-6.1-sol` | 272,000 | 872,000 | `low` | `low` through `ultra` | + The same `providerContextCaps.openai`, `modelContextWindows` and `modelAutoCompactTokenLimits` -levers apply as for Astra. There are no `openai-apikey/` rows or built-in price estimates for Sol -or Luna yet. +levers apply as for Astra. On the OpenAI API (`openai-apikey`), `gpt-6.1-sol` has 1,050,000 +context, 922,000 maximum input, 128,000 maximum output and efforts `low` through `max`, priced +at $2 input, $0.10 cached input and $10 output per 1M tokens (prompts over 272K bill at the +long-context rate). GitHub Copilot, OpenRouter, Vercel AI Gateway, Kilo and OpenCode Zen also +list it. When OpenAI ships a GPT model that this release does not know yet, add it through config instead of waiting for an update, the same way a new Claude id goes under `providers.anthropic.models`: @@ -156,7 +168,7 @@ waiting for an update, the same way a new Claude id goes under `providers.anthro ``` Each bare `gpt-*` id listed there on the Codex-login provider appears as a native model (here -**GPT-6-Nova**) with GPT-6 Sol's reasoning ladder and modalities, a 272,000-token default context +**GPT-6-Nova**) with GPT-6.1 Sol's reasoning ladder, default effort and modalities, a 272,000-token default context and an 872,000-token opt-in ceiling. Raise or narrow it with `modelContextWindows`, for example `"modelContextWindows": { "gpt-6-nova": 872000 }`. It is never account-gated: if your account cannot use the model, the request still goes out and you see the upstream error. Ids that are diff --git a/docs-site/src/content/docs/ru/guides/claude-code.md b/docs-site/src/content/docs/ru/guides/claude-code.md index bd8f7886319..7df5ba2bbec 100644 --- a/docs-site/src/content/docs/ru/guides/claude-code.md +++ b/docs-site/src/content/docs/ru/guides/claude-code.md @@ -193,7 +193,7 @@ Sonnet 5, Haiku 4.5 и более старые модели под **More models ```bash ocx claude desktop bind claude-sonnet-4-6 xai/grok-4.7 -ocx claude desktop bind claude-opus-4-6 native/gpt-6-sol +ocx claude desktop bind claude-opus-4-6 native/gpt-6.1-sol ocx claude desktop unbind claude-opus-4-6 ``` diff --git a/docs-site/src/content/docs/ru/reference/configuration/agents.md b/docs-site/src/content/docs/ru/reference/configuration/agents.md index d3deef428cb..6b0afa7d7fe 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/agents.md +++ b/docs-site/src/content/docs/ru/reference/configuration/agents.md @@ -11,7 +11,7 @@ description: Multi-agent surface, guidance при делегировании, pr | Поле | Тип | По умолчанию | Значение | | --- | --- | --- | --- | | `multiAgentMode?` | `"v1" \| "default" \| "v2"` | `"default"` | `v1` штампует все модели как v1; `v2` штампует все модели как v2. `default` восстанавливает upstream pin'ы (Sol/Terra — v2, Luna — v1) и для остальных следует native flag `multi_agent_v2`. Применяется к новым сессиям. | -| `subagentModels?` | `string[]` | `gpt-6-astra`, `gpt-6-sol`, `gpt-6-luna` | До пяти bare native-id, account-qualified id `/` или routed-id `provider/model`, которые показываются первыми в picker'е подагентов. Страница Subagents предлагает только bare native- и routed-id и при сохранении исключает точные account-qualified варианты; для точного выбора используйте `ocx agent subagents set` или отредактируйте конфигурацию. После [однократного обновления Astra](/reference/configuration/agents/#astra-roster-upgrade) явный пустой список сохраняется. | +| `subagentModels?` | `string[]` | `gpt-6-astra`, `gpt-6.1-sol`, `gpt-6-luna` | До пяти bare native-id, account-qualified id `/` или routed-id `provider/model`, которые показываются первыми в picker'е подагентов. Страница Subagents предлагает только bare native- и routed-id и при сохранении исключает точные account-qualified варианты; для точного выбора используйте `ocx agent subagents set` или отредактируйте конфигурацию. После [однократного обновления Astra](/reference/configuration/agents/#astra-roster-upgrade) явный пустой список сохраняется. | | `injectionModel?` | `string` | — | Предпочитаемая native- или routed-модель подагента, которую proxy использует в собственном guidance v2. | | `injectionEffort?` | `string` | — | Предпочитаемый effort (`low`–`ultra`), имеющий смысл только вместе с `injectionModel`. | | `injectionPrompt?` | `string` | — | Заменяет встроенное тело guidance для v2. Поддерживает `{{model}}`, `{{effort}}`, `{{roster}}` и `{{fallback}}`. Настроенного `injectionModel` достаточно, чтобы отобразить пользовательский prompt. | diff --git a/docs-site/src/content/docs/tr/guides/claude-code.md b/docs-site/src/content/docs/tr/guides/claude-code.md index 5b933a74709..e88cd7f0ac7 100644 --- a/docs-site/src/content/docs/tr/guides/claude-code.md +++ b/docs-site/src/content/docs/tr/guides/claude-code.md @@ -271,7 +271,7 @@ kimliğidir; bu yüzden bir seçici satırını bir opencodex rotasına bağlars ```bash ocx claude desktop bind claude-sonnet-4-6 xai/grok-4.7 -ocx claude desktop bind claude-opus-4-6 native/gpt-6-sol +ocx claude desktop bind claude-opus-4-6 native/gpt-6.1-sol ocx claude desktop unbind claude-opus-4-6 ``` diff --git a/docs-site/src/content/docs/tr/reference/configuration/agents.md b/docs-site/src/content/docs/tr/reference/configuration/agents.md index 81b2e600135..5fec449caa1 100644 --- a/docs-site/src/content/docs/tr/reference/configuration/agents.md +++ b/docs-site/src/content/docs/tr/reference/configuration/agents.md @@ -12,7 +12,7 @@ kontrol eder. | Alan | Tip | Varsayılan | Anlamı | | --- | --- | --- | --- | | `multiAgentMode?` | `"v1" \| "default" \| "v2"` | `"default"` | `v1` her katalog modelini v1 olarak damgalar; `v2` her modeli v2 olarak damgalar. `default` yukarı akış sabitlemelerini geri yükler (Sol/Terra v2, Luna v1) ve aksi takdirde yerel `multi_agent_v2` bayrağını takip eder. Yeni oturumlara uygulanır. | -| `subagentModels?` | `string[]` | `gpt-6-astra`, `gpt-6-sol`, `gpt-6-luna` | Alt ajan seçicisinde ilk olarak öne çıkan en fazla beş yalın yerel, hesap nitelikli `/` veya yönlendirilen `saglayici/model` kimliği. Kontrol paneli yalnızca yalın yerel ve yönlendirilen kimlikleri sunar ve kaydederken tam hesap nitelikli seçimleri atlar; tam seçimler için `ocx agent subagents set` kullanın veya yapılandırmayı düzenleyin. [Tek seferlik Astra yükseltmesinden](/reference/configuration/agents/#astra-roster-upgrade) sonra açık bir boş liste korunur. | +| `subagentModels?` | `string[]` | `gpt-6-astra`, `gpt-6.1-sol`, `gpt-6-luna` | Alt ajan seçicisinde ilk olarak öne çıkan en fazla beş yalın yerel, hesap nitelikli `/` veya yönlendirilen `saglayici/model` kimliği. Kontrol paneli yalnızca yalın yerel ve yönlendirilen kimlikleri sunar ve kaydederken tam hesap nitelikli seçimleri atlar; tam seçimler için `ocx agent subagents set` kullanın veya yapılandırmayı düzenleyin. [Tek seferlik Astra yükseltmesinden](/reference/configuration/agents/#astra-roster-upgrade) sonra açık bir boş liste korunur. | | `injectionModel?` | `string` | — | Proxy kaynaklı v2 yetkilendirme rehberliğinde kullanılan tercih edilen yerel veya yönlendirilen alt ajan modeli. | | `injectionEffort?` | `string` | — | Yalnızca `injectionModel` ile anlamlı olan tercih edilen çaba (`low` ile `ultra` arası). | | `injectionPrompt?` | `string` | — | Yerleşik v2 rehberlik gövdesinin yerini alır. `{{model}}`, `{{effort}}`, `{{roster}}` ve `{{fallback}}` destekler. Yapılandırılmış bir `injectionModel`, özel istemi oluşturmak için yeterlidir. | diff --git a/docs-site/src/content/docs/zh-cn/guides/claude-code.md b/docs-site/src/content/docs/zh-cn/guides/claude-code.md index c600581e8d0..1695a8a6ad7 100644 --- a/docs-site/src/content/docs/zh-cn/guides/claude-code.md +++ b/docs-site/src/content/docs/zh-cn/guides/claude-code.md @@ -171,7 +171,7 @@ opencodex 路由： ```bash ocx claude desktop bind claude-sonnet-4-6 xai/grok-4.7 -ocx claude desktop bind claude-opus-4-6 native/gpt-6-sol +ocx claude desktop bind claude-opus-4-6 native/gpt-6.1-sol ocx claude desktop unbind claude-opus-4-6 ``` diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/agents.md b/docs-site/src/content/docs/zh-cn/reference/configuration/agents.md index fcbfe7585ae..32b76f3bb5c 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/agents.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/agents.md @@ -10,7 +10,7 @@ description: 多代理界面、委派引导、首选模型、回退链、原生 | 字段 | 类型 | 默认值 | 含义 | | --- | --- | --- | --- | | `multiAgentMode?` | `"v1" \| "default" \| "v2"` | `"default"` | `v1` 会把目录中的每个模型都标记为 v1；`v2` 会把每个模型都标记为 v2。`default` 会恢复上游固定值（Sol/Terra 为 v2，Luna 为 v1），否则遵循原生 `multi_agent_v2` 标志。适用于新会话。 | -| `subagentModels?` | `string[]` | `gpt-6-astra`, `gpt-6-sol`, `gpt-6-luna` | 最多五个裸原生 id、账户限定的 `/` id 或路由 `provider/model` id 会优先显示在子代理选择器中。Subagents 页面只提供裸原生和路由 id，保存时会省略精确的账户限定选项；如需精确选择，请使用 `ocx agent subagents set` 或直接编辑配置。[Astra 一次性升级](/reference/configuration/agents/#astra-roster-upgrade)后，显式空列表会被保留。 | +| `subagentModels?` | `string[]` | `gpt-6-astra`, `gpt-6.1-sol`, `gpt-6-luna` | 最多五个裸原生 id、账户限定的 `/` id 或路由 `provider/model` id 会优先显示在子代理选择器中。Subagents 页面只提供裸原生和路由 id，保存时会省略精确的账户限定选项；如需精确选择，请使用 `ocx agent subagents set` 或直接编辑配置。[Astra 一次性升级](/reference/configuration/agents/#astra-roster-upgrade)后，显式空列表会被保留。 | | `injectionModel?` | `string` | — | 在代理生成的 v2 委派引导中使用的首选原生或路由后的子代理模型。 | | `injectionEffort?` | `string` | — | 首选 effort（`low` 到 `ultra`），只有在 `injectionModel` 存在时才有意义。 | | `injectionPrompt?` | `string` | — | 替换内置 v2 指引正文。支持 `{{model}}`、`{{effort}}`、`{{roster}}` 和 `{{fallback}}`。只要配置了 `injectionModel`，自定义提示词就会触发。 | diff --git a/docs-site/src/content/docs/zh-tw/guides/claude-code.md b/docs-site/src/content/docs/zh-tw/guides/claude-code.md index 5206d7066e5..3d8e58d7f55 100644 --- a/docs-site/src/content/docs/zh-tw/guides/claude-code.md +++ b/docs-site/src/content/docs/zh-tw/guides/claude-code.md @@ -260,7 +260,7 @@ opencodex 路由： ```bash ocx claude desktop bind claude-sonnet-4-6 xai/grok-4.7 -ocx claude desktop bind claude-opus-4-6 native/gpt-6-sol +ocx claude desktop bind claude-opus-4-6 native/gpt-6.1-sol ocx claude desktop unbind claude-opus-4-6 ``` diff --git a/docs-site/src/content/docs/zh-tw/reference/configuration/agents.md b/docs-site/src/content/docs/zh-tw/reference/configuration/agents.md index 314530a4454..6f417dffa89 100644 --- a/docs-site/src/content/docs/zh-tw/reference/configuration/agents.md +++ b/docs-site/src/content/docs/zh-tw/reference/configuration/agents.md @@ -10,7 +10,7 @@ Agent 設定控制要廣告哪個 Codex 協作介面，以及 opencodex 如何 | 欄位 | 型別 | 預設值 | 意義 | | --- | --- | --- | --- | | `multiAgentMode?` | `"v1" \| "default" \| "v2"` | `"default"` | `v1` 將每個目錄模型標記為 v1；`v2` 將每個模型標記為 v2。`default` 還原上游 pin（Sol/Terra v2、Luna v1），否則遵循原生的 `multi_agent_v2` 旗標。套用於新 session。 | -| `subagentModels?` | `string[]` | `gpt-6-astra`, `gpt-6-sol`, `gpt-6-luna` | 最多五個原生或路由 id，在子代理 picker 中優先顯示。[Astra 一次性升級](/reference/configuration/agents/#astra-roster-upgrade)後，明確的空清單會被保留。 | +| `subagentModels?` | `string[]` | `gpt-6-astra`, `gpt-6.1-sol`, `gpt-6-luna` | 最多五個原生或路由 id，在子代理 picker 中優先顯示。[Astra 一次性升級](/reference/configuration/agents/#astra-roster-upgrade)後，明確的空清單會被保留。 | | `injectionModel?` | `string` | — | 在代理撰寫的 v2 委派指引中使用的偏好原生或路由子代理模型。 | | `injectionEffort?` | `string` | — | 偏好 effort（`low` 到 `ultra`），僅在搭配 `injectionModel` 時有意義。 | | `injectionPrompt?` | `string` | — | 取代內建指引本文。支援 `{{model}}`、`{{effort}}`、`{{roster}}` 與 `{{fallback}}`。觸發閘門保持不變。 | diff --git a/scripts/ci/docker-smoke.ts b/scripts/ci/docker-smoke.ts index e108d129e42..a8424f93986 100644 --- a/scripts/ci/docker-smoke.ts +++ b/scripts/ci/docker-smoke.ts @@ -224,8 +224,8 @@ const stateProbe = ` const additions = { appOwnedMemoryBudgetMb: 256, fastRows: true, managementUsageMaxReadBytes: 67108864, openaiProviderTierVersion: 2, - subagentModels: ['gpt-6-astra', 'gpt-6-sol', 'gpt-6-luna'], - subagentModelsVersion: 2, + subagentModels: ['gpt-6-astra', 'gpt-6.1-sol', 'gpt-6-luna'], + subagentModelsVersion: 3, }; for (const config of [persisted, loaded]) { if (Object.keys(config).some(key => !Object.hasOwn(seed, key) && !Object.hasOwn(additions, key))) throw new Error('unexpected startup config addition'); diff --git a/scripts/model-metadata.source.json b/scripts/model-metadata.source.json index cfa750e50a1..53ad2fc6556 100644 --- a/scripts/model-metadata.source.json +++ b/scripts/model-metadata.source.json @@ -2885,6 +2885,31 @@ "maxLevel": "max" } }, + "openai.gpt-6.1-sol": { + "id": "openai.gpt-6.1-sol", + "name": "GPT-6.1 Sol", + "api": "bedrock-converse-stream", + "provider": "amazon-bedrock", + "baseUrl": "https://bedrock-runtime.us-east-1.amazonaws.com", + "reasoning": true, + "input": [ + "text", + "image" + ], + "cost": { + "input": 2, + "output": 10, + "cacheRead": 0.1, + "cacheWrite": 2.5 + }, + "contextWindow": 373000, + "maxTokens": 128000, + "thinking": { + "mode": "budget", + "minLevel": "low", + "maxLevel": "max" + } + }, "openai.gpt-5.6-terra": { "id": "openai.gpt-5.6-terra", "name": "GPT-5.6 Terra", @@ -5870,6 +5895,31 @@ "maxLevel": "max" } }, + "openai/gpt-6.1-sol": { + "id": "openai/gpt-6.1-sol", + "name": "GPT-6.1 Sol", + "api": "anthropic-messages", + "provider": "cloudflare-ai-gateway", + "baseUrl": "https://gateway.ai.cloudflare.com/v1///anthropic", + "reasoning": true, + "input": [ + "text", + "image" + ], + "cost": { + "input": 2, + "output": 10, + "cacheRead": 0.1, + "cacheWrite": 0 + }, + "contextWindow": 373000, + "maxTokens": 128000, + "thinking": { + "mode": "budget", + "minLevel": "low", + "maxLevel": "max" + } + }, "openai/gpt-5.6-terra": { "id": "openai/gpt-5.6-terra", "name": "GPT-5.6 Terra", @@ -11435,6 +11485,34 @@ "maxLevel": "max" } }, + "gpt-6.1-sol": { + "id": "gpt-6.1-sol", + "name": "GPT-6.1 Sol", + "api": "openai-responses", + "provider": "github-copilot", + "baseUrl": "https://api.githubcopilot.com", + "reasoning": true, + "input": [ + "text", + "image" + ], + "cost": { + "input": 2, + "output": 10, + "cacheRead": 0.1, + "cacheWrite": 2.5 + }, + "contextWindow": 373000, + "maxTokens": 128000, + "headers": { + "User-Agent": "opencode/1.3.15" + }, + "thinking": { + "mode": "effort", + "minLevel": "low", + "maxLevel": "max" + } + }, "gpt-5.6-terra": { "id": "gpt-5.6-terra", "name": "GPT-5.6 Terra", @@ -20930,6 +21008,31 @@ "maxLevel": "max" } }, + "openai/gpt-6.1-sol": { + "id": "openai/gpt-6.1-sol", + "name": "GPT-6.1 Sol", + "api": "openai-completions", + "provider": "kilo", + "baseUrl": "https://api.kilo.ai/api/gateway", + "reasoning": true, + "input": [ + "text", + "image" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 373000, + "maxTokens": 128000, + "thinking": { + "mode": "effort", + "minLevel": "low", + "maxLevel": "max" + } + }, "openai/gpt-5.6-sol-pro": { "id": "openai/gpt-5.6-sol-pro", "name": "OpenAI: GPT-5.6 Sol Pro", @@ -60134,6 +60237,32 @@ "maxLevel": "max" } }, + "gpt-6.1-sol": { + "id": "gpt-6.1-sol", + "name": "GPT-6.1 Sol", + "cost": { + "input": 2, + "output": 10, + "cacheRead": 0.1, + "cacheWrite": 2.5 + }, + "api": "openai-responses", + "provider": "openai", + "baseUrl": "", + "reasoning": true, + "input": [ + "text", + "image" + ], + "contextWindow": 373000, + "maxTokens": 128000, + "applyPatchToolType": "freeform", + "thinking": { + "mode": "effort", + "minLevel": "low", + "maxLevel": "max" + } + }, "gpt-5.6-terra": { "id": "gpt-5.6-terra", "name": "GPT-5.6 Terra", @@ -60943,6 +61072,33 @@ "maxLevel": "max" } }, + "gpt-6.1-sol": { + "id": "gpt-6.1-sol", + "name": "GPT-6.1 Sol", + "cost": { + "input": 2, + "output": 10, + "cacheRead": 0.1, + "cacheWrite": 2.5 + }, + "api": "openai-codex-responses", + "provider": "openai-codex", + "baseUrl": "https://chatgpt.com/backend-api", + "reasoning": true, + "input": [ + "text", + "image" + ], + "contextWindow": 373000, + "maxTokens": 128000, + "preferWebsockets": true, + "applyPatchToolType": "freeform", + "thinking": { + "mode": "effort", + "minLevel": "low", + "maxLevel": "max" + } + }, "gpt-5.6-terra": { "id": "gpt-5.6-terra", "name": "GPT-5.6 Terra", @@ -63249,6 +63405,31 @@ "maxLevel": "max" } }, + "gpt-6.1-sol": { + "id": "gpt-6.1-sol", + "name": "GPT-6.1 Sol", + "api": "openai-responses", + "provider": "opencode-zen", + "baseUrl": "https://opencode.ai/zen/v1", + "reasoning": true, + "input": [ + "text", + "image" + ], + "cost": { + "input": 2, + "output": 10, + "cacheRead": 0.2, + "cacheWrite": 2.5 + }, + "contextWindow": 373000, + "maxTokens": 128000, + "thinking": { + "mode": "effort", + "minLevel": "low", + "maxLevel": "max" + } + }, "gpt-5.6-terra": { "id": "gpt-5.6-terra", "name": "GPT-5.6 Terra", @@ -69393,6 +69574,31 @@ "maxLevel": "max" } }, + "openai/gpt-6.1-sol": { + "id": "openai/gpt-6.1-sol", + "name": "GPT-6.1 Sol", + "api": "openai-completions", + "provider": "openrouter", + "baseUrl": "https://openrouter.ai/api/v1", + "reasoning": true, + "input": [ + "text", + "image" + ], + "cost": { + "input": 2, + "output": 10, + "cacheRead": 0.1, + "cacheWrite": 2.5 + }, + "contextWindow": 373000, + "maxTokens": 128000, + "thinking": { + "mode": "effort", + "minLevel": "low", + "maxLevel": "max" + } + }, "openai/gpt-5.6-sol-pro": { "id": "openai/gpt-5.6-sol-pro", "name": "OpenAI: GPT-5.6 Sol Pro", @@ -80831,6 +81037,31 @@ "maxLevel": "max" } }, + "openai/gpt-6.1-sol": { + "id": "openai/gpt-6.1-sol", + "name": "GPT-6.1 Sol", + "api": "anthropic-messages", + "provider": "vercel-ai-gateway", + "baseUrl": "https://ai-gateway.vercel.sh", + "reasoning": true, + "input": [ + "text", + "image" + ], + "cost": { + "input": 2, + "output": 10, + "cacheRead": 0.1, + "cacheWrite": 2.5 + }, + "contextWindow": 373000, + "maxTokens": 128000, + "thinking": { + "mode": "budget", + "minLevel": "low", + "maxLevel": "max" + } + }, "openai/gpt-5.6-terra": { "id": "openai/gpt-5.6-terra", "name": "GPT-5.6 Terra", diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 794895ba674..3fc00931e08 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -872,6 +872,7 @@ "google-wire-compiler.test.ts": "adapters/google", "google-wire-shape.test.ts": "adapters/google", "gpt6-native-rows.test.ts": "codex-integration", + "gpt61-sol-rows.test.ts": "codex-integration", "grok-attribution.test.ts": "providers/xai", "grok-config-inject.test.ts": "providers/xai", "grok-effort-inject.test.ts": "providers/xai", @@ -1237,6 +1238,7 @@ "openrouter-provider-routing.test.ts": "providers", "openrouter-quota-reset-cooldown-4024.test.ts": "providers", "opper-provider.test.ts": "providers", + "tokenlab-protocols.test.ts": "providers", "tokenlab-provider.test.ts": "providers", "optional-shutdown-hooks.test.ts": "lib", "orca-import.test.ts": "codex-integration", diff --git a/src/adapters/devin/live-models.ts b/src/adapters/devin/live-models.ts index cb37c6e39e5..5b72bd56ce2 100644 --- a/src/adapters/devin/live-models.ts +++ b/src/adapters/devin/live-models.ts @@ -24,6 +24,9 @@ export const DEVIN_STATIC_MODELS = [ // 260923 preemptive: GPT-6 Sol and Luna (OpenAI announced 2026-09-22) added ahead of this provider's own catalog; mirrors the GPT-5.6 Sol/Luna rows. "gpt-6-sol", "gpt-6-luna", + // 260930 preemptive: GPT-6.1 Sol (devin.ai/blog/gpt-6-1-sol says it is live; the uid is not published). + // Spelled the way Devin spells gpt-5.6-sol; live discovery replaces this seed once a credential is present. + "gpt-6-1-sol", "claude-opus-4-8", "claude-fable-5-1", "claude-sonnet-5", @@ -61,6 +64,8 @@ export const DEVIN_MODEL_CONTEXT_WINDOWS: Record = { "gpt-6-astra": 1_000_000, "gpt-6-sol": 1_000_000, "gpt-6-luna": 1_000_000, + // 260930 preemptive: unmeasured; mirrors the GPT-6 Sol row Cognition serves. + "gpt-6-1-sol": 1_000_000, "claude-opus-4-8": 1_000_000, // 260923: read from the live catalog (devin/claude-opus-5-5 context_length 1_000_000). "claude-opus-5-5": 1_000_000, diff --git a/src/adapters/kiro/reasoning.ts b/src/adapters/kiro/reasoning.ts index 6906d2fbe25..55a9c1ca192 100644 --- a/src/adapters/kiro/reasoning.ts +++ b/src/adapters/kiro/reasoning.ts @@ -26,6 +26,8 @@ export const KIRO_NATIVE_EFFORT_FIELDS: Record= 0.155.0, so an installed catalog built by an older client lacks them. // Astra Minor is deliberately absent: it is gated, and nativeOpenAiSlugs() would drop it anyway. NATIVE_GPT6_SOL_MODEL, NATIVE_GPT6_LUNA_MODEL, + // GPT-6.1 Sol needs client_version >= 0.153.0 upstream; older installed catalogs lack it. + NATIVE_GPT61_SOL_MODEL, ]; export function configuredNativeAliasSlugs( @@ -196,6 +200,8 @@ export const NATIVE_OPENAI_CONTEXT_OVERRIDES: Record