diff --git a/docs-site/src/content/docs/reference/adapters.md b/docs-site/src/content/docs/reference/adapters.md index 5f15be73148..5045e6624fa 100644 --- a/docs-site/src/content/docs/reference/adapters.md +++ b/docs-site/src/content/docs/reference/adapters.md @@ -556,8 +556,15 @@ configuration that names the old id is rewritten at startup. stream. Cognition enforces a per-tool-description length limit (6,998 chars) and an exact-phrase blocklist; the adapter sanitizes known triggers and truncates over-long descriptions before encoding. -- Devin/Cognition API keys do not refresh. Run `ocx login devin` again when the key expires or is - revoked. +- Devin/Cognition API keys have no refresh endpoint. If Cognition rejects a stored key with 401, + OpenCodex marks that account for reauthentication and can use another signed-in account for + the turn. Run `ocx login devin` again for a revoked browser-login key. A CLI-imported account + can follow a later `devin auth login` key rotation only when its stored account ID or email + matches the minted CLI identity. Imports without a stored identity require an explicit + `ocx login devin` after rotation. A legacy `devin-cli` slot for that same account may already + hold the new key; a different account holding it blocks adoption. If the CLI file is + temporarily unreadable or the identity check is unavailable, retry after it recovers. A paused account stays paused + during this recovery and returns 403. - Only the credential is local when the CLI import path is used. The turn itself goes to Cognition either way, so the import and browser login paths differ in nothing but where the credential came from. Install the CLI with diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index da0966e7b43..1afa60781b3 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -34,6 +34,7 @@ "start-args.test.ts": "cli", "start-ownership-publication.test.ts": "cli", "responses-core-modules.test.ts": "responses", + "responses-devin-401-replay.test.ts": "responses", "responses-grok-devin-preflight.test.ts": "responses", "responses-passthrough-transient-policy.test.ts": "responses", "responses-spend-ledger-wiring.test.ts": "responses", diff --git a/src/oauth/devin.ts b/src/oauth/devin.ts index 75d9c68ce0f..53c82d7da3a 100644 --- a/src/oauth/devin.ts +++ b/src/oauth/devin.ts @@ -19,7 +19,8 @@ import { DEFAULT_REGION, type WindsurfRegion } from "./devin/types"; import { registerUser } from "./devin/register-user"; import { DEVIN_DEFAULT_API_SERVER, resolveDevinApiBaseUrl, validateDevinApiBaseUrl } from "./devin/api-base"; import { readDevinCliCredentialOutcome } from "./devin/cli-import"; -import { getCredential, listAccounts } from "./store"; +import { CloudAuthError, mintUserJwt } from "../adapters/devin/cloud-direct/auth"; +import { getCredential, listAccounts, type AuthStore } from "./store"; import { DEPRECATED_OAUTH_PROVIDER_ALIASES } from "./index"; export { DEVIN_DEFAULT_API_SERVER } from "./devin/api-base"; @@ -263,14 +264,143 @@ export async function loginDevin( return loginDevinBrowser(ctrl, DEFAULT_REGION); } +/** + * A CLI-imported account follows the CLI: after `devin auth login` rewrites the + * credential file, the copy stored at import time is stale while the file holds + * a live key. The session JWT carries no expiry, so the upstream 401 is the only + * signal, and this re-read runs only on that forced refresh. + * + * Adoption fails closed on identity. The session token carries only a + * session_id, so the file key's account is established by minting a user_jwt + * with it (GetUserJwt answers with auth_uid and email). A key Cognition refuses + * (401/403) or whose token lacks auth_uid is refused; a mint that fails for any + * other reason throws a non-terminal error, so the account is not flagged and + * the next 401 retries. The minted identity must then + * not contradict the slot's recorded accountId or email, and no other stored + * account may own the key or that identity. + * + * A CLI-imported slot has no bound identity. It cannot adopt a changed key; + * explicit `ocx login devin` is required after its first rotation. + */ +/** An identity probe must not hold the per-account refresh lock for the mint's full 30s. */ +const DEVIN_IDENTITY_MINT_TIMEOUT_MS = 5_000; + +/** Neither a live key nor a dead one: the refresh must fail without flagging the account. */ +class DevinIdentityProbeUnavailableError extends Error { + constructor() { + super("Could not confirm the Devin CLI session identity right now; retry shortly."); + this.name = "DevinIdentityProbeUnavailableError"; + } +} + +const normalizedEmail = (value: string | undefined): string | undefined => + value?.trim().toLowerCase() || undefined; + +const devinMintedIdentities = new WeakMap>(); + +/** Recheck the minted identity and key against the locked, freshly read store. */ +export function assertDevinCliAdoptionOwnership( + store: AuthStore, + provider: string, + accountId: string, + credential: OAuthCredentials, +): void { + const mintedIds = devinMintedIdentities.get(credential); + if (!mintedIds) return; + const email = normalizedEmail(credential.email); + for (const slot of [provider, ...devinAliasCredentialSlots(provider)]) { + for (const row of store[slot]?.accounts ?? []) { + // A legacy alias can still hold the same account id during rekey. + if (row.id === accountId) continue; + if (row.credential.access === credential.access + || (row.credential.accountId !== undefined && mintedIds.has(row.credential.accountId)) + || (email !== undefined && normalizedEmail(row.credential.email) === email)) { + throw new DevinIdentityProbeUnavailableError(); + } + } + } +} + +async function rereadDevinCliCredential( + stored: OAuthCredentials, + signal: AbortSignal | undefined, + currentAccountId: string | undefined, +): Promise { + const outcome = readDevinCliCredentialOutcome(); + // A file that exists but cannot be read or parsed may be mid-write by `devin auth login` + // or briefly locked; like a failed identity mint, that says nothing about the key. + if (outcome.kind === "unreadable" || outcome.kind === "incomplete") throw new DevinIdentityProbeUnavailableError(); + if (outcome.kind !== "ok" || outcome.file.apiKey === stored.access) return undefined; + // Without a stored identity, the CLI file may now belong to another user. + if (!stored.accountId && !normalizedEmail(stored.email)?.includes("@")) return undefined; + const apiBaseUrl = validateDevinApiBaseUrl(outcome.file.apiServerUrl); + if (apiBaseUrl === undefined) return undefined; + // A detached alias rekey can already hold this account's new key. Only a + // different account's key is a conflict; the locked write checks again. + for (const slot of ["devin", ...devinAliasCredentialSlots("devin")]) { + if (listAccounts(slot).some(({ id, credential }) => + id !== currentAccountId && credential.access === outcome.file.apiKey)) return undefined; + } + let minted: Record | undefined; + try { + const timeout = AbortSignal.timeout(DEVIN_IDENTITY_MINT_TIMEOUT_MS); + const probeSignal = signal ? AbortSignal.any([signal, timeout]) : timeout; + minted = decodeJwtPayload((await mintUserJwt(outcome.file.apiKey, apiBaseUrl, probeSignal)).jwt); + } catch (error) { + // Only Cognition refusing the key is evidence the file holds no live session. A timeout, + // DNS failure, 5xx or 429 says nothing about the key; flagging the account on one would + // strand it, because a needsReauth slot is never refreshed again. + if (error instanceof CloudAuthError && (error.status === 401 || error.status === 403)) return undefined; + throw new DevinIdentityProbeUnavailableError(); + } + const authUid = typeof minted?.auth_uid === "string" && minted.auth_uid ? minted.auth_uid : undefined; + if (authUid === undefined) return undefined; + // identityFromApiKey records `sub ?? auth_uid`, so a stored id may be either claim. + const mintedIds = new Set([authUid, ...(typeof minted?.sub === "string" && minted.sub ? [minted.sub] : [])]); + const rawEmail = typeof minted?.email === "string" ? minted.email.trim() : ""; + const email = rawEmail || undefined; + if (stored.accountId && !mintedIds.has(stored.accountId)) return undefined; + const storedEmail = normalizedEmail(stored.email); + if (storedEmail?.includes("@") && storedEmail !== normalizedEmail(email)) return undefined; + if (devinIdentityOwnedElsewhere(stored, currentAccountId, mintedIds, normalizedEmail(email))) return undefined; + const adopted = { ...credentialsFromApiKey(outcome.file.apiKey, apiBaseUrl, "local-cli"), accountId: authUid, ...(email ? { email } : {}) }; + devinMintedIdentities.set(adopted, mintedIds); + return adopted; +} + +/** + * Every stored Devin row except the one being refreshed, across the alias-linked slots + * (the active row `getCredential` reads is one of these). Rows are skipped by id; only a + * caller that cannot name the row falls back to matching its key. + */ +function devinIdentityOwnedElsewhere( + stored: OAuthCredentials, + currentAccountId: string | undefined, + mintedIds: ReadonlySet, + email: string | undefined, +): boolean { + for (const slot of ["devin", ...devinAliasCredentialSlots("devin")]) { + for (const { id, credential } of listAccounts(slot)) { + if (currentAccountId !== undefined ? id === currentAccountId : credential.access === stored.access) continue; + if (credential.accountId !== undefined && mintedIds.has(credential.accountId)) return true; + if (email && normalizedEmail(credential.email) === email) return true; + } + } + return false; +} + export async function refreshDevinToken( _refreshToken: string, - _signal?: AbortSignal, - _credential?: OAuthCredentials, + signal?: AbortSignal, + credential?: OAuthCredentials, + accountId?: string, ): Promise { + const reread = credential?.source === "local-cli" + ? await rereadDevinCliCredential(credential, signal, accountId) : undefined; + if (reread) return reread; // Cognition has no refresh endpoint. Extending the stored expiry here is what // the carried implementation did, and it makes a revoked key look valid - // forever. Throwing lets the request path mark the account needsReauth the - // first time a forced refresh happens. + // forever. Throwing on the upstream-401 forced refresh is what marks the + // account needsReauth. throw new Error("invalid_grant: Devin API keys do not refresh. Run ocx login devin again."); } diff --git a/src/oauth/index.ts b/src/oauth/index.ts index 196283b0953..18249758a53 100644 --- a/src/oauth/index.ts +++ b/src/oauth/index.ts @@ -30,6 +30,7 @@ import { type OAuthCredentialWriteReceipt, type OAuthRefreshIntent, type OAuthRefreshIntentCleanupPending, + type AuthStore, } from "./store"; import { loginXai, refreshXaiToken, XAI_LOCAL_CLI_DETACH_WARNING, XaiTokenRequestError } from "./xai"; import { ANTHROPIC_OAUTH_BETA, AnthropicTokenError, loginAnthropic, refreshAnthropicToken } from "./anthropic"; @@ -38,7 +39,7 @@ import { loginNous, NousTokenError, refreshNousToken, clearNousRefreshIntent, Re import { loginChatGPT, refreshChatGPTToken, type ChatGPTLoginFlow } from "./chatgpt"; import { loginAntigravity, refreshAntigravityToken } from "./google-antigravity"; import { loginCursor, refreshCursorToken } from "./cursor"; -import { loginDevin, refreshDevinToken } from "./devin"; +import { assertDevinCliAdoptionOwnership, loginDevin, refreshDevinToken } from "./devin"; import { validateDevinApiBaseUrl } from "./devin/api-base"; import { loginGithubCopilot, refreshGithubCopilotToken, validateCopilotApiBaseUrl } from "./github-copilot"; import { loginCommandCode, refreshCommandCodeToken } from "./command-code"; @@ -192,6 +193,8 @@ interface OAuthProviderDef { refreshToken: string, signal?: AbortSignal, credential?: OAuthCredentials, + /** Store row being refreshed; passed by the generic lock only. */ + accountId?: string, ): Promise; /** provider entry written into config.json on first login. */ providerConfig: OcxProviderConfig; @@ -640,6 +643,7 @@ const FORCE_REFRESH_PROVIDERS = new Set([ "kiro", "google-antigravity", "orcarouter-oauth", + "devin", ]); export async function forceRefreshOAuthAccessSnapshot( @@ -848,7 +852,11 @@ function authoritative(stored:OAuthCredentials,active:boolean,now:()=>number):OA function merged(fresh: OAuthCredentials, previous: OAuthCredentials): OAuthCredentials { return { ...fresh, - source: previous.source === "local-cli" ? "oauth" : fresh.source ?? previous.source ?? "oauth", + // Shared: a refresh function returns "local-cli" only when the credential it hands back + // still is the local CLI's (Devin re-reading the CLI file, Meta Muse echoing its durable + // CLI key). Relabelling that "oauth" would stop the next forced refresh from re-reading it. + source: fresh.source === "local-cli" ? "local-cli" + : previous.source === "local-cli" ? "oauth" : fresh.source ?? previous.source ?? "oauth", ...(fresh.projectId === undefined && previous.projectId ? { projectId: previous.projectId } : {}), ...(fresh.apiBaseUrl === undefined && previous.apiBaseUrl ? { apiBaseUrl: previous.apiBaseUrl } : {}), ...(fresh.email === undefined && previous.email ? { email: previous.email } : {}), @@ -1052,10 +1060,15 @@ export async function refreshGenericAccountWithLock( } const generation = credentialGeneration(stored); try { - const fresh = merged(await def.refresh(stored.refresh, deps.signal, stored), stored); + const refreshed = await def.refresh(stored.refresh, deps.signal, stored, accountId); + const fresh = merged(refreshed, stored); const outcome = await mergeAccountCredential(provider, accountId, fresh, { expectedGeneration: generation, afterPrePersistRead: deps.afterPrePersistRead, + ...(provider === "devin" ? { assertOwnership: (store: AuthStore) => { + if (store[provider]?.accounts.find(row => row.id === accountId)?.paused) throw new OAuthAccountPausedError(); + assertDevinCliAdoptionOwnership(store, provider, accountId, refreshed); + } } : {}), }); if (outcome.superseded) { if (outcome.stored.expires > Date.now() + REFRESH_SKEW_MS) return outcome.stored.access; diff --git a/src/oauth/kiro-terminal-failover.ts b/src/oauth/kiro-terminal-failover.ts index 968ac1e8b4d..e2a2dcafbb6 100644 --- a/src/oauth/kiro-terminal-failover.ts +++ b/src/oauth/kiro-terminal-failover.ts @@ -2,27 +2,41 @@ import type { OAuthAccessSnapshot } from "./index"; import type { OcxConfig } from "../types"; import { getValidAccessSnapshotForAccount } from "./index"; import { credentialGeneration, getAccountCredentialWithStatus, getAccountSet } from "./store"; -import { eligibleFailoverAccounts, isGenericOAuthFailoverEnabled, +import { eligibleFailoverAccounts, isGenericFailoverProvider, GENERIC_OAUTH_MAX_ACCOUNTS_PER_REQUEST } from "./generic-account-failover"; -/** Only an unchanged, allowlist-classified dead credential permits this alternate. */ -export async function tryKiroAlternateAfterTerminalRefresh( - config: OcxConfig, failedAccountId: string, failedGeneration: string, +/** + * Only an unchanged, allowlist-classified dead credential permits this alternate. + * + * Consent is the stored-login count, needsReauth rows included: the refused account was + * marked needsReauth a moment ago, and counting only healthy rows would make a two-account + * setup look like one exactly when the second account is needed. + */ +export async function tryAlternateAfterTerminalRefresh( + config: OcxConfig, providerName: string, failedAccountId: string, failedGeneration: string, ): Promise { - if (!isGenericOAuthFailoverEnabled(config, "kiro")) return null; - const failed = getAccountCredentialWithStatus("kiro", failedAccountId); + const provider = config.providers?.[providerName]; + if (!provider || !isGenericFailoverProvider(providerName, provider)) return null; + const order = getAccountSet(providerName)?.accounts.map(row => row.id) ?? []; + if (order.length < 2) return null; + const failed = getAccountCredentialWithStatus(providerName, failedAccountId); if (!failed?.needsReauth || credentialGeneration(failed.credential) !== failedGeneration) return null; - const order = getAccountSet("kiro")?.accounts.map(row => row.id) ?? []; const after = order.indexOf(failedAccountId); if (after < 0) return null; const ring = [...order.slice(after + 1), ...order.slice(0, after)]; - const eligible = new Set(eligibleFailoverAccounts("kiro")); + const eligible = new Set(eligibleFailoverAccounts(providerName)); let attempted = 0; for (const id of ring) { if (id === failedAccountId || !eligible.has(id)) continue; if (++attempted >= GENERIC_OAUTH_MAX_ACCOUNTS_PER_REQUEST) break; - try { return await getValidAccessSnapshotForAccount("kiro", id, { requireUsableAccount: true }); } + try { return await getValidAccessSnapshotForAccount(providerName, id, { requireUsableAccount: true }); } catch { /* Keep the original login-required result if every alternate is stale. */ } } return null; } + +export function tryKiroAlternateAfterTerminalRefresh( + config: OcxConfig, failedAccountId: string, failedGeneration: string, +): Promise { + return tryAlternateAfterTerminalRefresh(config, "kiro", failedAccountId, failedGeneration); +} diff --git a/src/oauth/store.ts b/src/oauth/store.ts index 71b5b485974..5c9fff453fe 100644 --- a/src/oauth/store.ts +++ b/src/oauth/store.ts @@ -1379,5 +1379,5 @@ export async function markAccountNeedsReauth( }, [provider, accountId]); } -export async function mergeAccountCredential(provider:string,accountId:string,credential:OAuthCredentials,opts:{expectedGeneration?:string;afterPrePersistRead?:()=>void|Promise}={}):Promise<{superseded:false}|{superseded:true;stored:OAuthCredentials}>{const safe=normalizeCredential(credential);if(!safe)throw new Error("Refusing to persist invalid OAuth credential");return await mutateStore(async store=>{await opts.afterPrePersistRead?.();const account=store[provider]?.accounts.find(x=>x.id===accountId);if(!account)throw new Error(`OAuth account disappeared before persist: ${provider}`);if(opts.expectedGeneration!==undefined&&credentialGeneration(account.credential)!==opts.expectedGeneration)return{superseded:true,stored:account.credential};account.credential=safe;delete account.needsReauth;return{superseded:false};},[provider,accountId,safe,opts.expectedGeneration]);} +export async function mergeAccountCredential(provider:string,accountId:string,credential:OAuthCredentials,opts:{expectedGeneration?:string;afterPrePersistRead?:()=>void|Promise;assertOwnership?: (store: AuthStore) => void}={}):Promise<{superseded:false}|{superseded:true;stored:OAuthCredentials}>{const safe=normalizeCredential(credential);if(!safe)throw new Error("Refusing to persist invalid OAuth credential");return await mutateStore(async store=>{await opts.afterPrePersistRead?.();const account=store[provider]?.accounts.find(x=>x.id===accountId);if(!account)throw new Error(`OAuth account disappeared before persist: ${provider}`);if(opts.expectedGeneration!==undefined&&credentialGeneration(account.credential)!==opts.expectedGeneration)return{superseded:true,stored:account.credential};opts.assertOwnership?.(store);account.credential=safe;delete account.needsReauth;return{superseded:false};},[provider,accountId,safe,opts.expectedGeneration]);} export async function markAccountNeedsReauthIfGeneration(provider:string,accountId:string,generation:string,writerGeneration=captureConfigGeneration()):Promise{const key=oauthAccountKey(provider,accountId);if(writerGeneration{const account=store[provider]?.accounts.find(x=>x.id===accountId);if(!account?.credential||credentialGeneration(account.credential)!==generation)return false;if(writerGeneration | undefined; @@ -194,9 +196,10 @@ export async function prepareResponsesTransport( }; const refreshResolvedOAuthSelection = async (sent: OAuthAccessSnapshot): Promise => { const current = captureOAuthAccountSelection(route.providerName); - const unchanged = current?.accountId === oauthSelection?.accountId - && current?.revision === oauthSelection?.revision; - const candidate = unchanged ? await forceRefreshOAuthAccessSnapshot(sent) : sent; + // Keyed on the account, not the selection revision: a selection that moved away and back + // (A -> B -> A) before the 401 landed still serves the rejected credential, and skipping + // the refresh would replay it and spend the one recovery attempt. + const candidate = current?.accountId === sent.accountId ? await forceRefreshOAuthAccessSnapshot(sent) : sent; const admitted = await commitResolvedOAuthSelection(candidate); if (!admitted) throw new Error("OAuth selection changed during credential recovery"); if (kiroLoadEnabled && options.accountLoad?.lease?.accountId !== admitted.accountId) { diff --git a/src/server/responses/run-turn-execution.ts b/src/server/responses/run-turn-execution.ts index 736be252156..4eb5eccec23 100644 --- a/src/server/responses/run-turn-execution.ts +++ b/src/server/responses/run-turn-execution.ts @@ -27,6 +27,8 @@ import { rotateGenericOAuthAccountOn429, failoverAccountSnapshot, } from "../../oauth/generic-account-failover"; +import { OAuthAccountPausedError, publicOAuthAuthenticationErrorMessage, type OAuthAccessSnapshot } from "../../oauth/index"; +import { tryAlternateAfterTerminalRefresh } from "../../oauth/kiro-terminal-failover"; import { resolveWireProtocolOverride } from "../adapter-resolve"; import { formatErrorResponse, bridgeToResponsesSSE, buildResponseJSON } from "../../bridge"; import { redactSecretString } from "../../lib/redact"; @@ -93,6 +95,9 @@ export async function executeResponsesRunTurn( | "genericFailovers" | "genericFailoverLimit" | "applyFailoverSnapshot" + | "refreshResolvedOAuthSelection" + | "isOAuth401ReplayProvider" + | "sentOAuthSnapshot" | "resolveSelectionAdapter" | "adapter" | "noteRoutedAttemptSend" @@ -122,6 +127,7 @@ export async function executeResponsesRunTurn( adapterBindings, refreshRunTurnAdapter, applyFailoverSnapshot, + refreshResolvedOAuthSelection, resolveSelectionAdapter, } = transportState; const { @@ -348,6 +354,99 @@ export async function executeResponsesRunTurn( yield* await preflightRunTurnFailover(stream, iterParsed); })(); }; + // Rebind the turn to an admitted account. The failed attempt emitted no client-visible bytes, + // so replay is safe, but a Cursor conversation/checkpoint is credential-scoped: carrying its + // account identity into the next account would not be. The rotated adapter derives its own. + const adoptRunTurnAccount = (admittedSnapshot: Pick): boolean => { + parsed._cursorIdentityScope = undefined; + parsed._cursorConversationId = undefined; + if (parsed._providerContinuation?.cursor) { + const { cursor: _discardedCursor, ...otherProviderState } = parsed._providerContinuation; + parsed._providerContinuation = otherProviderState; + } + const rotatedProvider = resolveWireProtocolOverride( + route.providerName, + route.modelId, + route.provider, + inboundWire, + route.staticPolicy, + ); + const rotatedAdapter = resolveSelectionAdapter(rotatedProvider, config.cacheRetention); + if (!rotatedAdapter.runTurn) return false; + transportState.runTurnAdapter = rotatedAdapter; + bindRouteReasoningReplayScope({ + parsed, + providerName: route.providerName, + provider: rotatedProvider, + adapterName: rotatedAdapter.name, + oauthCredentialSnapshot: { + accountId: admittedSnapshot.accountId, + generation: admittedSnapshot.generation, + }, + codexAuthContext: admissionState.authCtx, + forwardHeaders: requestState.selectedForwardHeaders, + }); + sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, rotatedAdapter.name, logCtx.accountLogLabel); + recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, rotatedAdapter.name); + return true; + }; + // The runTurn half of the upstream-401 replay adapter-dispatch runs for HTTP transports: + // force-refresh the credential that was sent, once per request. A refresh that cannot + // succeed marks the account needsReauth inside the OAuth owner, so the account stops being + // selected; the turn then moves to a surviving stored account, or, with none, the client + // gets the login instruction instead of an opaque upstream 401. + let oauth401ReplayAttempted = false; + let pausedAuthRecovery = false; + const recoverRunTurnAdapterOnPreflight401 = async ( + error: Extract, + ): Promise => { + const sent = transportState.sentOAuthSnapshot; + if (error.status !== 401 || !transportState.isOAuth401ReplayProvider || !sent || oauth401ReplayAttempted) return false; + oauth401ReplayAttempted = true; + const hop = reserveCredentialHop("auth-recovery", `${route.providerName}|${route.modelId}|runturn-oauth-401`); + if (!hop.allowed) return false; + try { + let admitted: OAuthAccessSnapshot | null; + try { + admitted = await applyFailoverSnapshot(await refreshResolvedOAuthSelection(sent)); + } catch (err) { + if (err instanceof OAuthAccountPausedError) { + pausedAuthRecovery = true; + Object.assign(error, { status: 403, errorType: "permission_error", message: publicOAuthAuthenticationErrorMessage(err) }); + hop.permit?.release(); + return false; + } + // Not only OAuthLoginRequiredError: a concurrent request that already flagged this + // account and moved the selection makes this refresh fail as "selection changed". The + // helper still requires the sent generation to be flagged needsReauth, so it is safe. + const alternate = transportState.genericFailovers < transportState.genericFailoverLimit + ? await tryAlternateAfterTerminalRefresh(config, route.providerName, sent.accountId, sent.generation) + : null; + admitted = alternate ? await applyFailoverSnapshot(alternate) : null; + if (admitted) transportState.genericFailovers += 1; + else Object.assign(error, { errorType: "authentication_error", message: publicOAuthAuthenticationErrorMessage(err) }); + } + if (!admitted || !adoptRunTurnAccount(admitted)) { + hop.permit?.release(); + return false; + } + sendBudgetState.pendingHopPermit = hop.permit; + return true; + } catch (err) { + // Applying the alternate can still fail, e.g. when a newer manual selection points back + // at the flagged account; the client then needs the login instruction, not the raw 401. + Object.assign(error, { errorType: "authentication_error", message: publicOAuthAuthenticationErrorMessage(err) }); + hop.permit?.release(); + return false; + } + }; + const recoverRunTurnAdapterOnPreflightError = async ( + error: Extract, + ): Promise => { + if (await recoverRunTurnAdapterOnPreflight401(error)) return "oauth-401"; + if (await rotateRunTurnAdapterOnPreflight429(error)) return "oauth-account-429"; + return undefined; + }; const rotateRunTurnAdapterOnPreflight429 = async ( error: Extract, ): Promise => { @@ -400,42 +499,10 @@ export async function executeResponsesRunTurn( hop.permit?.release(); return false; } - // A Cursor conversation/checkpoint is credential-scoped. The failed attempt emitted no - // client-visible bytes, so replay is safe, but carrying its account identity into the next - // account would not be. Let the rotated adapter derive a fresh identity and conversation. - parsed._cursorIdentityScope = undefined; - parsed._cursorConversationId = undefined; - if (parsed._providerContinuation?.cursor) { - const { cursor: _discardedCursor, ...otherProviderState } = parsed._providerContinuation; - parsed._providerContinuation = otherProviderState; - } - const rotatedProvider = resolveWireProtocolOverride( - route.providerName, - route.modelId, - route.provider, - inboundWire, - route.staticPolicy, - ); - const rotatedAdapter = resolveSelectionAdapter(rotatedProvider, config.cacheRetention); - if (!rotatedAdapter.runTurn) { + if (!adoptRunTurnAccount(admittedSnapshot)) { hop.permit?.release(); return false; } - transportState.runTurnAdapter = rotatedAdapter; - bindRouteReasoningReplayScope({ - parsed, - providerName: route.providerName, - provider: rotatedProvider, - adapterName: rotatedAdapter.name, - oauthCredentialSnapshot: { - accountId: admittedSnapshot.accountId, - generation: admittedSnapshot.generation, - }, - codexAuthContext: admissionState.authCtx, - forwardHeaders: requestState.selectedForwardHeaders, - }); - sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, rotatedAdapter.name, logCtx.accountLogLabel); - recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, rotatedAdapter.name); // The caller replays the turn on this rotation, and a runTurn adapter dispatches through // its own reservation ladder -- Cursor reserves once per physical send. Confirming here // would leave that ladder to charge the same replay a second time (#4709), so hand the @@ -463,13 +530,14 @@ export async function executeResponsesRunTurn( yield event; continue; } - if (!firstMeaningfulSeen && !replayUnsafe && event.type === "error" - && await rotateRunTurnAdapterOnPreflight429(event)) { + const recovery = !firstMeaningfulSeen && !replayUnsafe && event.type === "error" + ? await recoverRunTurnAdapterOnPreflightError(event) : undefined; + if (recovery) { const retryQueue = createAdapterEventQueue({ onBacklogExceeded: () => runTurnAbort.abort(), }); const pendingPermit = sendBudgetState.pendingHopPermit; - const retryAttempt = runTurnAttempt(retryQueue, "oauth-account-429", false, replayParsed); + const retryAttempt = runTurnAttempt(retryQueue, recovery, false, replayParsed); if (pendingPermit) { const releaseIfUnclaimed = () => { if (sendBudgetState.pendingHopPermit !== pendingPermit) return; @@ -522,15 +590,13 @@ export async function executeResponsesRunTurn( return streamAfterPreflight(preflight.stream, replayParsed, preflight.replayUnsafe); } if (preflight.ready) return streamAfterPreflight(preflight.stream, replayParsed, preflight.replayUnsafe); - if (preflight.replayUnsafe - || !preflight.error - || !(await rotateRunTurnAdapterOnPreflight429(preflight.error))) { - return preflight.stream; - } + const recovery = preflight.replayUnsafe || !preflight.error + ? undefined : await recoverRunTurnAdapterOnPreflightError(preflight.error); + if (!recovery) return preflight.stream; const retryQueue = createAdapterEventQueue({ onBacklogExceeded: () => runTurnAbort.abort(), }); - latestRetryAttempt = runTurnAttempt(retryQueue, "oauth-account-429", false, replayParsed); + latestRetryAttempt = runTurnAttempt(retryQueue, recovery, false, replayParsed); void latestRetryAttempt; source = retryQueue.stream(); } @@ -599,6 +665,14 @@ export async function executeResponsesRunTurn( // Preflight holds only heartbeats and the first meaningful event. A first-event 429 can be // replayed transparently; after any output reaches the bridge, a later error stays terminal. eventSource = await preflightRunTurnFailover(eventSource, wsFirstParsed, preflightDeadlineAt); + if (pausedAuthRecovery) { + cancelResponseCompletion(); + runTurnAbort.abort(); + queue.close(); + cleanupRunTurnAbort(); + releaseSearchProbeLease(); + return formatErrorResponse(403, "permission_error", publicOAuthAuthenticationErrorMessage(new OAuthAccountPausedError())); + } } if (grokDevinPreflight) { const preflight = await preflightAdapterEvents(eventSource, undefined, { @@ -729,6 +803,14 @@ export async function executeResponsesRunTurn( for await (const event of await preflightRunTurnFailover( (async function* () { yield* firstAttemptEvents; })(), )) runTurnEvents.push(event); + if (pausedAuthRecovery) { + cancelResponseCompletion(); + runTurnAbort.abort(); + queue.close(); + cleanupRunTurnAbort(); + releaseSearchProbeLease(); + return formatErrorResponse(403, "permission_error", publicOAuthAuthenticationErrorMessage(new OAuthAccountPausedError())); + } } if (grokDevinPreflight) { const preflight = await preflightAdapterEvents( diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index 15b2c77556a..8d4ab2b2f1d 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -91,6 +91,15 @@ proactive refresh, per-account quota probes (`accountQuotaProbeSkip` in Muse key-mint quota read, and xAI/Gemini web-search sidecar eligibility. The stored credential remains available for resume, while requests with no unpaused account fail with 403 rather than as a login failure. +Devin's local-CLI forced refresh requires a stored account ID or email before it can adopt a +changed CLI key. Identity-less imports take the terminal reauthentication path; they require +an explicit `ocx login devin` after rotation. For bound slots, it validates the CLI tenant host +and probes the key with a bounded `GetUserJwt` call. It adopts a changed key only when the minted +identity matches the stored slot and the key and identity are unowned across Devin and alias +accounts at the locked store write. The same account id in a legacy alias slot is not a +competing owner, even when that alias already holds the rotated key; another account holding the key +still blocks adoption. The generation check still protects concurrent edits. A losing adoption or +unreadable CLI file leaves the account unflagged; a paused account returns 403 with its stored key intact. The account actually sent supplies the generation fence; a rotated bearer always travels with its own profile ARN and region. Reactive rotation follows the stored two-account quorum, while refusal-aware first admission follows the proactive preference setting. diff --git a/structure/transports/responses-failover.md b/structure/transports/responses-failover.md index 76f0cfb246c..af32c1e9f93 100644 --- a/structure/transports/responses-failover.md +++ b/structure/transports/responses-failover.md @@ -246,6 +246,12 @@ surface the pre-output 429 immediately, because the outer response cannot forwar heartbeats while it is choosing a target. An earlier replay-unsafe heartbeat or meaningful output keeps the failure on the current target. +## runTurn pre-output 401 replay + +`src/server/responses/run-turn-execution.ts` handles a structured pre-output 401 for `isOAuth401ReplayProvider` runTurn routes with one generation-fenced forced refresh and an `auth-recovery` hop. A terminal refresh may admit a surviving account; otherwise the client gets the login instruction. Devin quota `permission_denied` (429) and plain permission denial (403) do not trigger this path. +An `oauth`-source Devin key rejected with 401 needs reauthentication because Cognition has no refresh endpoint. A `local-cli` slot with a stored account ID or email can adopt a changed CLI key after host and bounded identity validation; [OAuth/Devin ownership](../providers-and-adapters.md) defines the locked store check across aliases. An identity-less CLI import instead needs an explicit `ocx login devin` after rotation. Unreadable files and transient probes leave the account unflagged. A pause during refresh returns 403 without retry, reauth, or replacing the stored key. Kiro's terminal alternate requires `OAuthLoginRequiredError`; transient refresh failures do not enter it. +Tests: `tests/responses/responses-devin-401-replay.test.ts` and `tests/server/server-kiro-refusal-e2e.test.ts`. + ## Optional client transport hints `dropCodexSafetyBuffering` defaults to false. Canonical OpenAI forward Responses can remove only @@ -586,14 +592,5 @@ provider cannot establish the original serving identity and remains portable. ## Anthropic Fast downgrade recovery -The `anthropic` OAuth and `anthropic-apikey` registry entries use native `anthropic-speed` FastWire -only for `claude-opus-5-5`, `claude-opus-5` and `claude-opus-4-8`; there is no provider-wide Fast -fallback. In the main adapter dispatch loop, a fast refusal naming fast mode or the `speed` parameter -(400 or 429), or a 429 with a fast-pool remaining header of zero, may use one shared-budget repair -permit for a standard-speed resend. The resend charges the root workflow send counter once without a -second request-budget charge. The request retains the drop decision through later rebuilds and records -`anthropic-fast-downgrade`, cause `parameter-rejected`, and a `downgraded` / `response-declined` tier -outcome. A spent budget leaves the original refusal intact; generic 429 and 529 responses keep their -ordinary handling. This repair precedes same-target 429 waiting and credential rotation. Continuation -and sidecar owners do not use this repair. Coverage: `tests/routing/fastwire-policy.test.ts` and -`tests/responses/responses-anthropic-fast-downgrade.test.ts`. +The `anthropic` OAuth and `anthropic-apikey` registry entries use native `anthropic-speed` FastWire only for `claude-opus-5-5`, `claude-opus-5` and `claude-opus-4-8`; there is no provider-wide Fast fallback. In the main adapter dispatch loop, a fast refusal naming fast mode or the `speed` parameter (400 or 429), or a 429 with a fast-pool remaining header of zero, may use one shared-budget repair permit for a standard-speed resend. +The resend charges the root workflow send counter once without a second request-budget charge. The request retains the drop decision through later rebuilds and records `anthropic-fast-downgrade`, cause `parameter-rejected`, and a `downgraded` / `response-declined` tier outcome. A spent budget leaves the original refusal intact; generic 429 and 529 responses keep their ordinary handling. This repair precedes same-target 429 waiting and credential rotation. Continuation and sidecar owners do not use this repair. Coverage: `tests/routing/fastwire-policy.test.ts` and `tests/responses/responses-anthropic-fast-downgrade.test.ts`. diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index a02bf39bd4a..4269c2596ed 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -46,6 +46,7 @@ "start-args.test.ts": "cli", "start-ownership-publication.test.ts": "cli", "responses-core-modules.test.ts": "responses", + "responses-devin-401-replay.test.ts": "responses", "responses-grok-devin-preflight.test.ts": "responses", "responses-passthrough-transient-policy.test.ts": "responses", "responses-spend-ledger-wiring.test.ts": "responses", diff --git a/tests/responses/responses-devin-401-replay.test.ts b/tests/responses/responses-devin-401-replay.test.ts new file mode 100644 index 00000000000..6bbe4bd8877 --- /dev/null +++ b/tests/responses/responses-devin-401-replay.test.ts @@ -0,0 +1,484 @@ +import { afterAll, afterEach, beforeEach, expect, mock, test } from "bun:test"; +import { writeFileSync } from "node:fs"; +import type { ProviderAdapter } from "../../src/adapters/base"; +import type { AdapterEvent, OcxConfig, OcxProviderConfig } from "../../src/types"; +import { getAccountSet, replaceProviderAccountSet, saveCredential, setAccountPaused, setActiveAccount } from "../../src/oauth/store"; +import { forceRefreshOAuthAccessSnapshot, getValidAccessTokenSnapshot } from "../../src/oauth"; +import { clearGenericFailoverHealth } from "../../src/oauth/generic-account-failover"; +import { DEVIN_CLI_CREDENTIALS_ENV } from "../../src/oauth/devin/cli-import"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { createTempHome } from "../helpers/temp-home"; + +const DEAD = "devin-session-token$synthetic-dead"; +const LIVE = "devin-session-token$synthetic-live"; +const ROTATED = "devin-session-token$synthetic-rotated"; +const DEAD_OTHER = "devin-session-token$synthetic-dead-other"; +const originalFetch = globalThis.fetch; + +// GetUserJwt stand-in: the identity a key mints, keyed by the key inside the protobuf body. +let mintedIdentity: Record = {}; +let mintCalls = 0; +let mintFailure: (() => Response) | undefined; +let holdMint: (() => Promise) | undefined; +function fakeUserJwt(payload: object): string { + const part = (value: object) => Buffer.from(JSON.stringify(value)).toString("base64url"); + return `${part({ alg: "HS256", typ: "JWT" })}.${part({ ...payload, exp: 9_999_999_999 })}.c2lnbmF0dXJl`; +} +const mintFetch = (async (input: Parameters[0], init?: RequestInit) => { + const url = String(input instanceof Request ? input.url : input); + if (!url.endsWith("/exa.auth_pb.AuthService/GetUserJwt")) return originalFetch(input, init); + mintCalls++; + await holdMint?.(); + if (mintFailure) return mintFailure(); + const body = Buffer.from(init?.body as Uint8Array).toString("latin1"); + const key = Object.keys(mintedIdentity).find(candidate => body.includes(candidate)); + const identity = key ? mintedIdentity[key] : undefined; + if (!identity) return new Response("", { status: 401 }); + const jwt = Buffer.from(fakeUserJwt(identity)); + return new Response(Buffer.concat([Buffer.from([0x0a, jwt.length & 0x7f | 0x80, jwt.length >> 7]), jwt]), + { status: 200, headers: { "content-type": "application/proto" } }); +}) as typeof fetch; + +const resolver = await import("../../src/server/adapter-resolve"); +const originalResolve = resolver.resolveAdapter; +const originalResolverModule = { ...resolver }; +let sentKeys: string[] = []; +let rateLimited = false; +let holdDeadSend: (() => Promise) | undefined; +mock.module("../../src/server/adapter-resolve", () => ({ ...resolver, + resolveAdapter(provider: OcxProviderConfig, cache?: "none" | "short" | "long") { + if (provider.adapter !== "devin") return originalResolve(provider, cache); + return { + name: "devin", + buildRequest: () => ({ url: provider.baseUrl, method: "POST", headers: {}, body: "" }), + async *parseStream() { yield { type: "done" } as AdapterEvent; }, + async runTurn(_parsed, _incoming, emit) { + const key = String(provider.apiKey); + sentKeys.push(key); + if (rateLimited) { + emit({ type: "error", status: 429, errorType: "rate_limit_error", code: "resource_exhausted", + retryable: true, message: "Cognition chat failed (resource_exhausted)" }); + return; + } + if (key === DEAD) { + await holdDeadSend?.(); + emit({ type: "error", status: 401, errorType: "authentication_error", code: "unauthenticated", + retryable: false, message: "Devin cloud error unauthenticated: invalid api key" }); + return; + } + emit({ type: "text_delta", text: `served by ${key === ROTATED ? "rotated" : "live"}` }); + emit({ type: "done" }); + }, + } satisfies ProviderAdapter; + }, +})); +const { handleResponses } = await import("../../src/server/responses"); + +let home: ReturnType; +let release: (() => void) | undefined; +let previousCliPath: string | undefined; + +beforeEach(() => { + home = createTempHome("ocx-devin-401-replay-"); + release = acquireOwnedSpendHome(); + clearGenericFailoverHealth(); + sentKeys = []; + rateLimited = false; + holdDeadSend = undefined; + mintedIdentity = { [ROTATED]: { auth_uid: "uid-rotated", email: "rotated@example.com" } }; + mintCalls = 0; + mintFailure = undefined; + holdMint = undefined; + globalThis.fetch = mintFetch; + previousCliPath = process.env[DEVIN_CLI_CREDENTIALS_ENV]; + // Never let a test read the developer's real CLI credential. + process.env[DEVIN_CLI_CREDENTIALS_ENV] = home.path("devin-credentials.toml"); +}); +afterAll(() => { + mock.module("../../src/server/adapter-resolve", () => originalResolverModule); +}); +afterEach(() => { + try { + release?.(); + } finally { + if (previousCliPath === undefined) delete process.env[DEVIN_CLI_CREDENTIALS_ENV]; + else process.env[DEVIN_CLI_CREDENTIALS_ENV] = previousCliPath; + globalThis.fetch = originalFetch; + clearGenericFailoverHealth(); + home.remove(); + } +}); + +function writeCliFile(apiKey: string, apiServerUrl = "https://server.codeium.com"): void { + writeFileSync(home.path("devin-credentials.toml"), + `windsurf_api_key = "${apiKey}"\napi_server_url = "${apiServerUrl}"\n`); +} + +async function saveDevin(access: string, accountId: string, source: "oauth" | "local-cli" = "oauth") { + await saveCredential("devin", { + access, refresh: access, expires: Number.MAX_SAFE_INTEGER, accountId, source, + apiBaseUrl: "https://server.codeium.com", + }); +} + +// A CLI import records no identity: the session token it copies carries only a session_id. +async function saveCliImport(access: string, extra: { accountId?: string; email?: string } = {}) { + await saveCredential("devin", { + access, refresh: access, expires: Number.MAX_SAFE_INTEGER, source: "local-cli", + apiBaseUrl: "https://server.codeium.com", ...extra, + }, { preserveIdentityless: true }); +} + +function cliAccount() { + return getAccountSet("devin")?.accounts.find(row => row.credential.source === "local-cli"); +} + +function run(stream = false) { + const config = { + port: 0, defaultProvider: "devin", + providers: { devin: { adapter: "devin", authMode: "oauth", baseUrl: "https://server.codeium.com", models: ["swe-1-6"] } }, + } as OcxConfig; + return handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "devin/swe-1-6", input: "answer", stream }), + }), config, { model: "", provider: "", surface: "codex" }); +} + +function account(id: string) { + return getAccountSet("devin")?.accounts.find(row => row.credential.accountId === id); +} + +test.each([false, true])("a revoked key is marked needsReauth and the turn fails over (stream=%s)", async stream => { + await saveDevin(LIVE, "spare"); + await saveDevin(DEAD, "revoked"); + expect(getAccountSet("devin")?.activeAccountId).toBe(account("revoked")?.id); + + const response = await run(stream); + const body = await response.text(); + + expect(response.status).toBe(200); + expect(body).toContain("served by live"); + expect(sentKeys).toEqual([DEAD, LIVE]); + expect(account("revoked")?.needsReauth).toBe(true); + expect(getAccountSet("devin")?.activeAccountId).toBe(account("spare")?.id); + + // The dead account is not reselected by the next request. + sentKeys = []; + expect((await run()).status).toBe(200); + expect(sentKeys).toEqual([LIVE]); +}); + +test("a lone revoked account surfaces the login instruction", async () => { + await saveDevin(DEAD, "revoked"); + + // A buffered runTurn failure is a `status: "failed"` Response object, not an HTTP error. + const body = await (await run()).json() as { status: string; error: { type: string; message: string } }; + + expect(body.status).toBe("failed"); + expect(body.error.type).toBe("authentication_error"); + expect(body.error.message).toBe("Not logged in to devin. Run: ocx login devin"); + expect(sentKeys).toEqual([DEAD]); + expect(account("revoked")?.needsReauth).toBe(true); + + // Later turns fail fast on the flagged account instead of re-sending a dead key. + sentKeys = []; + const next = await run(); + expect(next.status).toBe(401); + expect(await next.text()).toContain("ocx login devin"); + expect(sentKeys).toEqual([]); +}); + +test("an identity-less CLI import rejects a rotated key and needs reauth", async () => { + await saveCliImport(DEAD); + writeCliFile(ROTATED); + + const body = await (await run()).json() as { error: { message: string } }; + expect(body.error.message).toBe("Not logged in to devin. Run: ocx login devin"); + expect(sentKeys).toEqual([DEAD]); + expect(mintCalls).toBe(0); + const row = cliAccount(); + expect(row?.needsReauth).toBe(true); + expect(row?.credential.access).toBe(DEAD); +}); + +test("a CLI file belonging to another user cannot replace a bound account", async () => { + await saveCliImport(DEAD, { accountId: "uid-original", email: "original@example.com" }); + writeCliFile(ROTATED); // Mints uid-rotated, a different Devin user. + + const body = await (await run()).json() as { error: { message: string } }; + expect(body.error.message).toBe("Not logged in to devin. Run: ocx login devin"); + expect(sentKeys).toEqual([DEAD]); + expect(mintCalls).toBe(1); + expect(cliAccount()?.needsReauth).toBe(true); + expect(cliAccount()?.credential.access).toBe(DEAD); +}); + +test("a legacy alias copy of the same account already holding the rotated key does not block adoption", async () => { + await saveCliImport(DEAD, { accountId: "uid-rotated" }); + const current = cliAccount()!; + await replaceProviderAccountSet("devin-cli", { + activeAccountId: current.id, + accounts: [{ ...current, credential: { + ...current.credential, access: ROTATED, refresh: ROTATED, + accountId: "uid-rotated", email: "rotated@example.com", + } }], + }); + writeCliFile(ROTATED); + + expect(await (await run()).text()).toContain("served by rotated"); + expect(sentKeys).toEqual([DEAD, ROTATED]); + expect(cliAccount()?.credential.access).toBe(ROTATED); + expect(getAccountSet("devin-cli")?.accounts[0]?.credential.access).toBe(ROTATED); + expect(cliAccount()?.needsReauth).not.toBe(true); +}); + +test("a distinct legacy alias account holding the rotated key blocks adoption", async () => { + await saveCliImport(DEAD); + await saveCredential("devin-cli", { + access: ROTATED, refresh: ROTATED, expires: Number.MAX_SAFE_INTEGER, + source: "oauth", apiBaseUrl: "https://server.codeium.com", + }); + writeCliFile(ROTATED); + + await (await run()).text(); + expect(sentKeys).not.toContain(ROTATED); + expect(mintCalls).toBe(0); + expect(cliAccount()?.credential.access).toBe(DEAD); + expect(cliAccount()?.needsReauth).toBe(true); +}); + +test("a distinct legacy alias account still blocks an owned CLI identity", async () => { + await saveCredential("devin-cli", { + access: LIVE, refresh: LIVE, expires: Number.MAX_SAFE_INTEGER, + accountId: "uid-rotated", source: "oauth", apiBaseUrl: "https://server.codeium.com", + }); + await saveCliImport(DEAD, { email: "rotated@example.com" }); + writeCliFile(ROTATED); + + await (await run()).text(); + expect(cliAccount()?.credential.access).toBe(DEAD); + expect(cliAccount()?.needsReauth).toBe(true); +}); + +test.each([ + ["unchanged", () => writeCliFile(DEAD)], + ["missing", () => {}], + ["off-allowlist host", () => writeCliFile(ROTATED, "https://attacker.example")], +])("a CLI-imported account with a %s credential file needs reauth", async (_label, arrange) => { + await saveCliImport(DEAD); + arrange(); + + const body = await (await run()).json() as { error: { message: string } }; + + expect(body.error.message).toBe("Not logged in to devin. Run: ocx login devin"); + expect(sentKeys).toEqual([DEAD]); + expect(cliAccount()?.needsReauth).toBe(true); + expect(cliAccount()?.credential.access).toBe(DEAD); + // The key is only ever sent to an allowlisted host, including the identity probe. + expect(mintCalls).toBe(0); +}); + +test("a CLI key another stored account already owns is not adopted", async () => { + await saveDevin(ROTATED, "other"); + await saveCliImport(DEAD, { accountId: "uid-rotated" }); + writeCliFile(ROTATED); + + await (await run()).text(); + + expect(cliAccount()?.needsReauth).toBe(true); + expect(cliAccount()?.credential.access).toBe(DEAD); +}); + +test.each([ + ["a key Cognition refuses with 401", async () => { mintedIdentity = {}; await saveCliImport(DEAD, { accountId: "uid-rotated" }); }], + ["a key Cognition refuses with 403", async () => { + mintFailure = () => new Response("", { status: 403 }); + await saveCliImport(DEAD, { accountId: "uid-rotated" }); + }], + ["a minted token without auth_uid", async () => { + mintedIdentity = { [ROTATED]: { email: "rotated@example.com" } }; + await saveCliImport(DEAD, { accountId: "uid-rotated" }); + }], + ["a slot whose recorded accountId differs", async () => { await saveCliImport(DEAD, { accountId: "uid-before" }); }], + ["a slot whose recorded email differs", async () => { await saveCliImport(DEAD, { email: "before@example.com" }); }], + ["an identity another stored account owns", async () => { + await saveDevin(LIVE, "uid-rotated"); + await saveCliImport(DEAD, { email: "rotated@example.com" }); + }], +])("the CLI key is not adopted for %s", async (_label, arrange) => { + await arrange(); + writeCliFile(ROTATED); + + await (await run()).text(); + + expect(sentKeys).not.toContain(ROTATED); + expect(cliAccount()?.needsReauth).toBe(true); + expect(cliAccount()?.credential.access).toBe(DEAD); +}); + +test.each([ + ["a transient 503", () => new Response("", { status: 503 })], + ["a 429", () => new Response("", { status: 429 })], + ["a network failure", () => { throw new TypeError("fetch failed"); }], +])("an identity probe that fails with %s does not flag the account", async (_label, failure) => { + await saveCliImport(DEAD, { accountId: "uid-rotated" }); + writeCliFile(ROTATED); + mintFailure = failure; + + const body = await (await run()).json() as { error: { type: string; message: string } }; + + expect(body.error.type).toBe("authentication_error"); + expect(body.error.message).not.toContain("ocx login devin"); + expect(cliAccount()?.needsReauth).not.toBe(true); + expect(cliAccount()?.credential.access).toBe(DEAD); + + // Once the probe recovers, the next 401 adopts the rotated key. + mintFailure = undefined; + expect(await (await run()).text()).toContain("served by rotated"); + expect(cliAccount()?.credential.access).toBe(ROTATED); +}); + +test("a credential file caught mid-write does not flag the account", async () => { + await saveCliImport(DEAD, { accountId: "uid-rotated" }); + // Half-written by `devin auth login`: the key line is there, the server line is not yet. + writeFileSync(home.path("devin-credentials.toml"), `windsurf_api_key = "${ROTATED}"\n`); + + const body = await (await run()).json() as { error: { type: string; message: string } }; + + expect(body.error.type).toBe("authentication_error"); + expect(body.error.message).not.toContain("ocx login devin"); + expect(cliAccount()?.needsReauth).not.toBe(true); + expect(mintCalls).toBe(0); + + // Once the write completes, the next 401 adopts the rotated key. + writeCliFile(ROTATED); + expect(await (await run()).text()).toContain("served by rotated"); + expect(cliAccount()?.credential.access).toBe(ROTATED); +}); + +test("concurrent identity-less CLI slots both require explicit reauth", async () => { + await saveCliImport(DEAD); + const first = await getValidAccessTokenSnapshot("devin"); + await saveCliImport(DEAD_OTHER); + const second = await getValidAccessTokenSnapshot("devin"); + expect(first.accountId).not.toBe(second.accountId); + writeCliFile(ROTATED); + const outcomes = await Promise.allSettled([ + forceRefreshOAuthAccessSnapshot(first), forceRefreshOAuthAccessSnapshot(second), + ]); + const rows = getAccountSet("devin")!.accounts; + expect(outcomes.every(outcome => outcome.status === "rejected")).toBe(true); + expect(rows.filter(row => row.credential.access === ROTATED)).toHaveLength(0); + expect(rows.every(row => row.needsReauth === true)).toBe(true); + expect(mintCalls).toBe(0); +}); + +test.each([false, true])("an account paused during 401 refresh returns 403 (stream=%s)", async stream => { + await saveCliImport(DEAD, { accountId: "uid-rotated" }); + const accountId = cliAccount()!.id; + writeCliFile(ROTATED); + const mintStarted = Promise.withResolvers(); + const releaseMint = Promise.withResolvers(); + holdMint = async () => { mintStarted.resolve(); await releaseMint.promise; }; + const responsePromise = run(stream); + await mintStarted.promise; + await setAccountPaused("devin", accountId, true); + releaseMint.resolve(); + const response = await responsePromise; + const body = await response.json() as { error: { type: string; message: string } }; + expect(response.status).toBe(403); + expect(body.error.type).toBe("permission_error"); + expect(body.error.message).toContain("OAuth account is paused"); + expect(sentKeys).toEqual([DEAD]); + expect(cliAccount()?.needsReauth).not.toBe(true); + expect(cliAccount()?.credential.access).toBe(DEAD); +}); + +test("a slot recorded from the key's `sub` claim still matches the minted identity", async () => { + mintedIdentity = { [ROTATED]: { auth_uid: "uid-rotated", sub: "sub-rotated", email: "rotated@example.com" } }; + await saveCliImport(DEAD, { accountId: "sub-rotated" }); + writeCliFile(ROTATED); + + expect(await (await run()).text()).toContain("served by rotated"); + expect(cliAccount()?.credential.accountId).toBe("uid-rotated"); +}); + +test("a slot whose recorded identity matches adopts the rotated key", async () => { + await saveCliImport(DEAD, { accountId: "uid-rotated", email: " Rotated@Example.com " }); + writeCliFile(ROTATED); + + expect(await (await run()).text()).toContain("served by rotated"); + expect(cliAccount()?.credential.access).toBe(ROTATED); + expect(mintCalls).toBe(1); +}); + +test("a slot bound only by matching email adopts the rotated key", async () => { + await saveCliImport(DEAD, { email: " Rotated@Example.com " }); + writeCliFile(ROTATED); + + expect(await (await run()).text()).toContain("served by rotated"); + expect(cliAccount()?.credential.access).toBe(ROTATED); + expect(cliAccount()?.credential.accountId).toBe("uid-rotated"); +}); + +test("a turn whose 401 lands after another turn already failed the account over still fails over", async () => { + await saveDevin(LIVE, "spare"); + await saveDevin(DEAD, "revoked"); + // Both turns send on the revoked account; the second 401 only arrives once the first turn has + // flagged it and moved the selection, so the second refresh sees a changed selection. + const bothSent = Promise.withResolvers(); + const firstDone = Promise.withResolvers(); + let deadSends = 0; + holdDeadSend = async () => { + const order = ++deadSends; + if (order === 2) bothSent.resolve(); + await bothSent.promise; + if (order === 2) await firstDone.promise; + }; + + const first = run().then(async response => { + const text = await response.text(); + firstDone.resolve(); + return text; + }); + const second = run().then(response => response.text()); + const [firstBody, secondBody] = await Promise.all([first, second]); + + expect(deadSends).toBe(2); + expect(firstBody).toContain("served by live"); + expect(secondBody).toContain("served by live"); + expect(account("revoked")?.needsReauth).toBe(true); + expect(getAccountSet("devin")?.activeAccountId).toBe(account("spare")?.id); +}); + +test("a 429 is not treated as an authentication failure", async () => { + await saveDevin(LIVE, "limited"); + rateLimited = true; + + const body = await (await run()).json() as { error: { type: string } }; + + expect(body.error.type).toBe("rate_limit_error"); + expect(sentKeys).toEqual([LIVE]); + expect(account("limited")?.needsReauth).not.toBe(true); +}); + +test("a selection that leaves the revoked account and returns to it before the 401 still refreshes it", async () => { + await saveDevin(LIVE, "spare"); + await saveDevin(DEAD, "revoked"); + const revoked = account("revoked")!.id; + // Same account id at 401 time, newer selection revision: the refresh must still run, or the + // rejected key is replayed and the one allowed recovery is spent on it. + holdDeadSend = async () => { + holdDeadSend = undefined; + await setActiveAccount("devin", account("spare")!.id); + await setActiveAccount("devin", revoked); + }; + + const body = await (await run()).json() as { error: { message: string } }; + + expect(sentKeys.filter(key => key === DEAD)).toHaveLength(1); + expect(account("revoked")?.needsReauth).toBe(true); + // The operator's newer manual selection names the revoked account, so no automatic move + // overrides it; the client is told to log in rather than shown the raw upstream 401. + expect(body.error.message).toBe("Not logged in to devin. Run: ocx login devin"); +}); diff --git a/tests/server/server-kiro-refusal-e2e.test.ts b/tests/server/server-kiro-refusal-e2e.test.ts index abd4624cbc4..74d17cc9514 100644 --- a/tests/server/server-kiro-refusal-e2e.test.ts +++ b/tests/server/server-kiro-refusal-e2e.test.ts @@ -474,6 +474,28 @@ describe("Kiro refusal recovery through Responses", () => { } finally { await server.stop(true); } }); + test("a transient Kiro refresh failure does not enter terminal failover", async () => { + const [a] = await seed(); saveConfig(config()); + const sends: string[] = []; + globalThis.fetch = (async (input, init) => { + const url = input instanceof Request ? input.url : String(input); + if (url.endsWith("/refreshToken")) return new Response("", { status: 503 }); + if (url.startsWith("https://runtime.") && url.endsWith(".kiro.dev/")) { + const auth = new Headers(init?.headers).get("authorization") ?? ""; + sends.push(auth); + return auth === "Bearer access-a" ? new Response("expired", { status: 401 }) : answer("served by b"); + } + return realFetch(input, init); + }) as typeof fetch; + const server = startServer(0); + try { + const response = await post(server); + expect(response.status).toBe(401); + expect(sends).toEqual(["Bearer access-a"]); + expect(getAccountSet("kiro")!.accounts.find(row => row.id === a!.id)?.needsReauth).not.toBe(true); + } finally { await server.stop(true); } + }); + for (const [status, reason] of [[400, "MONTHLY_REQUEST_COUNT"], [403, "TEMPORARILY_SUSPENDED"]] as const) { test(`failed alternate resolution returns original ${status} with normalized Kiro message`, async () => { const [, b] = await seed(); saveConfig(config());