-
Notifications
You must be signed in to change notification settings - Fork 1.2k
fix(codex): keep native-main admission open across startup convergence and stop #5748
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,109 @@ | ||
| import { | ||
| isMainAccountPolicyBindingPending, | ||
| waitForNativeMainStartupGate, | ||
| } from "./native-profile-startup"; | ||
|
|
||
| /** | ||
| * How long a request waits for an owned startup to finish binding the main-account policy | ||
| * before it is refused as draining. | ||
| * | ||
| * `codexMainAccountHardLock` is on by default (#5694), so the admission fence beside this | ||
| * constant is no longer an opt-in rarity: every request that arrives while a process-owned | ||
| * startup is still recovering, sweeping stages, and binding the pinned home lands on it. That | ||
| * window is milliseconds on a warm Linux host and seconds on Windows, which is why the fence | ||
| * waits for the gate instead of failing the request the moment it arrives. | ||
| * | ||
| * 15 s is deliberately just above one startup claim wait: both exclusive claims | ||
| * (`withNativeMainExclusiveClaim` in `convergeOwnedStartup`) allow 10 s before they give up, so a | ||
| * request that arrived during the last claim is not refused in the instant before the gate would | ||
| * have opened. Anything longer stops being a courtesy to the client and starts being a hung | ||
| * request, and a gate still blocked after this leaves a real answer -- retained recovery or a | ||
| * manual-recovery requirement -- where waiting cannot help, which is what the draining error says. | ||
| */ | ||
| export const MAIN_ACCOUNT_POLICY_BINDING_WAIT_MS = 15_000; | ||
|
|
||
| /** | ||
| * Repoll interval for the one window where the settle promise is already resolved and the gate | ||
| * still reads pending: a manual-recovery fence publishes a resolved promise while an in-flight | ||
| * convergence is still marked pending, and re-awaiting that same resolved promise would spin the | ||
| * microtask queue -- which never lets the deadline timer fire. | ||
| */ | ||
| const MAIN_ACCOUNT_POLICY_BINDING_REPOLL_MS = 25; | ||
|
|
||
| export interface MainAccountPolicyBindingWait { | ||
| /** Ends the wait with the signal's reason, as an aborted request must. */ | ||
| signal?: AbortSignal; | ||
| /** Overrides {@link MAIN_ACCOUNT_POLICY_BINDING_WAIT_MS} for a bounded focused test. */ | ||
| timeoutMs?: number; | ||
| } | ||
|
|
||
| /** | ||
| * Wait, bounded, for an owned startup's main-account policy binding to settle. | ||
| * | ||
| * Returns `true` once the binding is no longer pending, `false` when the deadline passed with it | ||
| * still in flight. Callers fail closed on `false`; the gate itself is re-read every iteration | ||
| * because a rearm replaces the settle promise rather than resolving the one already held. | ||
| */ | ||
| export async function waitForMainAccountPolicyBinding( | ||
| wait: MainAccountPolicyBindingWait = {}, | ||
| ): Promise<boolean> { | ||
| const signal = wait.signal; | ||
| const timeoutMs = Math.max(0, wait.timeoutMs ?? MAIN_ACCOUNT_POLICY_BINDING_WAIT_MS); | ||
| const deadline = Date.now() + timeoutMs; | ||
| for (;;) { | ||
| if (!isMainAccountPolicyBindingPending()) return true; | ||
| const remaining = deadline - Date.now(); | ||
| if (remaining <= 0) return false; | ||
| const settledNow = await raceSettle(waitForNativeMainStartupGate(), remaining, signal); | ||
| // Settling does not always clear the flag: the gate may have been rearmed, or the resolved | ||
| // promise above may belong to a fence that never released the entry. Yield to the event loop | ||
| // before asking again so this stays a poll rather than a busy-wait. | ||
| if (settledNow && isMainAccountPolicyBindingPending()) { | ||
| await pause(Math.min(remaining, MAIN_ACCOUNT_POLICY_BINDING_REPOLL_MS), signal); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** `true` when the settle promise resolved first, `false` on the deadline. */ | ||
| async function raceSettle( | ||
| settle: Promise<unknown>, | ||
| timeoutMs: number, | ||
| signal?: AbortSignal, | ||
| ): Promise<boolean> { | ||
| if (signal?.aborted) throw signal.reason; | ||
| let timer: ReturnType<typeof setTimeout> | undefined; | ||
| let onAbort: (() => void) | undefined; | ||
| const deadline = new Promise<boolean>(resolve => { | ||
| timer = setTimeout(() => resolve(false), timeoutMs); | ||
| // A request waiting on a startup gate must never be the reason the process stays alive. | ||
| timer.unref?.(); | ||
| }); | ||
| const aborted = new Promise<never>((_resolve, reject) => { | ||
| if (!signal) return; | ||
| onAbort = () => reject(signal.reason); | ||
| signal.addEventListener("abort", onAbort); | ||
| }); | ||
| try { | ||
| // A rejected settle is not an error here: the loop re-reads the gate and decides. | ||
| return await Promise.race([settle.then(() => true, () => true), deadline, aborted]); | ||
| } finally { | ||
| if (timer !== undefined) clearTimeout(timer); | ||
| if (signal && onAbort) signal.removeEventListener("abort", onAbort); | ||
| } | ||
| } | ||
|
|
||
| /** One short, abortable, unref'd yield so the loop cannot monopolize the microtask queue. */ | ||
| function pause(ms: number, signal?: AbortSignal): Promise<void> { | ||
| if (signal?.aborted) return Promise.reject(signal.reason); | ||
| return new Promise<void>((resolve, reject) => { | ||
| const finish = (settle: () => void) => { | ||
| clearTimeout(timer); | ||
| signal?.removeEventListener("abort", onAbort); | ||
| settle(); | ||
| }; | ||
| const timer = setTimeout(() => finish(resolve), ms); | ||
| timer.unref?.(); | ||
| const onAbort = () => finish(() => reject(signal?.reason)); | ||
| signal?.addEventListener("abort", onAbort); | ||
| }); | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🗄️ Data Integrity & Integration | 🟠 Major | ⚡ Quick win
🔎 Supported by static analysis
🏁 Script executed:
Repository: lidge-jun/opencodex
Length of output: 11298
🏁 Script executed:
Repository: lidge-jun/opencodex
Length of output: 41706
🏁 Script executed:
Repository: lidge-jun/opencodex
Length of output: 45551
🏁 Script executed:
Repository: lidge-jun/opencodex
Length of output: 25722
🏁 Script executed:
Repository: lidge-jun/opencodex
Length of output: 42668
Preserve durable native-main recovery blocks on lifecycle release.
At
src/codex/native-profile-startup.ts:409-424, the last release deletes the entry and then resets any matching snapshot toready(null). The map check is always true after the deletion. This clearsmanual-recovery,stage-cleanup-required,owner-conflict, andowner-unavailablestates.When no service-ownership fence is active,
isNativeMainTrafficBlocked()then returnsfalse. The no-op lifecycle insrc/server/index.ts:617-621does not re-probe recovery. A later request can therefore select and materialize native__main__while the journal or stage residue remains. This violates the native-main read fence insrc/codex/account-usability.ts:83-91.Tie the reset to the exact
recovery-pendingsnapshot armed by this startup entry. Checking onlyreasonis insufficient becauseblockNativeMainRecovery(..., "journal")also creates arecovery-pendingsnapshot. Keep all other verdicts unchanged. Add release tests formanual-recoveryandstage-cleanup-required, and updatestructure/codex-home.mdso it does not state that the gate always returns toready.Suggested fix
interface StartupEntry { homeId: string; refs: number; epoch: number; + gateSnapshot?: NativeMainStartupGateSnapshot; owner: NativeMainOwnerReference; unsubscribe: () => void; recoveryStarted: boolean; @@ +function armEntryRecoveryPending(entry: StartupEntry): void { + snapshot = { status: "blocked", homeId: entry.homeId, reason: "recovery-pending" }; + entry.gateSnapshot = snapshot; +} + function convergeOwnedStartup(entry: StartupEntry): void { if (entry.recoveryStarted || startupEntries.get(entry.homeId) !== entry) return; entry.recoveryStarted = true; const currentEpoch = entry.epoch; - snapshot = { status: "blocked", homeId: entry.homeId, reason: "recovery-pending" }; + armEntryRecoveryPending(entry); @@ if (entry.policyBindingPending && (owner.status === "held" || owner.status === "acquiring")) { - snapshot = { status: "blocked", homeId: entry.homeId, reason: "recovery-pending" }; + armEntryRecoveryPending(entry); settled = entry.settled; return true; @@ if (owner.status === "acquiring") { - snapshot = { status: "blocked", homeId: entry.homeId, reason: "recovery-pending" }; + armEntryRecoveryPending(entry); return; @@ entry = { homeId, refs: 0, epoch: ++epoch, + gateSnapshot: snapshot, owner, @@ - if (snapshot.homeId === homeId && !startupEntries.has(homeId)) { + if ( + snapshot.homeId === homeId + && snapshot.status === "blocked" + && snapshot.reason === "recovery-pending" + && snapshot === entry!.gateSnapshot + && !startupEntries.has(homeId) + ) { epoch += 1; snapshot = ready(null); settled = Promise.resolve(snapshot);🤖 Prompt for AI Agents