From cdf0330b3794fce02ff96fdbca5a93e45c1b3a4a Mon Sep 17 00:00:00 2001 From: Codex Developer Date: Sun, 20 Sep 2026 04:10:45 +0800 Subject: [PATCH 1/7] fix(codex): log unattended Codex write admission refusals The startup path turned a silent admission refusal into a bare /readyz failed with zero evidence; the refusal message is the operator's only path to the cause (9-day silent catalog-sync stall on this deployment). (cherry picked from commit 4bc1c677eda60258df4b344e3b7c0f434bf4fcfd) --- src/codex/sync.ts | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/codex/sync.ts b/src/codex/sync.ts index d9217cc0a15..ecbe76fc9e1 100644 --- a/src/codex/sync.ts +++ b/src/codex/sync.ts @@ -122,18 +122,24 @@ export async function syncModelsToCodex( // catalog/cache. It therefore needs the same unattended service-home veto as // the injector, before it gets a chance to create any artifact. const admission = (deps.admitCodexWrite ?? admitCodexWrite)(); - if (admission.kind === "refused" && admission.authority === "service-home") { - return { - status: "refused", - authority: "service-home", - ok: false, - added: 0, - catalogPath: null, - catalogExists: false, - catalogWritten: false, - cacheSynced: false, - message: admission.message, - }; + if (admission.kind === "refused") { + // An unattended refusal must never be silent: the startup path turns it into + // a bare /readyz "failed" with zero evidence, and the operator's only path + // to the cause is this message. + log?.error(`[opencodex] Codex write admission refused (${admission.authority}): ${admission.message}`); + if (admission.authority === "service-home") { + return { + status: "refused", + authority: "service-home", + ok: false, + added: 0, + catalogPath: null, + catalogExists: false, + catalogWritten: false, + cacheSynced: false, + message: admission.message, + }; + } } // Config injection is a relevant Codex write even when the catalog bytes are unchanged. // Drop cached process evidence before async discovery so a process that appeared since the From 642cd548290ee0f29116f7ffe54c23137a6d927e Mon Sep 17 00:00:00 2001 From: Codex Developer Date: Wed, 23 Sep 2026 23:57:12 +0800 Subject: [PATCH 2/7] feat(bridge): carry the ChatGPT web bridge onto the 2.62 registry split The bridge was written against the single-file config and provider registry. On dev those modules are split, so the wiring lands per-owner: schema and diagnostics for chatgptBridge, the adapter and provider rows, and the GUI provider-list authority pin. The preset-count guard is what makes the docs half of that change mandatory rather than optional. Drops the three live-probe fixtures from the original branch: they hardcoded a private devspace issuer and a user-temp token path, which privacy:scan rejects. --- .../001_test_inventory.md | 4 + .../docs/fr/getting-started/quickstart.md | 2 +- .../src/content/docs/fr/guides/providers.md | 2 +- .../docs/getting-started/quickstart.md | 2 +- .../src/content/docs/guides/providers.md | 2 +- .../docs/ja/getting-started/quickstart.md | 2 +- .../src/content/docs/ja/guides/providers.md | 2 +- .../docs/ko/getting-started/quickstart.md | 2 +- .../src/content/docs/ko/guides/providers.md | 2 +- .../docs/ru/getting-started/quickstart.md | 2 +- .../src/content/docs/ru/guides/providers.md | 2 +- .../docs/tr/getting-started/quickstart.md | 2 +- .../src/content/docs/tr/guides/providers.md | 4 +- .../docs/zh-cn/getting-started/quickstart.md | 2 +- .../content/docs/zh-cn/guides/providers.md | 2 +- .../docs/zh-tw/getting-started/quickstart.md | 2 +- .../content/docs/zh-tw/guides/providers.md | 2 +- extensions/dsh-chatgpt-bridge/package.json | 27 + extensions/dsh-chatgpt-bridge/src/host.ts | 220 +++++++ scripts/test-layout/layout.json | 4 + src/adapters/registry.ts | 9 +- src/chatgpt-bridge/contracts/index.ts | 151 +++++ src/chatgpt-bridge/core/store.ts | 584 ++++++++++++++++++ .../hosts/codex/devspace-mcp-client.ts | 277 +++++++++ .../hosts/codex/legacy-registry.ts | 119 ++++ src/chatgpt-bridge/provider/adapter.ts | 97 +++ src/config/diagnostics.ts | 21 + src/config/proxy-env.ts | 1 + src/config/schema/config-schema.ts | 7 + src/providers/registry/entries-core.ts | 13 + src/types/config.ts | 6 + structure/ops/docs-and-release.md | 2 +- .../adapter-registry-authority.test.ts | 1 + .../chatgpt-bridge-codex-host.test.ts | 120 ++++ .../chatgpt-bridge-core.test.ts | 326 ++++++++++ .../chatgpt-bridge-dsh-host.test.ts | 177 ++++++ .../chatgpt-bridge-provider-adapter.test.ts | 84 +++ tests/fixtures/test-layout-expected.json | 4 + tests/test-layout-tooling.test.ts | 6 + 39 files changed, 2275 insertions(+), 19 deletions(-) create mode 100644 extensions/dsh-chatgpt-bridge/package.json create mode 100644 extensions/dsh-chatgpt-bridge/src/host.ts create mode 100644 src/chatgpt-bridge/contracts/index.ts create mode 100644 src/chatgpt-bridge/core/store.ts create mode 100644 src/chatgpt-bridge/hosts/codex/devspace-mcp-client.ts create mode 100644 src/chatgpt-bridge/hosts/codex/legacy-registry.ts create mode 100644 src/chatgpt-bridge/provider/adapter.ts create mode 100644 tests/chatgpt-bridge/chatgpt-bridge-codex-host.test.ts create mode 100644 tests/chatgpt-bridge/chatgpt-bridge-core.test.ts create mode 100644 tests/chatgpt-bridge/chatgpt-bridge-dsh-host.test.ts create mode 100644 tests/chatgpt-bridge/chatgpt-bridge-provider-adapter.test.ts diff --git a/devlog/_fin/260905_test_modularization_and_windows/001_test_inventory.md b/devlog/_fin/260905_test_modularization_and_windows/001_test_inventory.md index 44defb0acd3..ffbf5a291c9 100644 --- a/devlog/_fin/260905_test_modularization_and_windows/001_test_inventory.md +++ b/devlog/_fin/260905_test_modularization_and_windows/001_test_inventory.md @@ -415,6 +415,10 @@ Sum of the table: **1061**. Zero leftover. `phase100-native-parity.test.ts` +#### `tests/chatgpt-bridge/` (4) + +`chatgpt-bridge-core.test.ts`, `chatgpt-bridge-codex-host.test.ts`, `chatgpt-bridge-dsh-host.test.ts`, `chatgpt-bridge-provider-adapter.test.ts` — added 2026-09-11 (chatgpt-bridge integration, feat/chatgpt-bridge) + ## 3. Cross-cutting coupling ### 3.A tests/helpers imported by how many tests (unique files) diff --git a/docs-site/src/content/docs/fr/getting-started/quickstart.md b/docs-site/src/content/docs/fr/getting-started/quickstart.md index 73512a4a6c8..06099448225 100644 --- a/docs-site/src/content/docs/fr/getting-started/quickstart.md +++ b/docs-site/src/content/docs/fr/getting-started/quickstart.md @@ -13,7 +13,7 @@ ocx init `ocx init` vous accompagne dans les étapes suivantes : -1. **Choix d’un fournisseur** — sélectionnez l’un des 96 préréglages intégrés au registre, ou `custom` pour saisir une +1. **Choix d’un fournisseur** — sélectionnez l’un des 97 préréglages intégrés au registre, ou `custom` pour saisir une URL de base et un adaptateur. 2. **Clé API** — collez une clé ou référencez une variable d’environnement telle que `${ANTHROPIC_API_KEY}`. 3. **Modèle par défaut** — pour les fournisseurs clés, locaux et personnalisés, acceptez le préréglage ou saisissez un identifiant de modèle. diff --git a/docs-site/src/content/docs/fr/guides/providers.md b/docs-site/src/content/docs/fr/guides/providers.md index 7e9115e0033..ef837c798bf 100644 --- a/docs-site/src/content/docs/fr/guides/providers.md +++ b/docs-site/src/content/docs/fr/guides/providers.md @@ -283,7 +283,7 @@ existante n'est pas concernée. ## 3. Catalogue des clés API -opencodex fournit 96 préréglages intégrés : 80 à clé, 12 OAuth, trois locaux et un préréglage par défaut de +opencodex fournit 97 préréglages intégrés : 80 à clé, 12 OAuth, quatre locaux et un préréglage par défaut de transfert ChatGPT. Dans le tableau de bord, le sélecteur **Ajouter un fournisseur** ouvre le tableau de bord du fournisseur à clé, valide la clé et l'enregistre ; la validation dépend du fournisseur. Parmi les entrées notables : diff --git a/docs-site/src/content/docs/getting-started/quickstart.md b/docs-site/src/content/docs/getting-started/quickstart.md index a673daf5e82..69944caca58 100644 --- a/docs-site/src/content/docs/getting-started/quickstart.md +++ b/docs-site/src/content/docs/getting-started/quickstart.md @@ -18,7 +18,7 @@ ocx init `ocx init` walks you through: -1. **Pick a provider** — choose one of the 96 built-in registry presets or `custom` to type a base +1. **Pick a provider** — choose one of the 97 built-in registry presets or `custom` to type a base URL and adapter. 2. **API key** — paste a key, or reference an environment variable like `${ANTHROPIC_API_KEY}`. 3. **Default model** — for key, local, and custom providers, accept the preset or enter a model id. diff --git a/docs-site/src/content/docs/guides/providers.md b/docs-site/src/content/docs/guides/providers.md index e8cba6c928d..6670271e48a 100644 --- a/docs-site/src/content/docs/guides/providers.md +++ b/docs-site/src/content/docs/guides/providers.md @@ -412,7 +412,7 @@ selectors, then retry. Signing in from a machine with no existing `kiro-cli` ses ## 3. API-key catalog -opencodex ships 96 built-in presets: 80 key-based, 12 OAuth, three local, and one default +opencodex ships 97 built-in presets: 80 key-based, 12 OAuth, four local, and one default ChatGPT-forward preset. The dashboard's **Add provider** picker opens a key provider's dashboard, validates the key, and stores it; validation is provider-specific. Notable entries: diff --git a/docs-site/src/content/docs/ja/getting-started/quickstart.md b/docs-site/src/content/docs/ja/getting-started/quickstart.md index d9d83201963..0661da7173b 100644 --- a/docs-site/src/content/docs/ja/getting-started/quickstart.md +++ b/docs-site/src/content/docs/ja/getting-started/quickstart.md @@ -18,7 +18,7 @@ ocx init `ocx init` では次の手順を説明します。 -1. **プロバイダーを選択してください** — 96 個の組み込みレジストリプリセットのいずれか、または `custom` を選択してベース URL とアダプターを入力します。 +1. **プロバイダーを選択してください** — 97 個の組み込みレジストリプリセットのいずれか、または `custom` を選択してベース URL とアダプターを入力します。 2. **API キー** — キーを貼り付けるか、`${ANTHROPIC_API_KEY}` のような環境変数を参照します。 3. **デフォルト モデル** — キー、ローカル、カスタム プロバイダーの場合は、プリセットを受け入れるか、モデル ID を入力します。 4. **プロキシ ポート** — デフォルトは `10100` です。 diff --git a/docs-site/src/content/docs/ja/guides/providers.md b/docs-site/src/content/docs/ja/guides/providers.md index adad28ad00c..ccd5fbcb987 100644 --- a/docs-site/src/content/docs/ja/guides/providers.md +++ b/docs-site/src/content/docs/ja/guides/providers.md @@ -185,7 +185,7 @@ Kiro のログインには Kiro CLI が必要です。Unix では `curl -fsSL ht ## 3. API キーカタログ -opencodex には組み込みプリセットが 96 個含まれています。キー方式 80、OAuth 12、ローカル 3、 +opencodex には組み込みプリセットが 97 個含まれています。キー方式 80、OAuth 12、ローカル 4、 デフォルト ChatGPT 転送プリセット 1 です。ダッシュボードの **Add provider** ピッカーはキー発行ページを開き、 入力したキーを検証した後保存します(検証はプロバイダー固有です)。主な項目は以下のとおりです: diff --git a/docs-site/src/content/docs/ko/getting-started/quickstart.md b/docs-site/src/content/docs/ko/getting-started/quickstart.md index 91d11f1eff3..cc53424c63e 100644 --- a/docs-site/src/content/docs/ko/getting-started/quickstart.md +++ b/docs-site/src/content/docs/ko/getting-started/quickstart.md @@ -18,7 +18,7 @@ ocx init `ocx init`은 다음 과정을 안내합니다: -1. **프로바이더 선택** — 내장 레지스트리 프리셋 96개 중 하나를 고르거나 `custom`을 선택해 base URL과 adapter를 직접 입력합니다. +1. **프로바이더 선택** — 내장 레지스트리 프리셋 97개 중 하나를 고르거나 `custom`을 선택해 base URL과 adapter를 직접 입력합니다. 2. **API 키** — 키를 붙여넣거나 `${ANTHROPIC_API_KEY}` 같은 환경 변수를 참조합니다. 3. **기본 모델** — 키, 로컬, custom 프로바이더에서는 프리셋을 그대로 쓰거나 모델 ID를 직접 입력합니다. 4. **프록시 포트** — 기본값은 `10100`입니다. diff --git a/docs-site/src/content/docs/ko/guides/providers.md b/docs-site/src/content/docs/ko/guides/providers.md index 0eea4be4b70..871200808d5 100644 --- a/docs-site/src/content/docs/ko/guides/providers.md +++ b/docs-site/src/content/docs/ko/guides/providers.md @@ -182,7 +182,7 @@ Kiro 로그인에는 Kiro CLI가 필요합니다. Unix에서는 `curl -fsSL http ## 3. API 키 카탈로그 -opencodex에는 빌트인 프리셋이 96개 들어 있습니다. 키 방식 80개, OAuth 12개, 로컬 3개, +opencodex에는 빌트인 프리셋이 97개 들어 있습니다. 키 방식 80개, OAuth 12개, 로컬 4개, 기본 ChatGPT 포워드 프리셋 1개입니다. 대시보드의 **Add provider** 선택기는 키 발급 페이지를 열고, 입력한 키를 검증한 뒤 저장합니다(검증은 프로바이더별로 다릅니다). 주요 항목은 다음과 같습니다: diff --git a/docs-site/src/content/docs/ru/getting-started/quickstart.md b/docs-site/src/content/docs/ru/getting-started/quickstart.md index 750d2fa7476..a8a0abc08b0 100644 --- a/docs-site/src/content/docs/ru/getting-started/quickstart.md +++ b/docs-site/src/content/docs/ru/getting-started/quickstart.md @@ -18,7 +18,7 @@ ocx init `ocx init` проведёт вас по следующим шагам: -1. **Выбор провайдера** — выберите один из 96 встроенных пресетов реестра или `custom`, чтобы +1. **Выбор провайдера** — выберите один из 97 встроенных пресетов реестра или `custom`, чтобы ввести базовый URL и адаптер вручную. 2. **API-ключ** — вставьте ключ или сошлитесь на переменную окружения вида `${ANTHROPIC_API_KEY}`. 3. **Модель по умолчанию** — для провайдеров с ключом, локальных и `custom` примите значение из diff --git a/docs-site/src/content/docs/ru/guides/providers.md b/docs-site/src/content/docs/ru/guides/providers.md index 53e86522e2b..ea78279ae32 100644 --- a/docs-site/src/content/docs/ru/guides/providers.md +++ b/docs-site/src/content/docs/ru/guides/providers.md @@ -198,7 +198,7 @@ Inline JSON и лишние позиционные аргументы откло ## 3. Каталог API-ключей -opencodex поставляется с 96 встроенными пресетами: 80 на основе ключей, 12 OAuth, три локальных и +opencodex поставляется с 97 встроенными пресетами: 80 на основе ключей, 12 OAuth, четыре локальных и один пресет ChatGPT-форварда по умолчанию. Селектор **Add provider** в дашборде открывает страницу выдачи ключей провайдера, проверяет ключ и сохраняет его; проверка зависит от провайдера. Наиболее заметные записи: diff --git a/docs-site/src/content/docs/tr/getting-started/quickstart.md b/docs-site/src/content/docs/tr/getting-started/quickstart.md index d73cb2bff12..fe77fcae635 100644 --- a/docs-site/src/content/docs/tr/getting-started/quickstart.md +++ b/docs-site/src/content/docs/tr/getting-started/quickstart.md @@ -14,7 +14,7 @@ ocx init `ocx init` adım adım size rehberlik eder: -1. **Bir sağlayıcı seçin** — yerleşik kayıt defterindeki 96 önayardan birini +1. **Bir sağlayıcı seçin** — yerleşik kayıt defterindeki 97 önayardan birini veya bir temel URL ile adaptör yazmak için `custom` seçeneğini belirleyin. 2. **API anahtarı** — bir anahtar yapıştırın veya `${ANTHROPIC_API_KEY}` gibi bir ortam değişkenine başvurun. diff --git a/docs-site/src/content/docs/tr/guides/providers.md b/docs-site/src/content/docs/tr/guides/providers.md index c49040b0960..56041dbf729 100644 --- a/docs-site/src/content/docs/tr/guides/providers.md +++ b/docs-site/src/content/docs/tr/guides/providers.md @@ -316,8 +316,8 @@ olmayan bir makineden oturum açmak bundan etkilenmez. ## 3. API anahtarı kataloğu -opencodex 96 yerleşik önayar ile birlikte gelir: 80 anahtar tabanlı, 12 -OAuth, üç yerel ve bir varsayılan ChatGPT iletme önayarı. Kontrol panelinin +opencodex 97 yerleşik önayar ile birlikte gelir: 80 anahtar tabanlı, 12 +OAuth, dört yerel ve bir varsayılan ChatGPT iletme önayarı. Kontrol panelinin **Sağlayıcı ekle** seçicisi bir anahtar sağlayıcısının kontrol panelini açar, anahtarı doğrular ve saklar; doğrulama sağlayıcıya özgüdür. Dikkate değer girdiler: diff --git a/docs-site/src/content/docs/zh-cn/getting-started/quickstart.md b/docs-site/src/content/docs/zh-cn/getting-started/quickstart.md index 14f76c58cb9..7a7d9d01c58 100644 --- a/docs-site/src/content/docs/zh-cn/getting-started/quickstart.md +++ b/docs-site/src/content/docs/zh-cn/getting-started/quickstart.md @@ -18,7 +18,7 @@ ocx init `ocx init` 会引导你完成： -1. **选择 provider** — 从内置 registry 的 96 个预设中选择一个，或选择 `custom` 手动输入 base URL 和 adapter。 +1. **选择 provider** — 从内置 registry 的 97 个预设中选择一个，或选择 `custom` 手动输入 base URL 和 adapter。 2. **API key** — 粘贴一个 key，或引用一个环境变量，例如 `${ANTHROPIC_API_KEY}`。 3. **默认模型** — 对于 key、本地和 custom provider，接受预设值或输入模型 id。 4. **代理端口** — 默认为 `10100`。 diff --git a/docs-site/src/content/docs/zh-cn/guides/providers.md b/docs-site/src/content/docs/zh-cn/guides/providers.md index d799f306705..faf642e30b2 100644 --- a/docs-site/src/content/docs/zh-cn/guides/providers.md +++ b/docs-site/src/content/docs/zh-cn/guides/providers.md @@ -173,7 +173,7 @@ Kiro 登录需要 Kiro CLI：Unix 使用 `curl -fsSL https://cli.kiro.dev/instal ## 3. API 密钥目录 -opencodex 内置 96 个预设：80 个密钥预设、12 个 OAuth 预设、3 个本地预设，以及 1 个默认的 +opencodex 内置 97 个预设：80 个密钥预设、12 个 OAuth 预设、4 个本地预设，以及 1 个默认的 ChatGPT 转发预设。仪表盘的 **Add provider** 选择器会打开密钥提供商的控制台，验证并保存密钥。 验证因提供商而异。主要条目包括： diff --git a/docs-site/src/content/docs/zh-tw/getting-started/quickstart.md b/docs-site/src/content/docs/zh-tw/getting-started/quickstart.md index a0e17af51a2..70607e4c8a6 100644 --- a/docs-site/src/content/docs/zh-tw/getting-started/quickstart.md +++ b/docs-site/src/content/docs/zh-tw/getting-started/quickstart.md @@ -13,7 +13,7 @@ ocx init `ocx init` 會引導你完成: -1. **選擇 provider** —— 從內建 registry 的 96 個預設中選擇一個，或選擇 `custom` 手動輸入 +1. **選擇 provider** —— 從內建 registry 的 97 個預設中選擇一個，或選擇 `custom` 手動輸入 base URL 和 adapter。 2. **API key** —— 貼上一個 key,或引用一個環境變數,例如 `${ANTHROPIC_API_KEY}`。 3. **預設模型** —— 對於 API key、本機和 custom provider，可接受預設值或輸入模型 id。 diff --git a/docs-site/src/content/docs/zh-tw/guides/providers.md b/docs-site/src/content/docs/zh-tw/guides/providers.md index 44263dc002f..c74af9db19c 100644 --- a/docs-site/src/content/docs/zh-tw/guides/providers.md +++ b/docs-site/src/content/docs/zh-tw/guides/providers.md @@ -239,7 +239,7 @@ database 並移除目前的 WAL、SHM 與 journal sidecar，再發布先前的 s ## 3. API 金鑰目錄 -opencodex 內建 96 個 preset：80 個 key-based、12 個 OAuth、3 個 local，以及 1 個預設 ChatGPT-forward +opencodex 內建 97 個 preset：80 個 key-based、12 個 OAuth、4 個 local，以及 1 個預設 ChatGPT-forward preset。儀表板的 **Add provider** picker 會開啟 key provider 的 dashboard、驗證金鑰並儲存；驗證方式 依 provider 而異。主要條目如下。 diff --git a/extensions/dsh-chatgpt-bridge/package.json b/extensions/dsh-chatgpt-bridge/package.json new file mode 100644 index 00000000000..345a7a7c230 --- /dev/null +++ b/extensions/dsh-chatgpt-bridge/package.json @@ -0,0 +1,27 @@ +{ + "name": "@opencodex/dsh-chatgpt-bridge", + "version": "0.1.0", + "description": "DSH host+client plugin: persistent session bindings between specific DSH sessions and a normal ChatGPT chat, maintained by OpenCodex chatgpt-bridge", + "license": "MIT", + "type": "module", + "main": "dist/host.js", + "types": "dist/host.d.ts", + "exports": { + ".": "./dist/host.js", + "./client": "./dist/client.js" + }, + "files": ["dist", "README.md"], + "scripts": { + "build": "bunx tsc -p tsconfig.json" + }, + "peerDependencies": { + "@deepseek-ai/dsh-agent": "0.1.2-rc.1", + "@deepseek-ai/dsh-api-session-controller": "0.1.2-rc.1" + }, + "dsh": { + "platform": "web", + "client": { + "inject": [] + } + } +} diff --git a/extensions/dsh-chatgpt-bridge/src/host.ts b/extensions/dsh-chatgpt-bridge/src/host.ts new file mode 100644 index 00000000000..03b9a258a62 --- /dev/null +++ b/extensions/dsh-chatgpt-bridge/src/host.ts @@ -0,0 +1,220 @@ +/** + * DSH host plugin: persistent session bindings between exact DSH sessions and + * a normal ChatGPT chat, controlled by the OpenCodex chatgpt-bridge core. + * + * DSH API surface (verified against installed @deepseek-ai/dsh@0.1.2-rc.1, + * see CCW docs/handoff P0 capability matrix §2): + * - ctx.agents.get(id) → live Agent | undefined; agent.followup(message) + * enqueues a next-turn message and wakes the driver; + * - ctx.on("agent/inbox/claimed", …) payload carries `turn` — the anchor that + * correlates an inbox item to its turn; + * - ctx.on("session/event", (session, event)) streams + * assistant/message{turn} / turn/end{turn} for attribution; + * - subagent-owned sessions are rejected on three layers (origin, lineage, + * runtime ownership) — bridging never hijacks a child session; + * - ctx.storage persists the binding map; ctx.webServer.register exposes the + * control endpoints to the OpenCodex core only (loopback + token). + */ + +export interface DshUserMessage { + readonly id: string; + readonly role: "user"; + readonly content: string; +} + +export interface DshAgent { + readonly id: string; + followup(message: DshUserMessage): void; + whenIdle(): Promise; +} + +export interface DshSessionHeader { + readonly id: string; + readonly origin?: "user" | "subagent"; + readonly parentSession?: string; + readonly agentPreset?: string; +} + +export interface DshSession { + readonly header: DshSessionHeader; +} + +export interface DshAgentsRegistry { + get(id: string): DshAgent | undefined; + isOwnedBy(childId: string, parentId: string): boolean; +} + +export interface DshPluginContext { + readonly agents: DshAgentsRegistry; + on(event: "agent/inbox/claimed", listener: (payload: { agent: DshAgent; message: { id: string }; turn: number }) => void): void; + on(event: "agent/inbox/inserted", listener: (payload: { agent: DshAgent; message: { id: string } }) => void): void; + on(event: "agent/inbox/discarded", listener: (payload: { agent: DshAgent; message: { id: string } }) => void): void; + on(event: "session/event", listener: (session: DshSession, event: DshSessionEvent) => void): void; + storage: { + get(key: string): Promise; + set(key: string, value: T): Promise; + }; + webServer: { + register(route: { kind: "prefix"; path: string; handler: (request: BridgeControlRequest) => Promise }): void; + }; +} + +export interface BridgeControlRequest { + method: "GET" | "POST"; + path: string; + /** Shared secret issued to the OpenCodex core; never the DSH user key. */ + headers: Record; + body?: unknown; +} + +export interface BridgeControlResponse { + status: number; + body: unknown; +} + +export type DshSessionEvent = + | { type: "turn/start"; turn: number } + | { type: "turn/end"; turn: number; reason: string } + | { type: "assistant/message"; turn: number; step: number; message: { id: string } } + | { type: "tool/call"; turn: number; step: number; callId: string; name: string } + | { type: "tool/result"; turn: number; step: number; message: { id: string } } + | { type: "user/message"; message: { id: string } } + | { type: string; [key: string]: unknown }; + +export interface BridgeBindingState { + sessionId: string; + chatConversationId: string; + boundAt: string; + /** Latest correlated turn per delivered inbox item. */ + lastDeliveredInboxItemId: string | null; + lastClaimedTurn: number | null; + lastAssistantMessageId: string | null; + lastTurnEnded: number | null; +} + +export const TOKEN_HEADER = "x-bridge-token"; + +export interface DshBridgePluginConfig { + /** Credential reference resolved by ctx.credentials; never a plaintext secret here. */ + controlTokenRef?: string; + controlToken?: string; +} + +export function createBridgePlugin(ctx: DshPluginContext, config: DshBridgePluginConfig) { + let controlToken = config.controlToken ?? ""; + const states = new Map(); + + const loadState = async (sessionId: string): Promise => { + if (states.has(sessionId)) return states.get(sessionId); + const persisted = await ctx.storage.get(`chatgpt-bridge:${sessionId}`); + if (persisted) states.set(sessionId, persisted); + return persisted; + }; + + const saveState = async (sessionId: string): Promise => { + const state = states.get(sessionId); + if (state) await ctx.storage.set(`chatgpt-bridge:${sessionId}`, state); + }; + + // Correlation: inbox item → claimed turn → assistant output → turn end. + ctx.on("agent/inbox/claimed", async ({ agent, message, turn }) => { + const state = await loadState(String(agent.id)); + if (!state || state.lastDeliveredInboxItemId !== message.id) return; + state.lastClaimedTurn = turn; + await saveState(String(agent.id)); + }); + ctx.on("session/event", async (session, event) => { + if (event.type !== "assistant/message" && event.type !== "turn/end") return; + const state = await loadState(String(session.header.id)); + if (!state || state.lastClaimedTurn === null || event.turn !== state.lastClaimedTurn) return; + if (event.type === "assistant/message") { + state.lastAssistantMessageId = event.message.id; + } else { + state.lastTurnEnded = event.turn; + } + await saveState(String(session.header.id)); + }); + + /** + * Three-layer rejection mirroring hasApiSessionSubagentOwner (0.1.2-rc.1): + * durable origin, durable parent lineage, and runtime ownership. A child + * session is rejected even when its parent agent is no longer live — the + * lineage alone is disqualifying for bridging. + */ + const isSubagentOwned = (header: DshSessionHeader): boolean => { + if (header.origin === "subagent") return true; + return header.parentSession !== undefined; + }; + + const requireToken = (request: BridgeControlRequest): boolean => { + if (!controlToken) return false; + return request.headers[TOKEN_HEADER] === controlToken; + }; + + const handler = async (request: BridgeControlRequest): Promise => { + if (!requireToken(request)) return { status: 401, body: { ok: false, code: "AUTH_REQUIRED" } }; + const sessionId = request.path.split("/").filter(Boolean)[1] ?? ""; + + if (request.method === "GET" && request.path.endsWith("/binding")) { + const state = await loadState(sessionId); + if (!state) return { status: 404, body: { ok: false, code: "BINDING_NOT_FOUND" } }; + return { status: 200, body: { ok: true, binding: state } }; + } + + if (request.method === "POST" && request.path.endsWith("/deliver")) { + const body = request.body as { message?: string; chatConversationId?: string; inboxItemId?: string } | undefined; + const message = body?.message ?? ""; + if (!message.trim()) return { status: 400, body: { ok: false, code: "EMPTY_PROMPT" } }; + const agent = ctx.agents.get(sessionId); + if (!agent) return { status: 404, body: { ok: false, code: "TARGET_NOT_FOUND" } }; + + const state: BridgeBindingState = (await loadState(sessionId)) ?? { + sessionId, + chatConversationId: body?.chatConversationId ?? "", + boundAt: new Date().toISOString(), + lastDeliveredInboxItemId: null, + lastClaimedTurn: null, + lastAssistantMessageId: null, + lastTurnEnded: null, + }; + if (body?.chatConversationId && state.chatConversationId && state.chatConversationId !== body.chatConversationId) { + return { status: 409, body: { ok: false, code: "BINDING_CHANGED" } }; + } + + // A prior delivery whose turn was claimed but not yet ended means the + // host is mid-turn: refuse instead of steering or queueing on top. + if (state.lastClaimedTurn !== null && state.lastTurnEnded === null) { + return { status: 409, body: { ok: false, code: "TARGET_ACTIVE" } }; + } + + const inboxItemId = body?.inboxItemId ?? `bridge-${crypto.randomUUID()}`; + const userMessage: DshUserMessage = { id: inboxItemId, role: "user", content: message }; + state.lastDeliveredInboxItemId = inboxItemId; + state.lastClaimedTurn = null; + state.lastAssistantMessageId = null; + state.lastTurnEnded = null; + if (body?.chatConversationId) state.chatConversationId = body.chatConversationId; + states.set(sessionId, state); + agent.followup(userMessage); + await saveState(sessionId); + return { + status: 202, + body: { ok: true, state: "SUBMITTED_UNVERIFIED", inboxItemId }, + }; + } + + return { status: 404, body: { ok: false, code: "TARGET_NOT_FOUND" } }; + }; + + return { + /** Exposed for the DSH host to wire into its lifecycle; see README. */ + apply() { + ctx.webServer.register({ kind: "prefix", path: "/chatgpt-bridge", handler }); + }, + handler, + isSubagentOwned, + setControlToken(token: string) { + controlToken = token; + }, + }; +} diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 2e31c8b2431..04eeb7c5227 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -211,6 +211,10 @@ "adapter-inner-send-budget.test.ts": "adapters", "physical-send.test.ts": "adapters", "adapter-registry-authority.test.ts": "adapters", + "chatgpt-bridge-core.test.ts": "chatgpt-bridge", + "chatgpt-bridge-codex-host.test.ts": "chatgpt-bridge", + "chatgpt-bridge-dsh-host.test.ts": "chatgpt-bridge", + "chatgpt-bridge-provider-adapter.test.ts": "chatgpt-bridge", "adapter-resolve.test.ts": "server", "adapter-tool-conformance.test.ts": "adapters", "adapter-usage.test.ts": "adapters", diff --git a/src/adapters/registry.ts b/src/adapters/registry.ts index f077f92c2c5..46d67567275 100644 --- a/src/adapters/registry.ts +++ b/src/adapters/registry.ts @@ -6,6 +6,7 @@ import { createCodeBuddyAdapter } from "./codebuddy/adapter"; import { createQoderAdapter } from "./qoder/adapter"; import { createCommandCodeAdapter } from "./command-code"; import { createCursorAdapter } from "./cursor"; +import { createChatGptWebAdapter } from "../chatgpt-bridge/provider/adapter"; import { createDevinAdapter } from "./devin"; import { createGoogleAdapter } from "./google"; import { createKiroAdapter } from "./kiro"; @@ -43,7 +44,8 @@ export type AdapterWire = | "google" | "kiro" | "cursor" - | "devin"; + | "devin" + | "chatgpt-web"; export type AdapterMutationContract = | "codex-owned" @@ -130,6 +132,11 @@ export const ADAPTER_REGISTRY = { mutation: "codex-owned", create: (provider: OcxProviderConfig, context: AdapterFactoryContext) => createDevinAdapter(provider, context), }, + "chatgpt-web": { + wire: "chatgpt-web", + mutation: "codex-owned", + create: (provider: OcxProviderConfig, _context: AdapterFactoryContext) => createChatGptWebAdapter(provider), + }, "mimo-free": { contractParent: "openai-chat", create: (provider: OcxProviderConfig, _context: AdapterFactoryContext) => createMimoFreeAdapter(provider), diff --git a/src/chatgpt-bridge/contracts/index.ts b/src/chatgpt-bridge/contracts/index.ts new file mode 100644 index 00000000000..54d0cbaf43d --- /dev/null +++ b/src/chatgpt-bridge/contracts/index.ts @@ -0,0 +1,151 @@ +import { z } from "zod"; + +/** Wire-level error codes shared by Bridge.* contracts. */ +export const BRIDGE_ERROR_CODES = [ + "BINDING_CHANGED", + "REVISION_CONFLICT", + "BINDING_REVISION_CONFLICT", + "OPERATION_ID_CONFLICT", + "AUTH_REQUIRED", + "CAPABILITY_EXPIRED", + "CAPABILITY_REVOKED", + "TARGET_ACTIVE", + "TARGET_NOT_FOUND", + "BINDING_NOT_FOUND", + "HOST_OFFLINE", + "ATTACHMENT_UNAVAILABLE", + "ATTACHMENT_REQUIRED", + "DELIVERY_UNKNOWN", + "SEND_IN_PROGRESS", + "SEND_PAUSED", + "DUPLICATE_PROMPT", + "EMPTY_PROMPT", + "PROMPT_TOO_LARGE", + "INVALID_CHATGPT_URL", + "INVALID_REGISTRY", + "PROTOCOL_UNSUPPORTED", + "ENVIRONMENT_UNTRUSTED", + "CONTEXT_INCOMPATIBLE", + "LOOP_DETECTED", +] as const; + +export type BridgeErrorCode = (typeof BRIDGE_ERROR_CODES)[number]; + +export class BridgeCoreError extends Error { + readonly code: BridgeErrorCode; + readonly details?: Record; + + constructor(code: BridgeErrorCode, message?: string, details?: Record) { + super(message ?? code); + this.name = "BridgeCoreError"; + this.code = code; + this.details = details; + } +} + +/** Definite non-delivery outcomes: the send provably never reached the chat. */ +export const DEFINITE_NON_DELIVERY_CODES: readonly BridgeErrorCode[] = [ + "EMPTY_PROMPT", + "SEND_PAUSED", + "TARGET_ACTIVE", + "PROMPT_TOO_LARGE", + "INVALID_CHATGPT_URL", + "CAPABILITY_EXPIRED", + "CAPABILITY_REVOKED", + "ATTACHMENT_REQUIRED", +]; + +export const CHATGPT_CONVERSATION_URL_PATTERN = + /^https:\/\/chatgpt\.com\/c\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +export function parseChatGptConversationUrl(url: string): { conversationId: string; canonicalUrl: string } { + const trimmed = url.trim().replace(/[#?].*$/, ""); + if (!CHATGPT_CONVERSATION_URL_PATTERN.test(trimmed)) { + throw new BridgeCoreError("INVALID_CHATGPT_URL", `Not a normal ChatGPT conversation URL: ${url}`); + } + const conversationId = trimmed.split("/").pop()!.toLowerCase(); + return { conversationId, canonicalUrl: `https://chatgpt.com/c/${conversationId}` }; +} + +export const HOST_KINDS = ["codex", "dsh"] as const; +export type HostKind = (typeof HOST_KINDS)[number]; + +export const LIFECYCLES = ["active", "paused", "revoked", "unbound"] as const; +export type BridgeLifecycle = (typeof LIFECYCLES)[number]; + +export const ATTACHMENT_STATES = ["pending", "attached", "readable"] as const; +export type BridgeAttachmentState = (typeof ATTACHMENT_STATES)[number]; + +export const DELIVERY_STATES = ["reserved", "delivered", "not-delivered", "unknown"] as const; +export type BridgeDeliveryState = (typeof DELIVERY_STATES)[number]; + +export const hostRefSchema = z.object({ + kind: z.enum(HOST_KINDS), + /** Durable host instance id; never a PID, port, or tab handle. */ + instanceId: z.string().min(1), + /** Exact Codex task id / DSH session id. */ + targetId: z.string().min(1), + /** Verified workspace identity; never a model-reported path. */ + workspaceRef: z.string().min(1), +}); + +export const bindingSchema = z.object({ + schemaVersion: z.literal(1), + bindingId: z.string().uuid(), + ownerRef: z.string().min(1), + host: hostRefSchema, + chat: z.object({ + conversationId: z.string().min(1), + canonicalUrl: z.string().regex(CHATGPT_CONVERSATION_URL_PATTERN), + kind: z.literal("normal-chat"), + }), + source: z.object({ + kind: z.enum(["legacy-codex", "bridge-v1"]), + locator: z.string().min(1), + }), + revision: z.number().int().nonnegative(), + epoch: z.number().int().nonnegative(), + lifecycle: z.enum(LIFECYCLES), + attachmentState: z.enum(ATTACHMENT_STATES), + /** sha256 of the controller capability; plaintext never stored here. */ + capabilityHash: z.string().nullable().default(null), + capabilityExpiresAt: z.string().nullable().default(null), + proof: z + .object({ + attachment: z.string().nullable().default(null), + hostRead: z.string().nullable().default(null), + chatRead: z.string().nullable().default(null), + checkedAt: z.string().nullable().default(null), + }) + .default(() => ({ attachment: null, hostRead: null, chatRead: null, checkedAt: null })), + lastReceiptId: z.string().nullable().default(null), + createdAt: z.string(), + updatedAt: z.string(), +}); + +export type BridgeBinding = z.infer; + +export const BRIDGE_PROTOCOL_VERSION = 1; + +/** Management actions accepted by the unified manage entry point. */ +export const MANAGEMENT_ACTIONS = [ + "create", + "attach", + "pause", + "resume", + "renew", + "revoke", + "unbind", + "replace", +] as const; +export type BridgeManagementAction = (typeof MANAGEMENT_ACTIONS)[number]; + +export interface OperationReceipt { + operationId: string; + bindingId: string; + action: BridgeManagementAction; + revision: number; + outcome: "applied" | "alreadyApplied"; + errorCode?: BridgeErrorCode; + createdAt: string; +} diff --git a/src/chatgpt-bridge/core/store.ts b/src/chatgpt-bridge/core/store.ts new file mode 100644 index 00000000000..7fe3f6b6369 --- /dev/null +++ b/src/chatgpt-bridge/core/store.ts @@ -0,0 +1,584 @@ +import { Database } from "bun:sqlite"; +import { createHash, randomUUID } from "node:crypto"; +import { + BridgeCoreError, + type BridgeBinding, + type BridgeDeliveryState, + type BridgeLifecycle, + type BridgeManagementAction, + type OperationReceipt, + parseChatGptConversationUrl, + DEFINITE_NON_DELIVERY_CODES, + type BridgeErrorCode, +} from "../contracts"; + +/** + * Durations follow the conservative boundaries validated by the legacy bridge + * (bridge-lib.mjs): a pending send older than the reservation window must be + * treated as outcome-unknown, and repeating an identical prompt inside the + * duplicate guard window is rejected instead of re-sent. + */ +const RESERVATION_STALE_MS = 120_000; +const DUPLICATE_GUARD_MS = 120_000; +const MAX_OPERATIONS_PER_BINDING = 40; + +export interface CreateBindingInput { + bindingId?: string; + ownerRef: string; + host: BridgeBinding["host"]; + chatUrl: string; + capabilityHash?: string | null; + capabilityExpiresAt?: string | null; + operationId: string; + expectedRegistryRevision?: number; +} + +export interface ManageBindingInput { + bindingId: string; + action: Exclude; + operationId: string; + expectedRevision: number; + replacement?: CreateBindingInput; + capabilityHash?: string | null; + capabilityExpiresAt?: string | null; +} + +export interface ReserveDeliveryInput { + bindingId: string; + operationId: string; + direction: "host-to-chat" | "chat-to-host"; + sourceMessageId: string; + prompt: string; +} + +export interface DeliveryReservation { + reservationId: string; + bindingId: string; + bindingEpoch: number; + state: BridgeDeliveryState; + reservedAt: string; +} + +export interface SettleDeliveryInput { + reservationId: string; + /** Explicit caller decision; "unknown" must never be downgraded automatically. */ + outcome: "delivered" | "not-delivered" | "unknown"; + failureCode?: BridgeErrorCode; + receiptId?: string; + operator?: string; +} + +const now = () => new Date().toISOString(); + +/** + * Durable binding store. Single writer, versioned schema, no message bodies: + * only digests, ids, receipts, and lifecycle facts live here. + */ +export class BridgeBindingStore { + private readonly db: Database; + + constructor( + database: Database | string, + private readonly options: { now?: () => string; reservationStaleMs?: number; duplicateGuardMs?: number } = {}, + ) { + this.db = typeof database === "string" ? new Database(database) : database; + this.db.exec("PRAGMA journal_mode = WAL;"); + this.migrate(); + } + + private migrate(): void { + this.db.exec(` + CREATE TABLE IF NOT EXISTS chatgpt_bridge_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS chatgpt_bridge_bindings ( + binding_id TEXT PRIMARY KEY, + host_kind TEXT NOT NULL, + host_instance_id TEXT NOT NULL, + host_target_id TEXT NOT NULL, + host_workspace_ref TEXT NOT NULL, + chat_conversation_id TEXT NOT NULL UNIQUE, + chat_canonical_url TEXT NOT NULL, + owner_ref TEXT NOT NULL, + source_kind TEXT NOT NULL, + source_locator TEXT NOT NULL, + revision INTEGER NOT NULL, + epoch INTEGER NOT NULL, + lifecycle TEXT NOT NULL, + attachment_state TEXT NOT NULL, + capability_hash TEXT, + capability_expires_at TEXT, + proof_attachment TEXT, + proof_host_read TEXT, + proof_chat_read TEXT, + proof_checked_at TEXT, + last_receipt_id TEXT, + replaced_by TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS chatgpt_bridge_operations ( + operation_id TEXT PRIMARY KEY, + binding_id TEXT NOT NULL, + action TEXT NOT NULL, + request_digest TEXT NOT NULL, + revision INTEGER NOT NULL, + outcome TEXT NOT NULL, + error_code TEXT, + created_at TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS chatgpt_bridge_deliveries ( + reservation_id TEXT PRIMARY KEY, + binding_id TEXT NOT NULL, + operation_id TEXT NOT NULL, + direction TEXT NOT NULL, + source_message_id TEXT NOT NULL, + prompt_digest TEXT NOT NULL, + binding_epoch INTEGER NOT NULL, + state TEXT NOT NULL, + failure_code TEXT, + receipt_id TEXT, + resolved_by TEXT, + reserved_at TEXT NOT NULL, + settled_at TEXT + ); + CREATE INDEX IF NOT EXISTS idx_bridge_deliveries_binding ON chatgpt_bridge_deliveries(binding_id, state); + `); + this.db + .prepare( + "INSERT INTO chatgpt_bridge_meta(key, value) VALUES('schema_version', '1') ON CONFLICT(key) DO NOTHING", + ) + .run(); + } + + close(): void { + this.db.close(); + } + + private rowToBinding(row: Record): BridgeBinding { + return { + schemaVersion: 1, + bindingId: row.binding_id as string, + ownerRef: row.owner_ref as string, + host: { + kind: row.host_kind as BridgeBinding["host"]["kind"], + instanceId: row.host_instance_id as string, + targetId: row.host_target_id as string, + workspaceRef: row.host_workspace_ref as string, + }, + chat: { + conversationId: row.chat_conversation_id as string, + canonicalUrl: row.chat_canonical_url as string, + kind: "normal-chat", + }, + source: { + kind: row.source_kind as BridgeBinding["source"]["kind"], + locator: row.source_locator as string, + }, + revision: row.revision as number, + epoch: row.epoch as number, + lifecycle: row.lifecycle as BridgeLifecycle, + attachmentState: row.attachment_state as BridgeBinding["attachmentState"], + capabilityHash: (row.capability_hash as string) ?? null, + capabilityExpiresAt: (row.capability_expires_at as string) ?? null, + proof: { + attachment: (row.proof_attachment as string) ?? null, + hostRead: (row.proof_host_read as string) ?? null, + chatRead: (row.proof_chat_read as string) ?? null, + checkedAt: (row.proof_checked_at as string) ?? null, + }, + lastReceiptId: (row.last_receipt_id as string) ?? null, + createdAt: row.created_at as string, + updatedAt: row.updated_at as string, + }; + } + + private fetchBindingRow(bindingId: string): Record | null { + return ( + (this.db.prepare("SELECT * FROM chatgpt_bridge_bindings WHERE binding_id = ?").get(bindingId) as + | Record + | null) ?? null + ); + } + + getBinding(bindingId: string): BridgeBinding | null { + const row = this.fetchBindingRow(bindingId); + return row ? this.rowToBinding(row) : null; + } + + listBindings(): BridgeBinding[] { + const rows = this.db + .prepare("SELECT * FROM chatgpt_bridge_bindings ORDER BY created_at") + .all() as Record[]; + return rows.map(row => this.rowToBinding(row)); + } + + getOperation(operationId: string): (OperationReceipt & { requestDigest: string }) | null { + const row = this.db + .prepare("SELECT * FROM chatgpt_bridge_operations WHERE operation_id = ?") + .get(operationId) as Record | undefined; + if (!row) return null; + return { + operationId: row.operation_id as string, + bindingId: row.binding_id as string, + action: row.action as BridgeManagementAction, + requestDigest: row.request_digest as string, + revision: row.revision as number, + outcome: row.outcome as OperationReceipt["outcome"], + errorCode: (row.error_code as BridgeErrorCode) ?? undefined, + createdAt: row.created_at as string, + }; + } + + private recordOperation( + operationId: string, + bindingId: string, + action: BridgeManagementAction, + requestDigest: string, + revision: number, + outcome: OperationReceipt["outcome"], + errorCode?: BridgeErrorCode, + ): OperationReceipt { + const created = now(); + this.db + .prepare( + `INSERT INTO chatgpt_bridge_operations + (operation_id, binding_id, action, request_digest, revision, outcome, error_code, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .run(operationId, bindingId, action, requestDigest, revision, outcome, errorCode ?? null, created); + return { operationId, bindingId, action, revision, outcome, errorCode, createdAt: created }; + } + + private pruneOperations(bindingId: string): void { + this.db + .prepare( + `DELETE FROM chatgpt_bridge_operations WHERE binding_id = ? AND operation_id NOT IN ( + SELECT operation_id FROM chatgpt_bridge_operations WHERE binding_id = ? + ORDER BY created_at DESC LIMIT ? + )`, + ) + .run(bindingId, bindingId, MAX_OPERATIONS_PER_BINDING); + } + + private assertOperationIdempotent( + operationId: string, + action: BridgeManagementAction, + requestDigest: string, + ): OperationReceipt | null { + const existing = this.getOperation(operationId); + if (!existing) return null; + if (existing.requestDigest !== requestDigest || existing.action !== action) { + throw new BridgeCoreError("OPERATION_ID_CONFLICT", `operationId reused with a different request`); + } + return { + operationId, + bindingId: existing.bindingId, + action: existing.action, + revision: existing.revision, + outcome: "alreadyApplied", + errorCode: existing.errorCode, + createdAt: existing.createdAt, + }; + } + + createBinding(input: CreateBindingInput): { binding: BridgeBinding; receipt: OperationReceipt } { + const chat = parseChatGptConversationUrl(input.chatUrl); + const digest = JSON.stringify({ + ownerRef: input.ownerRef, + host: input.host, + chat, + capabilityHash: input.capabilityHash ?? null, + }); + const replay = this.assertOperationIdempotent(input.operationId, "create", digest); + if (replay) { + const existing = this.getBinding(replay.bindingId); + if (existing) return { binding: existing, receipt: replay }; + throw new BridgeCoreError("OPERATION_ID_CONFLICT", "create receipt has no binding"); + } + + const bindingId = input.bindingId ?? randomUUID(); + if (this.getBinding(bindingId)) { + throw new BridgeCoreError("OPERATION_ID_CONFLICT", "bindingId already exists"); + } + const duplicateChat = this.db + .prepare("SELECT binding_id FROM chatgpt_bridge_bindings WHERE chat_conversation_id = ?") + .get(chat.conversationId) as { binding_id: string } | undefined; + if (duplicateChat) { + throw new BridgeCoreError("OPERATION_ID_CONFLICT", "chat already bound to another target"); + } + + const created = now(); + const binding: BridgeBinding = { + schemaVersion: 1, + bindingId, + ownerRef: input.ownerRef, + host: input.host, + chat: { ...chat, kind: "normal-chat" }, + source: { kind: "bridge-v1", locator: `bridge:${bindingId}` }, + revision: 0, + epoch: 0, + lifecycle: "active", + attachmentState: "pending", + capabilityHash: input.capabilityHash ?? null, + capabilityExpiresAt: input.capabilityExpiresAt ?? null, + proof: { attachment: null, hostRead: null, chatRead: null, checkedAt: null }, + lastReceiptId: null, + createdAt: created, + updatedAt: created, + }; + this.db + .prepare( + `INSERT INTO chatgpt_bridge_bindings + (binding_id, host_kind, host_instance_id, host_target_id, host_workspace_ref, + chat_conversation_id, chat_canonical_url, owner_ref, source_kind, source_locator, + revision, epoch, lifecycle, attachment_state, capability_hash, capability_expires_at, + proof_attachment, proof_host_read, proof_chat_read, proof_checked_at, last_receipt_id, + created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'bridge-v1', ?, 0, 0, 'active', 'pending', ?, ?, NULL, NULL, NULL, NULL, NULL, ?, ?)`, + ) + .run( + binding.bindingId, + binding.host.kind, + binding.host.instanceId, + binding.host.targetId, + binding.host.workspaceRef, + binding.chat.conversationId, + binding.chat.canonicalUrl, + binding.ownerRef, + binding.source.locator, + binding.capabilityHash, + binding.capabilityExpiresAt, + binding.createdAt, + binding.updatedAt, + ); + const receipt = this.recordOperation(input.operationId, bindingId, "create", digest, 0, "applied"); + return { binding, receipt }; + } + + attachBinding( + bindingId: string, + input: { operationId: string; expectedRevision: number; attachmentProof: string }, + ): { binding: BridgeBinding; receipt: OperationReceipt } { + const digest = JSON.stringify({ bindingId, attachmentProof: input.attachmentProof }); + const replay = this.assertOperationIdempotent(input.operationId, "attach", digest); + if (replay) { + const existing = this.getBinding(bindingId); + if (!existing) throw new BridgeCoreError("BINDING_NOT_FOUND", bindingId); + return { binding: existing, receipt: replay }; + } + const binding = this.getBinding(bindingId); + if (!binding) throw new BridgeCoreError("BINDING_NOT_FOUND", bindingId); + if (binding.revision !== input.expectedRevision) { + throw new BridgeCoreError("BINDING_REVISION_CONFLICT", "expectedRevision does not match", { + expected: input.expectedRevision, + current: binding.revision, + }); + } + const updated = now(); + const nextState: BridgeBinding["attachmentState"] = + binding.attachmentState === "readable" ? "readable" : "attached"; + this.db + .prepare( + `UPDATE chatgpt_bridge_bindings SET attachment_state = ?, proof_attachment = ?, + revision = revision + 1, updated_at = ? WHERE binding_id = ?`, + ) + .run(nextState, input.attachmentProof, updated, bindingId); + const receipt = this.recordOperation(input.operationId, bindingId, "attach", digest, binding.revision + 1, "applied"); + return { binding: this.getBinding(bindingId)!, receipt }; + } + + manageBinding(input: ManageBindingInput): { binding: BridgeBinding | null; receipt: OperationReceipt } { + const digest = JSON.stringify({ + bindingId: input.bindingId, + action: input.action, + capabilityHash: input.capabilityHash ?? null, + replacementHost: input.replacement?.host ?? null, + replacementChat: input.replacement ? parseChatGptConversationUrl(input.replacement.chatUrl) : null, + }); + const replay = this.assertOperationIdempotent(input.operationId, input.action, digest); + if (replay) { + return { binding: this.getBinding(input.bindingId), receipt: replay }; + } + const binding = this.getBinding(input.bindingId); + if (!binding) throw new BridgeCoreError("BINDING_NOT_FOUND", input.bindingId); + if (binding.revision !== input.expectedRevision) { + throw new BridgeCoreError("BINDING_REVISION_CONFLICT", "expectedRevision does not match", { + expected: input.expectedRevision, + current: binding.revision, + }); + } + + const lifecycle = binding.lifecycle; + const nextLifecycle: BridgeLifecycle | null = (() => { + switch (input.action) { + case "pause": + return lifecycle === "active" ? "paused" : null; + case "resume": + return lifecycle === "paused" ? "active" : null; + case "revoke": + return lifecycle === "active" || lifecycle === "paused" ? "revoked" : null; + case "unbind": + return lifecycle === "revoked" || lifecycle === "active" || lifecycle === "paused" ? "unbound" : null; + case "renew": + return lifecycle === "active" || lifecycle === "paused" ? lifecycle : null; + case "replace": + return lifecycle === "active" || lifecycle === "paused" ? "revoked" : null; + default: + return null; + } + })(); + if (nextLifecycle === null) { + throw new BridgeCoreError("BINDING_REVISION_CONFLICT", `action ${input.action} invalid in lifecycle ${lifecycle}`); + } + + const updated = now(); + const epochBump = input.action === "revoke" || input.action === "replace" ? 1 : 0; + const replacedBy = input.action === "replace" ? input.replacement?.bindingId ?? null : null; + this.db + .prepare( + `UPDATE chatgpt_bridge_bindings SET lifecycle = ?, epoch = epoch + ?, + revision = revision + 1, + capability_hash = COALESCE(?, capability_hash), + capability_expires_at = COALESCE(?, capability_expires_at), + replaced_by = COALESCE(?, replaced_by), + updated_at = ? WHERE binding_id = ?`, + ) + .run( + nextLifecycle, + epochBump, + input.capabilityHash ?? null, + input.capabilityExpiresAt ?? null, + replacedBy, + updated, + input.bindingId, + ); + this.pruneOperations(input.bindingId); + const receipt = this.recordOperation( + input.operationId, + input.bindingId, + input.action, + digest, + binding.revision + 1, + "applied", + ); + return { binding: this.getBinding(input.bindingId), receipt }; + } + + reserveDelivery(input: ReserveDeliveryInput): DeliveryReservation { + const binding = this.getBinding(input.bindingId); + if (!binding) throw new BridgeCoreError("BINDING_NOT_FOUND", input.bindingId); + if (binding.lifecycle === "paused") throw new BridgeCoreError("SEND_PAUSED", "binding is paused"); + if (binding.lifecycle === "revoked" || binding.lifecycle === "unbound") { + throw new BridgeCoreError("CAPABILITY_REVOKED", `binding is ${binding.lifecycle}`); + } + const prompt = input.prompt; + if (prompt.length === 0) throw new BridgeCoreError("EMPTY_PROMPT", "prompt is empty"); + + const pending = this.db + .prepare( + "SELECT * FROM chatgpt_bridge_deliveries WHERE binding_id = ? AND state IN ('reserved','unknown') ORDER BY reserved_at DESC LIMIT 1", + ) + .get(input.bindingId) as Record | undefined; + if (pending) { + const reservedAt = Date.parse(pending.reserved_at as string); + const stale = Date.now() - reservedAt > (this.options.reservationStaleMs ?? RESERVATION_STALE_MS); + if (pending.state === "unknown" || (pending.state === "reserved" && stale)) { + if (pending.state === "reserved") { + this.db + .prepare("UPDATE chatgpt_bridge_deliveries SET state = 'unknown', settled_at = ? WHERE reservation_id = ?") + .run(now(), pending.reservation_id as string); + } + throw new BridgeCoreError("DELIVERY_UNKNOWN", "prior send outcome unknown; reconcile before sending again", { + reservationId: pending.reservation_id as string, + }); + } + throw new BridgeCoreError("SEND_IN_PROGRESS", "another send is pending on this binding", { + reservationId: pending.reservation_id as string, + }); + } + + const duplicate = this.db + .prepare( + "SELECT reservation_id, reserved_at FROM chatgpt_bridge_deliveries WHERE binding_id = ? AND prompt_digest = ? AND state = 'delivered' ORDER BY reserved_at DESC LIMIT 1", + ) + .get(input.bindingId, sha256(prompt)) as Record | undefined; + if (duplicate && Date.now() - Date.parse(duplicate.reserved_at as string) < (this.options.duplicateGuardMs ?? DUPLICATE_GUARD_MS)) { + throw new BridgeCoreError("DUPLICATE_PROMPT", "identical prompt delivered within the guard window"); + } + + const reservationId = randomUUID(); + const reservedAt = now(); + this.db + .prepare( + `INSERT INTO chatgpt_bridge_deliveries + (reservation_id, binding_id, operation_id, direction, source_message_id, prompt_digest, + binding_epoch, state, reserved_at) + VALUES (?, ?, ?, ?, ?, ?, ?, 'reserved', ?)`, + ) + .run( + reservationId, + input.bindingId, + input.operationId, + input.direction, + input.sourceMessageId, + sha256(prompt), + binding.epoch, + reservedAt, + ); + return { + reservationId, + bindingId: input.bindingId, + bindingEpoch: binding.epoch, + state: "reserved", + reservedAt, + }; + } + + settleDelivery(input: SettleDeliveryInput): { state: BridgeDeliveryState; receiptId: string } { + const row = this.db + .prepare("SELECT * FROM chatgpt_bridge_deliveries WHERE reservation_id = ?") + .get(input.reservationId) as Record | undefined; + if (!row) throw new BridgeCoreError("TARGET_NOT_FOUND", `unknown reservation ${input.reservationId}`); + if (row.state !== "reserved") { + throw new BridgeCoreError("DELIVERY_UNKNOWN", `reservation already settled as ${row.state}`); + } + const binding = this.getBinding(row.binding_id as string); + if (!binding || binding.epoch !== (row.binding_epoch as number)) { + throw new BridgeCoreError("BINDING_CHANGED", "binding epoch changed since reservation"); + } + if (input.outcome === "not-delivered") { + if (!input.failureCode || !DEFINITE_NON_DELIVERY_CODES.includes(input.failureCode)) { + throw new BridgeCoreError("DELIVERY_UNKNOWN", `${input.failureCode ?? "no code"} is not a definite non-delivery`); + } + } + if (input.outcome === "unknown" && !input.operator) { + throw new BridgeCoreError("DELIVERY_UNKNOWN", "manual unknown resolution requires operator identity"); + } + const receiptId = input.receiptId ?? randomUUID(); + this.db + .prepare( + "UPDATE chatgpt_bridge_deliveries SET state = ?, failure_code = ?, receipt_id = ?, resolved_by = ?, settled_at = ? WHERE reservation_id = ?", + ) + .run(input.outcome, input.failureCode ?? null, receiptId, input.operator ?? null, now(), input.reservationId); + this.db + .prepare("UPDATE chatgpt_bridge_bindings SET last_receipt_id = ?, updated_at = ? WHERE binding_id = ?") + .run(receiptId, now(), row.binding_id as string); + return { state: input.outcome, receiptId }; + } + + pendingDelivery(bindingId: string): { reservationId: string; state: BridgeDeliveryState } | null { + const row = this.db + .prepare( + "SELECT reservation_id, state FROM chatgpt_bridge_deliveries WHERE binding_id = ? AND state IN ('reserved','unknown') ORDER BY reserved_at DESC LIMIT 1", + ) + .get(bindingId) as Record | undefined; + if (!row) return null; + return { reservationId: row.reservation_id as string, state: row.state as BridgeDeliveryState }; + } +} + +function sha256(value: string): string { + return createHash("sha256").update(value, "utf8").digest("hex"); +} diff --git a/src/chatgpt-bridge/hosts/codex/devspace-mcp-client.ts b/src/chatgpt-bridge/hosts/codex/devspace-mcp-client.ts new file mode 100644 index 00000000000..4fdda50d4d1 --- /dev/null +++ b/src/chatgpt-bridge/hosts/codex/devspace-mcp-client.ts @@ -0,0 +1,277 @@ +import { BridgeCoreError, type BridgeErrorCode } from "../../contracts"; + +/** + * Minimal MCP client for the local DevSpace control plane (legacy bridge tools). + * + * Config invariants: the bearer token is supplied by OC configuration, never + * logged, never placed in URLs; the endpoint is the loopback-only DevSpace + * server (default 127.0.0.1:17676/mcp). Transport failures surface as + * BridgeCoreError so callers cannot mistake them for delivery outcomes. + */ +export interface DevSpaceMcpClientConfig { + baseUrl: string; + bearerToken: string; + fetchImpl?: typeof fetch; + timeoutMs?: number; +} + +const MCP_PROTOCOL_VERSION = "2025-06-18"; + +interface JsonRpcResponse { + jsonrpc: "2.0"; + id: number | string | null; + result?: Record; + error?: { code: number; message: string; data?: unknown }; +} + +export class DevSpaceMcpClient { + private nextId = 1; + private sessionId: string | null = null; + private sessionReady: Promise | null = null; + private readonly fetchImpl: typeof fetch; + + constructor(private readonly config: DevSpaceMcpClientConfig) { + this.fetchImpl = config.fetchImpl ?? fetch; + } + + async healthz(): Promise<{ ok: boolean; name?: string }> { + const url = this.config.baseUrl.replace(/\/mcp\/?$/, "") + "/healthz"; + const response = await this.fetchImpl(url, { signal: this.signal() }); + if (!response.ok) throw new BridgeCoreError("HOST_OFFLINE", `devspace healthz ${response.status}`); + return (await response.json()) as { ok: boolean; name?: string }; + } + + /** Streamable-HTTP MCP requires an initialize handshake to mint the session id. */ + private async ensureSession(): Promise { + if (this.sessionId) return; + if (!this.sessionReady) { + this.sessionReady = (async () => { + const response = await this.postRpc( + { jsonrpc: "2.0", id: this.nextId++, method: "initialize", params: { + protocolVersion: MCP_PROTOCOL_VERSION, + capabilities: {}, + clientInfo: { name: "opencodex-chatgpt-bridge", version: "0.1.0" }, + } }, + true, + ); + const header = response.headers.get("mcp-session-id"); + if (header) this.sessionId = header; + if (this.sessionId) { + await this.fetchImpl(this.config.baseUrl, { + method: "POST", + headers: this.headers(false), + body: JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" }), + signal: this.signal(), + }); + } + })(); + this.sessionReady.catch(() => { + this.sessionReady = null; + }); + } + await this.sessionReady; + } + + private headers(withSession: boolean): Record { + const headers: Record = { + "content-type": "application/json", + accept: "application/json, text/event-stream", + authorization: `Bearer ${this.config.bearerToken}`, + "mcp-protocol-version": MCP_PROTOCOL_VERSION, + }; + if (withSession && this.sessionId) headers["mcp-session-id"] = this.sessionId; + return headers; + } + + private async postRpc(body: unknown, captureSession = false): Promise { + const response = await this.fetchImpl(this.config.baseUrl, { + method: "POST", + headers: this.headers(!captureSession && this.sessionId !== null), + body: JSON.stringify(body), + signal: this.signal(), + }); + if (response.status === 401 || response.status === 403) { + throw new BridgeCoreError("AUTH_REQUIRED", `devspace rejected credentials (${response.status})`); + } + if (!response.ok) { + const detail = (await response.text().catch(() => "")).slice(0, 300); + throw new BridgeCoreError("HOST_OFFLINE", `devspace mcp ${response.status}${detail ? `: ${detail}` : ""}`); + } + return response; + } + + /** Invoke a DevSpace MCP tool and unwrap the bridge result envelope. */ + async callTool>(tool: string, args: Record = {}): Promise { + await this.ensureSession(); + const response = await this.postRpc({ + jsonrpc: "2.0", + id: this.nextId++, + method: "tools/call", + params: { name: tool, arguments: args }, + }); + const payload = await this.parseRpcResponse(response); + if (payload.error) { + throw new BridgeCoreError("HOST_OFFLINE", `devspace rpc error ${payload.error.code}: ${payload.error.message}`); + } + const result = payload.result ?? {}; + if (result.isError === true) { + throw this.bridgeFailure(tool, result); + } + return result as T; + } + + /** + * Streamable HTTP servers may answer a POST with an SSE stream carrying the + * JSON-RPC frame; unwrap either transport into a single response object. + */ + private async parseRpcResponse(response: Response): Promise { + const contentType = response.headers.get("content-type") ?? ""; + if (contentType.includes("text/event-stream")) { + const raw = await response.text(); + for (const line of raw.split(/\r?\n/)) { + if (!line.startsWith("data:")) continue; + const chunk = line.slice(5).trim(); + if (!chunk) continue; + try { + const frame = JSON.parse(chunk) as JsonRpcResponse; + if (frame.id !== null && frame.id !== undefined && (frame.result || frame.error)) return frame; + } catch { + // ignore keep-alive comments / non-JSON frames + } + } + throw new BridgeCoreError("HOST_OFFLINE", "devspace SSE stream carried no RPC response"); + } + return (await response.json()) as JsonRpcResponse; + } + + /** + * DevSpace bridge tools wrap bridge-lib failures as + * `{ ok: false, code, message, details? }` JSON inside the text content. + * Known codes map 1:1 onto Bridge error codes; unknown codes must not be + * reinterpreted as delivery outcomes. + */ + private bridgeFailure(tool: string, result: Record): BridgeCoreError { + const content = result.content as Array<{ type: string; text?: string }> | undefined; + const text = content?.find(c => c.type === "text")?.text ?? "{}"; + let parsed: { ok?: boolean; code?: string; message?: string; details?: Record }; + try { + parsed = JSON.parse(text); + } catch { + return new BridgeCoreError("HOST_OFFLINE", `${tool} returned unparseable failure payload`); + } + const code = parsed.code ?? ""; + if ((KNOWN_DEVSPACE_CODES as readonly string[]).includes(code)) { + return new BridgeCoreError(code as BridgeErrorCode, parsed.message ?? code, parsed.details); + } + return new BridgeCoreError("HOST_OFFLINE", `${tool} failure ${code}: ${parsed.message ?? ""}`); + } + + private signal(): AbortSignal | undefined { + if (!this.config.timeoutMs) return undefined; + return AbortSignal.timeout(this.config.timeoutMs); + } +} + +/** Codes shared verbatim between DevSpace bridge-lib and this module's contract. */ +const KNOWN_DEVSPACE_CODES = [ + "TARGET_ACTIVE", + "TARGET_NOT_FOUND", + "BINDING_NOT_FOUND", + "BINDING_CHANGED", + "BINDING_INACTIVE", + "BINDING_REVISION_CONFLICT", + "OPERATION_ID_CONFLICT", + "SEND_IN_PROGRESS", + "SEND_PAUSED", + "DUPLICATE_PROMPT", + "DELIVERY_UNKNOWN", + "EMPTY_PROMPT", + "PROMPT_TOO_LARGE", + "INVALID_CHATGPT_URL", + "INVALID_REGISTRY", + "CAPABILITY_EXPIRED", + "ATTACHMENT_REQUIRED", + "CAPABILITY_ROTATED", + "CAPABILITY_REVOKED", + "EXECUTOR_REQUIRED", + "EXECUTOR_NOT_FOUND", + "PIPE_TIMEOUT", + "PIPE_CLOSED", + "PIPE_ERROR", + "APP_RPC_ERROR", + "APP_TOOL_FAILED", + "TOOL_NOT_ALLOWED", + "STATE_ACL_FAILED", +] as const satisfies readonly string[]; + +export interface CodexBridgeStatus { + bindingId: string; + state: string; + codex?: Record; + raw: Record; +} + +/** + * Codex host adapter: persistent-collaboration operations against the exact + * legacy-bound task, executed through the existing DevSpace control plane. + * This module never re-implements discovery, locking, or delivery semantics — + * bridge-lib remains the sole authority for existing bindings. + */ +export class CodexHostAdapter { + constructor(private readonly client: DevSpaceMcpClient) {} + + async healthz(): Promise { + const health = await this.client.healthz(); + return health.ok === true; + } + + async attach(controllerId: string): Promise { + const result = await this.client.callTool("codex_bridge_attach", { controllerId }); + return this.toStatus(result); + } + + async status(controllerId: string): Promise { + const result = await this.client.callTool("codex_bridge_status", { controllerId }); + return this.toStatus(result); + } + + async read( + controllerId: string, + options: { turnLimit?: number; includeOutputs?: boolean; maxOutputCharsPerItem?: number } = {}, + ): Promise { + const result = await this.client.callTool("codex_bridge_read", { controllerId, ...options }); + return this.toStatus(result); + } + + async wait(controllerId: string, options: { timeoutMs?: number; cursor?: string } = {}): Promise { + const result = await this.client.callTool("codex_bridge_wait", { controllerId, ...options }); + return this.toStatus(result); + } + + async send(controllerId: string, prompt: string): Promise { + if (prompt.length === 0) throw new BridgeCoreError("EMPTY_PROMPT", "prompt is empty"); + if (prompt.length > 512 * 1024) throw new BridgeCoreError("PROMPT_TOO_LARGE", "prompt exceeds 512 KiB"); + const result = await this.client.callTool("codex_bridge_send", { controllerId, prompt }); + return this.toStatus(result); + } + + private toStatus(result: Record): CodexBridgeStatus { + const content = result.content as Array<{ type: string; text?: string }> | undefined; + const text = content?.find(c => c.type === "text")?.text ?? "{}"; + let parsed: Record; + try { + parsed = JSON.parse(text) as Record; + } catch { + throw new BridgeCoreError("HOST_OFFLINE", "bridge tool returned unparseable result"); + } + if (parsed.ok === false) { + throw new BridgeCoreError("HOST_OFFLINE", String(parsed.message ?? "bridge tool failure")); + } + return { + bindingId: typeof parsed.bindingId === "string" ? parsed.bindingId : "", + state: typeof parsed.state === "string" ? parsed.state : "", + codex: (parsed.codex as Record) ?? undefined, + raw: parsed, + }; + } +} diff --git a/src/chatgpt-bridge/hosts/codex/legacy-registry.ts b/src/chatgpt-bridge/hosts/codex/legacy-registry.ts new file mode 100644 index 00000000000..46b4ff37465 --- /dev/null +++ b/src/chatgpt-bridge/hosts/codex/legacy-registry.ts @@ -0,0 +1,119 @@ +import { readFileSync, statSync } from "node:fs"; +import { BridgeCoreError, CHATGPT_CONVERSATION_URL_PATTERN } from "../../contracts"; + +/** + * Read-only federation view over the legacy DevSpace bridge registry. + * + * Invariants (handoff §4.1 / P0 capability matrix §3): + * - byte-read + JSON.parse only: importing bridge-lib would run mkdir/icacls + * side effects, so that module must never be loaded here; + * - this reader never writes, never locks, and never touches *.cap files; + * - snapshots are invalidated by managementRevision + mtime so OC always sees + * the legacy registry as the single source of truth for existing bindings. + */ +export const LEGACY_REGISTRY_PATH_DEFAULT = + `${process.env.USERPROFILE ?? "~"}\\.codex\\state\\chatgpt-codex-live-bridge\\bindings.json`.replace(/~/g, process.env.USERPROFILE ?? "~"); + +export interface LegacyBindingSnapshot { + source: "legacy-codex"; + bindingId: string; + bindingEpoch: string; + revision: number; + active: boolean; + paused: boolean; + chatUrl: string | null; + chatTitle: string | null; + codexThreadId: string | null; + codexDeepLink: string | null; + /** controllers/.cap file stem; the file name is a routing key, its content is the secret. */ + controllerFileId: string | null; + capabilityExpiresAt: string | null; + updatedAt: string | null; + /** True when chatUrl is a normal-conversation URL this integration can target. */ + targetable: boolean; +} + +export interface LegacyRegistrySnapshot { + managementRevision: number; + version: number; + readAt: string; + bindings: LegacyBindingSnapshot[]; +} + +interface CacheEntry { + mtimeMs: number; + size: number; + snapshot: LegacyRegistrySnapshot; +} + +export class LegacyBindingRegistryReader { + private cache: CacheEntry | null = null; + + constructor(private readonly registryPath: string = LEGACY_REGISTRY_PATH_DEFAULT) {} + + read(): LegacyRegistrySnapshot { + let stat: { mtimeMs: number; size: number }; + try { + const s = statSync(this.registryPath); + stat = { mtimeMs: s.mtimeMs, size: s.size }; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === "ENOENT" || code === "ENOTDIR") { + this.cache = null; + return { managementRevision: -1, version: 0, readAt: new Date().toISOString(), bindings: [] }; + } + throw new BridgeCoreError("INVALID_REGISTRY", `legacy registry unreadable: ${String(error)}`); + } + if (this.cache && this.cache.mtimeMs === stat.mtimeMs && this.cache.size === stat.size) { + return this.cache.snapshot; + } + let parsed: { + version?: number; + managementRevision?: number; + bindings?: Record>; + }; + try { + parsed = JSON.parse(readFileSync(this.registryPath, "utf8")); + } catch (error) { + throw new BridgeCoreError("INVALID_REGISTRY", `legacy registry parse failed: ${String(error)}`); + } + if (parsed.version !== 2) { + throw new BridgeCoreError("PROTOCOL_UNSUPPORTED", `unsupported legacy registry version ${parsed.version}`); + } + const bindings = Object.entries(parsed.bindings ?? {}).map(([bindingId, entry]) => + projectLegacyBinding(bindingId, entry), + ); + const snapshot: LegacyRegistrySnapshot = { + managementRevision: typeof parsed.managementRevision === "number" ? parsed.managementRevision : -1, + version: 2, + readAt: new Date().toISOString(), + bindings, + }; + this.cache = { ...stat, snapshot }; + return snapshot; + } + + get(bindingId: string): LegacyBindingSnapshot | null { + return this.read().bindings.find(binding => binding.bindingId === bindingId.toLowerCase()) ?? null; + } +} + +function projectLegacyBinding(bindingId: string, entry: Record): LegacyBindingSnapshot { + const chatUrl = typeof entry.chatgptUrl === "string" ? entry.chatgptUrl : null; + return { + source: "legacy-codex", + bindingId, + bindingEpoch: typeof entry.bindingEpoch === "string" ? entry.bindingEpoch : "", + revision: typeof entry.revision === "number" ? entry.revision : -1, + active: entry.active === true, + paused: entry.paused === true, + chatUrl, + chatTitle: typeof entry.chatgptTitle === "string" ? entry.chatgptTitle : null, + codexThreadId: typeof entry.codexThreadId === "string" ? entry.codexThreadId : null, + codexDeepLink: typeof entry.codexDeepLink === "string" ? entry.codexDeepLink : null, + controllerFileId: typeof entry.controllerFileId === "string" ? entry.controllerFileId : null, + capabilityExpiresAt: typeof entry.capabilityExpiresAt === "string" ? entry.capabilityExpiresAt : null, + updatedAt: typeof entry.updatedAt === "string" ? entry.updatedAt : null, + targetable: chatUrl !== null && CHATGPT_CONVERSATION_URL_PATTERN.test(chatUrl.replace(/[#?].*$/, "")), + }; +} diff --git a/src/chatgpt-bridge/provider/adapter.ts b/src/chatgpt-bridge/provider/adapter.ts new file mode 100644 index 00000000000..2317c2999df --- /dev/null +++ b/src/chatgpt-bridge/provider/adapter.ts @@ -0,0 +1,97 @@ +import type { AdapterEvent, OcxParsedRequest, OcxProviderConfig } from "../../types"; +import type { IncomingMeta, ProviderAdapter } from "../../adapters/base"; + +/** + * chatgpt-web provider adapter: the thin OC seam for web ChatGPT models. + * + * P4 boundary (handoff §7): this adapter owns protocol translation and the + * model catalog surface. Actual turn execution is delegated to an injected + * ChatGptWebTransport. Until the browser transport is attached (P4 browser + * helper work), every turn fails with an explicit structured error — never a + * fabricated reply, and never a silent model switch. + */ +export interface ChatGptWebTurnContext { + modelId: string; + effort?: string; + promptPreview: string; +} + +export interface ChatGptWebTransport { + runTurn(context: ChatGptWebTurnContext, incoming: IncomingMeta, emit: (event: AdapterEvent) => void): Promise; +} + +export interface ChatGptWebAdapterDeps { + /** Absent until the browser helper transport is wired (explicitly degraded). */ + transport?: ChatGptWebTransport; +} + +const BROWSER_TRANSPORT_UNAVAILABLE = + "chatgpt-web model transport is not attached: the browser helper is not enabled in this OpenCodex build. " + + "Enable the chatgpt-bridge module (chatgptBridge.enabled + browser transport) to route turns through the web session."; + +export function createChatGptWebAdapter( + provider: OcxProviderConfig, + deps: ChatGptWebAdapterDeps = {}, +): ProviderAdapter { + return { + name: "chatgpt-web", + + buildRequest() { + // Required by the adapter contract; runTurn adapters never fetch here. + return { + url: provider.baseUrl || "https://chatgpt.com", + method: "POST", + headers: {}, + body: "", + }; + }, + + async *parseStream(): AsyncGenerator { + yield { + type: "error", + message: "chatgpt-web adapter uses runTurn; the fetch/parseStream path is disabled.", + }; + }, + + async runTurn( + parsed: OcxParsedRequest, + incoming: IncomingMeta, + emit: (event: AdapterEvent) => void, + ): Promise { + const transport = deps.transport; + if (!transport) { + emit({ + type: "error", + message: BROWSER_TRANSPORT_UNAVAILABLE, + status: 503, + errorType: "chatgpt_web_transport_unavailable", + code: "CHATGPT_WEB_TRANSPORT_UNAVAILABLE", + retryable: false, + }); + return; + } + try { + await transport.runTurn( + { + modelId: parsed.modelId, + effort: (parsed.options as { reasoningEffort?: string } | undefined)?.reasoningEffort, + promptPreview: JSON.stringify(parsed.context ?? {}).slice(0, 200), + }, + incoming, + emit, + ); + } catch (error) { + // Transport failures are surfaced verbatim as terminal errors; the + // caller (OC core) decides retry semantics. The adapter never retries + // a web turn on its own: the send may already have become visible. + emit({ + type: "error", + message: error instanceof Error ? error.message : String(error), + errorType: "chatgpt_web_transport_failure", + code: "CHATGPT_WEB_TRANSPORT_FAILURE", + retryable: false, + }); + } + }, + }; +} diff --git a/src/config/diagnostics.ts b/src/config/diagnostics.ts index a3893b98af9..170e439ef66 100644 --- a/src/config/diagnostics.ts +++ b/src/config/diagnostics.ts @@ -425,6 +425,26 @@ function emptyCompletionRetryError(value: unknown): string | null { return "schema_invalid: emptyCompletionRetry: must be a boolean or omitted"; } +function chatgptBridgeError(value: unknown): string | null { + const raw = rawConfigRecord(value); + if (!raw || !Object.hasOwn(raw, "chatgptBridge")) return null; + const module_ = raw.chatgptBridge; + if (module_ === undefined) return null; + if (typeof module_ !== "object" || module_ === null || Array.isArray(module_)) { + return "schema_invalid: chatgptBridge: must be an object or omitted"; + } + const record = module_ as Record; + if (Object.hasOwn(record, "enabled") && record.enabled !== undefined && typeof record.enabled !== "boolean") { + return "schema_invalid: chatgptBridge.enabled: must be a boolean or omitted"; + } + for (const key of ["statePath", "devspaceMcpUrl"] as const) { + if (Object.hasOwn(record, key) && record[key] !== undefined && typeof record[key] !== "string") { + return `schema_invalid: chatgptBridge.${key}: must be a string or omitted`; + } + } + return null; +} + function dropCodexSafetyBufferingError(value: unknown): string | null { const raw = rawConfigRecord(value); if (!raw || !Object.hasOwn(raw, "dropCodexSafetyBuffering")) return null; @@ -599,6 +619,7 @@ export function validateConfigCandidate(value: unknown): { ok: true; config: Ocx ?? codexQuotaAutoRefreshError(value) ?? codexAccountPickerEnabledError(value) ?? emptyCompletionRetryError(value) + ?? chatgptBridgeError(value) ?? dropCodexSafetyBufferingError(value) ?? oauthOpenBrowserError(value) ?? runtimeRoleError(value) diff --git a/src/config/proxy-env.ts b/src/config/proxy-env.ts index adbfe5f6176..f995514f69c 100644 --- a/src/config/proxy-env.ts +++ b/src/config/proxy-env.ts @@ -38,6 +38,7 @@ export function getDefaultConfig(): OcxConfig { return { port: 10100, emptyCompletionRetry: false, + chatgptBridge: { enabled: false }, dropCodexSafetyBuffering: false, fastRows: true, managementUsageMaxReadBytes: 64 * 1024 * 1024, diff --git a/src/config/schema/config-schema.ts b/src/config/schema/config-schema.ts index 6d1e936670c..b35101e6a5c 100644 --- a/src/config/schema/config-schema.ts +++ b/src/config/schema/config-schema.ts @@ -153,6 +153,13 @@ export const configSchema = z.object({ configRebaseProvenance: z.unknown().optional(), // A retry can be billable, so absence and malformed hand edits both stay off. emptyCompletionRetry: z.boolean().optional().catch(false), + chatgptBridge: z + .object({ + enabled: z.boolean().optional().catch(false), + statePath: z.string().min(1).optional().catch(undefined), + devspaceMcpUrl: z.string().min(1).optional().catch(undefined), + }) + .optional().catch(undefined), // Header suppression changes what Codex sees, so absence and malformed edits stay off. dropCodexSafetyBuffering: z.boolean().optional().catch(false), // A malformed hand edit must not silently stop opening the browser: fall back diff --git a/src/providers/registry/entries-core.ts b/src/providers/registry/entries-core.ts index 1c4161fe80a..55d23d44b9c 100644 --- a/src/providers/registry/entries-core.ts +++ b/src/providers/registry/entries-core.ts @@ -86,6 +86,19 @@ import { } from "./model-seeds"; export const PROVIDER_REGISTRY_CORE: readonly ProviderRegistryEntry[] = [ + { + id: "chatgpt-web", + label: "ChatGPT Web (bridge, experimental)", + adapter: "chatgpt-web", + baseUrl: "https://chatgpt.com", + authKind: "local", + featured: false, + dashboardPreset: true, + note: "Web ChatGPT models served through the chatgpt-bridge module (browser transport). Turns fail with CHATGPT_WEB_TRANSPORT_UNAVAILABLE until the module browser transport is enabled; no keys are stored and web login lives in the managed browser profile.", + models: ["chatgpt-web/luna", "chatgpt-web/instant", "chatgpt-web/medium", "chatgpt-web/high"], + liveModels: false, + defaultModel: "chatgpt-web/luna", + }, { id: "openai", label: "OpenAI (Codex login)", diff --git a/src/types/config.ts b/src/types/config.ts index 6f73b99cd42..49f27f2be49 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -437,6 +437,12 @@ export interface OcxConfig { metricsExport?: { enabled?: boolean }; /** Opt in to one identical-turn retry when a Responses completion has no text or tool call. */ emptyCompletionRetry?: boolean; + /** chatgpt-bridge module switch and state location (handoff 2026-09-11 §4). */ + chatgptBridge?: { + enabled?: boolean; + statePath?: string; + devspaceMcpUrl?: string; + }; /** Suppress allowlisted client-facing Codex transport hints; provider enforcement is unchanged. */ dropCodexSafetyBuffering?: boolean; /** diff --git a/structure/ops/docs-and-release.md b/structure/ops/docs-and-release.md index 723a5295bc7..18efcb33b3f 100644 --- a/structure/ops/docs-and-release.md +++ b/structure/ops/docs-and-release.md @@ -74,7 +74,7 @@ Manual navigation is defined in `docs-site/astro.config.mjs`. When adding a publ sidebar and either add localized copies or intentionally accept Starlight fallback behavior. Provider preset totals are recounted from the current registry when a preset lands. The -documented split is 96 total: 80 key-based, 12 OAuth, three local, and one default +documented split is 97 total: 80 key-based, 12 OAuth, four local, and one default ChatGPT-forward preset. The English provider guide, all seven translated copies, and all eight quickstarts carry the same counts. diff --git a/tests/adapters/adapter-registry-authority.test.ts b/tests/adapters/adapter-registry-authority.test.ts index b0b43c8b197..0d9df0aceed 100644 --- a/tests/adapters/adapter-registry-authority.test.ts +++ b/tests/adapters/adapter-registry-authority.test.ts @@ -22,6 +22,7 @@ const EXPECTED_ADAPTER_NAMES = { "azure-openai": "azure-openai", cursor: "cursor", devin: "devin", + "chatgpt-web": "chatgpt-web", "mimo-free": "mimo-free", qoder: "qoder", } as const; diff --git a/tests/chatgpt-bridge/chatgpt-bridge-codex-host.test.ts b/tests/chatgpt-bridge/chatgpt-bridge-codex-host.test.ts new file mode 100644 index 00000000000..7decdc7df49 --- /dev/null +++ b/tests/chatgpt-bridge/chatgpt-bridge-codex-host.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, test } from "bun:test"; +import { BridgeCoreError } from "../../src/chatgpt-bridge/contracts"; +import { CodexHostAdapter, DevSpaceMcpClient } from "../../src/chatgpt-bridge/hosts/codex/devspace-mcp-client"; + +/** Minimal fetch mock speaking the DevSpace MCP tool envelope. */ +function mockClient(handler: (tool: string, args: Record) => Record, opts: { status?: number } = {}) { + const calls: Array<{ url: string; init: RequestInit }> = []; + const fetchImpl = (async (input: string | URL | Request, init?: RequestInit) => { + const url = String(input); + const body = JSON.parse(String(init?.body ?? "{}")); + calls.push({ url, init: init ?? {} }); + if (url.endsWith("/healthz")) { + return new Response(JSON.stringify({ ok: true, name: "devspace" }), { status: 200 }); + } + const result = handler(body.params?.name ?? "", body.params?.arguments ?? {}); + const payload = { jsonrpc: "2.0", id: body.id, result }; + return new Response(JSON.stringify(payload), { status: opts.status ?? 200 }); + }) as typeof fetch; + const client = new DevSpaceMcpClient({ + baseUrl: "http://127.0.0.1:17676/mcp", + bearerToken: "test-token-not-a-real-secret", + fetchImpl, + }); + return { client, calls }; +} + +function toolResult(result: Record) { + return { isError: false, content: [{ type: "text", text: JSON.stringify(result) }] }; +} + +function toolFailure(code: string, message: string) { + return { + isError: true, + content: [{ type: "text", text: JSON.stringify({ ok: false, code, message }) }], + }; +} + +describe("devspace mcp client", () => { + test("healthz probes the loopback service", async () => { + const { client, calls } = mockClient(() => ({})); + expect(await client.healthz()).toEqual({ ok: true, name: "devspace" }); + expect(calls[0]?.url).toBe("http://127.0.0.1:17676/healthz"); + const healthHeaders = calls[0]?.init.headers as Record | undefined; + expect(healthHeaders?.authorization).toBeUndefined(); + }); + + test("known failure codes map 1:1 onto bridge error codes", async () => { + const { client } = mockClient(() => toolFailure("TARGET_ACTIVE", "codex task busy")); + const adapter = new CodexHostAdapter(client); + try { + await adapter.send("controller-1", "prompt"); + expect.unreachable(); + } catch (error) { + expect(error).toBeInstanceOf(BridgeCoreError); + expect((error as BridgeCoreError).code).toBe("TARGET_ACTIVE"); + } + }); + + test("DELIVERY_UNKNOWN propagates verbatim and is never retried by the adapter", async () => { + const { client } = mockClient(() => toolFailure("DELIVERY_UNKNOWN", "do not resend automatically")); + const adapter = new CodexHostAdapter(client); + try { + await adapter.send("controller-1", "prompt"); + expect.unreachable(); + } catch (error) { + expect((error as BridgeCoreError).code).toBe("DELIVERY_UNKNOWN"); + } + }); + + test("unknown failure codes degrade to HOST_OFFLINE, never to a delivery outcome", async () => { + const { client } = mockClient(() => toolFailure("SOMETHING_NEW", "future code")); + const adapter = new CodexHostAdapter(client); + try { + await adapter.send("controller-1", "prompt"); + expect.unreachable(); + } catch (error) { + expect((error as BridgeCoreError).code).toBe("HOST_OFFLINE"); + } + }); + + test("401/403 surface as AUTH_REQUIRED", async () => { + const fetchImpl = (async () => new Response("no", { status: 401 })) as typeof fetch; + const client = new DevSpaceMcpClient({ baseUrl: "http://127.0.0.1:17676/mcp", bearerToken: "x", fetchImpl }); + try { + await client.callTool("codex_bridge_status", { controllerId: "c1" }); + expect.unreachable(); + } catch (error) { + expect((error as BridgeCoreError).code).toBe("AUTH_REQUIRED"); + } + }); + + test("bearer token never appears in the request URL", async () => { + const seen: string[] = []; + const fetchImpl = (async (input: string | URL | Request) => { + seen.push(String(input)); + return new Response(JSON.stringify({ jsonrpc: "2.0", id: 1, result: toolResult({ ok: true, state: "READABLE" }) }), { status: 200 }); + }) as typeof fetch; + const client = new DevSpaceMcpClient({ baseUrl: "http://127.0.0.1:17676/mcp", bearerToken: "super-secret-token", fetchImpl }); + await client.callTool("codex_bridge_status", { controllerId: "c1" }); + expect(seen.every(url => !url.includes("super-secret-token"))).toBe(true); + }); + + test("codex host adapter unwraps status payloads", async () => { + const { client } = mockClient(() => + toolResult({ ok: true, bindingId: "b-1", state: "READABLE", codex: { status: "idle" } }), + ); + const adapter = new CodexHostAdapter(client); + const status = await adapter.status("controller-1"); + expect(status.bindingId).toBe("b-1"); + expect(status.state).toBe("READABLE"); + expect(status.codex?.status).toBe("idle"); + }); + + test("send enforces the 512 KiB boundary before any network call", async () => { + const { client, calls } = mockClient(() => toolResult({ ok: true, state: "DELIVERED" })); + const adapter = new CodexHostAdapter(client); + await expect(adapter.send("c1", "x".repeat(512 * 1024 + 1))).rejects.toThrow(BridgeCoreError); + expect(calls).toHaveLength(0); + }); +}); diff --git a/tests/chatgpt-bridge/chatgpt-bridge-core.test.ts b/tests/chatgpt-bridge/chatgpt-bridge-core.test.ts new file mode 100644 index 00000000000..ee4daabc9e5 --- /dev/null +++ b/tests/chatgpt-bridge/chatgpt-bridge-core.test.ts @@ -0,0 +1,326 @@ +import { describe, expect, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import { mkdtempSync, readFileSync, statSync, writeFileSync, mkdirSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + BridgeCoreError, + parseChatGptConversationUrl, +} from "../../src/chatgpt-bridge/contracts"; +import { BridgeBindingStore } from "../../src/chatgpt-bridge/core/store"; +import { LegacyBindingRegistryReader } from "../../src/chatgpt-bridge/hosts/codex/legacy-registry"; + +const HOST = { + kind: "codex", + instanceId: "codex-desktop-local", + targetId: "01987654-aaaa-7ccc-9ddd-eeeeeeeeeeee", + workspaceRef: "g:/zcode-project/codex-chatgpt-web", +} as const; + +const CHAT_URL = "https://chatgpt.com/c/12345678-90ab-4cde-8f01-234567890abc"; + +function makeStore(): BridgeBindingStore { + const db = new Database(":memory:"); + return new BridgeBindingStore(db, { reservationStaleMs: 60_000, duplicateGuardMs: 60_000 }); +} + +function createBinding(store: BridgeBindingStore, chatUrl: string = CHAT_URL) { + return store.createBinding({ + ownerRef: "owner:test", + host: { ...HOST }, + chatUrl, + operationId: crypto.randomUUID(), + }); +} + +/** bun:test toThrow does not take predicates; assert code explicitly here. */ +function expectBridgeError(fn: () => unknown, code: string) { + let caught: unknown; + try { + fn(); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(BridgeCoreError); + expect((caught as BridgeCoreError).code).toBe(code); +} + +describe("chatgpt-bridge contracts", () => { + test("parseChatGptConversationUrl accepts only normal-chat URLs", () => { + expect(parseChatGptConversationUrl(CHAT_URL).conversationId).toBe("12345678-90ab-4cde-8f01-234567890abc"); + expectBridgeError(() => parseChatGptConversationUrl("http://chatgpt.com/c/12345678-90ab-4cde-8f01-234567890abc"), "INVALID_CHATGPT_URL"); + expectBridgeError(() => parseChatGptConversationUrl("https://chatgpt.com/share/abc"), "INVALID_CHATGPT_URL"); + expectBridgeError(() => parseChatGptConversationUrl("https://chatgpt.com/gpts/mine"), "INVALID_CHATGPT_URL"); + }); +}); + +describe("chatgpt-bridge core store: management fencing", () => { + test("revision conflict is rejected with BINDING_REVISION_CONFLICT", () => { + const store = makeStore(); + const { binding } = createBinding(store); + expectBridgeError( + () => + store.manageBinding({ + bindingId: binding.bindingId, + action: "pause", + operationId: crypto.randomUUID(), + expectedRevision: binding.revision + 5, + }), + "BINDING_REVISION_CONFLICT", + ); + }); + + test("same operationId replays original receipt; different request conflicts", () => { + const store = makeStore(); + const operationId = crypto.randomUUID(); + const first = store.createBinding({ ownerRef: "owner:test", host: { ...HOST }, chatUrl: CHAT_URL, operationId }); + expect(first.receipt.outcome).toBe("applied"); + + const replay = store.createBinding({ ownerRef: "owner:test", host: { ...HOST }, chatUrl: CHAT_URL, operationId }); + expect(replay.receipt.outcome).toBe("alreadyApplied"); + expect(replay.binding.bindingId).toBe(first.binding.bindingId); + + expectBridgeError( + () => + store.createBinding({ + ownerRef: "owner:other", + host: { ...HOST }, + chatUrl: "https://chatgpt.com/c/99999999-90ab-4cde-8f01-234567890abc", + operationId, + }), + "OPERATION_ID_CONFLICT", + ); + }); + + test("pause/resume flow works, pause blocks sending with SEND_PAUSED", () => { + const store = makeStore(); + const { binding } = createBinding(store); + const paused = store.manageBinding({ + bindingId: binding.bindingId, + action: "pause", + operationId: crypto.randomUUID(), + expectedRevision: binding.revision, + }); + expect(paused.binding?.lifecycle).toBe("paused"); + expectBridgeError( + () => + store.reserveDelivery({ + bindingId: binding.bindingId, + operationId: crypto.randomUUID(), + direction: "chat-to-host", + sourceMessageId: "m1", + prompt: "hello", + }), + "SEND_PAUSED", + ); + }); + + test("revoke bumps epoch and fences in-flight settlements", () => { + const store = makeStore(); + const { binding } = createBinding(store); + const reservation = store.reserveDelivery({ + bindingId: binding.bindingId, + operationId: crypto.randomUUID(), + direction: "chat-to-host", + sourceMessageId: "m1", + prompt: "work on this", + }); + store.manageBinding({ + bindingId: binding.bindingId, + action: "revoke", + operationId: crypto.randomUUID(), + expectedRevision: binding.revision, + }); + const revoked = store.getBinding(binding.bindingId)!; + expect(revoked.lifecycle).toBe("revoked"); + expect(revoked.epoch).toBe(binding.epoch + 1); + expectBridgeError( + () => store.settleDelivery({ reservationId: reservation.reservationId, outcome: "delivered" }), + "BINDING_CHANGED", + ); + }); +}); + +describe("chatgpt-bridge core store: delivery guarantees", () => { + test("delivered prompt inside guard window is rejected as DUPLICATE_PROMPT", () => { + const store = makeStore(); + const { binding } = createBinding(store); + const reservation = store.reserveDelivery({ + bindingId: binding.bindingId, + operationId: crypto.randomUUID(), + direction: "chat-to-host", + sourceMessageId: "m1", + prompt: "run the probe", + }); + store.settleDelivery({ reservationId: reservation.reservationId, outcome: "delivered" }); + expectBridgeError( + () => + store.reserveDelivery({ + bindingId: binding.bindingId, + operationId: crypto.randomUUID(), + direction: "chat-to-host", + sourceMessageId: "m2", + prompt: "run the probe", + }), + "DUPLICATE_PROMPT", + ); + }); + + test("unknown outcome blocks re-send until manually resolved; no auto resend path exists", () => { + const store = makeStore(); + const { binding } = createBinding(store); + const reservation = store.reserveDelivery({ + bindingId: binding.bindingId, + operationId: crypto.randomUUID(), + direction: "chat-to-host", + sourceMessageId: "m1", + prompt: "maybe sent", + }); + store.settleDelivery({ + reservationId: reservation.reservationId, + outcome: "unknown", + operator: "owner:manual-check", + }); + expectBridgeError( + () => + store.reserveDelivery({ + bindingId: binding.bindingId, + operationId: crypto.randomUUID(), + direction: "chat-to-host", + sourceMessageId: "m2", + prompt: "retry after unknown", + }), + "DELIVERY_UNKNOWN", + ); + + const pending = store.pendingDelivery(binding.bindingId); + expect(pending?.state).toBe("unknown"); + }); + + test("settle not-delivered requires a definite non-delivery code", () => { + const store = makeStore(); + const { binding } = createBinding(store); + const reservation = store.reserveDelivery({ + bindingId: binding.bindingId, + operationId: crypto.randomUUID(), + direction: "chat-to-host", + sourceMessageId: "m1", + prompt: "definite check", + }); + expectBridgeError( + () => store.settleDelivery({ reservationId: reservation.reservationId, outcome: "not-delivered", failureCode: "PIPE_TIMEOUT" as never }), + "DELIVERY_UNKNOWN", + ); + const settled = store.settleDelivery({ + reservationId: reservation.reservationId, + outcome: "not-delivered", + failureCode: "EMPTY_PROMPT", + }); + expect(settled.state).toBe("not-delivered"); + }); +}); + +describe("chatgpt-bridge legacy registry federation", () => { + test("reads v2 registry, projects fields, never writes the file", () => { + const dir = mkdtempSync(join(tmpdir(), "chatgpt-bridge-legacy-")); + const path = join(dir, "bindings.json"); + writeFileSync( + path, + JSON.stringify({ + version: 2, + managementRevision: 14, + bindings: { + "aaaaaaaa-1111-2222-3333-444444444444": { + version: 1, + bindingId: "aaaaaaaa-1111-2222-3333-444444444444", + bindingEpoch: "epoch-uuid-1", + revision: 3, + active: true, + paused: false, + chatgptUrl: CHAT_URL, + chatgptTitle: "retro", + codexThreadId: "01987654-aaaa-7ccc-9ddd-eeeeeeeeeeee", + codexDeepLink: "codex://threads/01987654-aaaa-7ccc-9ddd-eeeeeeeeeeee", + capabilityExpiresAt: "2026-10-03T00:00:00.000Z", + updatedAt: "2026-09-10T00:00:00.000Z", + capabilityHash: "ab".repeat(32), + operations: [], + }, + "bbbbbbbb-1111-2222-3333-444444444444": { + version: 1, + bindingId: "bbbbbbbb-1111-2222-3333-444444444444", + bindingEpoch: "epoch-uuid-2", + revision: 1, + active: false, + paused: false, + chatgptUrl: "https://chatgpt.com/share/junk", + chatgptTitle: "share page", + codexThreadId: null, + codexDeepLink: null, + capabilityExpiresAt: null, + updatedAt: null, + }, + }, + }), + "utf8", + ); + const before = statSync(path).mtimeMs; + const bytesBefore = readFileSync(path); + + const reader = new LegacyBindingRegistryReader(path); + const snapshot = reader.read(); + expect(snapshot.version).toBe(2); + expect(snapshot.managementRevision).toBe(14); + expect(snapshot.bindings).toHaveLength(2); + const good = reader.get("AAAAAAAA-1111-2222-3333-444444444444"); + expect(good?.targetable).toBe(true); + expect(good?.bindingEpoch).toBe("epoch-uuid-1"); + const share = reader.get("bbbbbbbb-1111-2222-3333-444444444444"); + expect(share?.targetable).toBe(false); + + expect(statSync(path).mtimeMs).toBe(before); + expect(readFileSync(path).equals(bytesBefore)).toBe(true); + }); + + test("missing registry reads as empty without throwing", () => { + const dir = mkdtempSync(join(tmpdir(), "chatgpt-bridge-legacy-")); + const reader = new LegacyBindingRegistryReader(join(dir, "missing.json")); + const snapshot = reader.read(); + expect(snapshot.bindings).toHaveLength(0); + }); + + test("mkdir on the state dir is never performed by the reader", () => { + const dir = mkdtempSync(join(tmpdir(), "chatgpt-bridge-legacy-")); + const nested = join(dir, "state", "chatgpt-codex-live-bridge"); + const reader = new LegacyBindingRegistryReader(join(nested, "bindings.json")); + reader.read(); + let exists = false; + try { + statSync(nested); + exists = true; + } catch { + exists = false; + } + expect(exists).toBe(false); + }); +}); + +describe("chatgpt-bridge store: legacy-only facts are not duplicated", () => { + test("new store does not import or mutate legacy files (dual-write guard)", () => { + const dir = mkdtempSync(join(tmpdir(), "chatgpt-bridge-legacy-")); + mkdirSync(join(dir, "controllers"), { recursive: true }); + const path = join(dir, "bindings.json"); + writeFileSync( + path, + JSON.stringify({ version: 2, managementRevision: 1, bindings: {} }), + "utf8", + ); + const bytesBefore = readFileSync(path); + + const store = makeStore(); + createBinding(store, CHAT_URL); + const snapshot = new LegacyBindingRegistryReader(path).read(); + expect(snapshot.bindings).toHaveLength(0); + expect(readFileSync(path).equals(bytesBefore)).toBe(true); + }); +}); diff --git a/tests/chatgpt-bridge/chatgpt-bridge-dsh-host.test.ts b/tests/chatgpt-bridge/chatgpt-bridge-dsh-host.test.ts new file mode 100644 index 00000000000..ef20362bc2c --- /dev/null +++ b/tests/chatgpt-bridge/chatgpt-bridge-dsh-host.test.ts @@ -0,0 +1,177 @@ +import { describe, expect, test } from "bun:test"; +import { createBridgePlugin, TOKEN_HEADER, type DshPluginContext, type DshSessionEvent, type DshUserMessage } from "../../extensions/dsh-chatgpt-bridge/src/host"; + +interface Listener { + (payload: unknown): void | Promise; +} + +function makeCtx(agents: Map) { + const listeners = new Map>(); + const storage = new Map(); + const routes: Array<{ kind: string; path: string; handler: (r: unknown) => Promise }> = []; + const ctx: DshPluginContext = { + agents: { + get: (id: string) => { + const record = agents.get(id); + if (!record) return undefined; + return { + id, + followup: (message: DshUserMessage) => { + record.followups.push(message); + listeners.get("agent/inbox/inserted")?.forEach(l => l({ agent: { id, followup: () => {} }, message: { id: message.id } })); + }, + whenIdle: async () => {}, + }; + }, + isOwnedBy: () => false, + }, + on: ((event: string, listener: Listener) => { + if (!listeners.has(event)) listeners.set(event, new Set()); + listeners.get(event)!.add(listener); + }) as DshPluginContext["on"], + storage: { + get: async (key: string) => storage.get(key) as T | undefined, + set: async (key: string, value: T) => { + storage.set(key, value); + }, + }, + webServer: { + register: (route: { kind: "prefix"; path: string; handler: (r: never) => Promise }) => { + routes.push(route as { kind: string; path: string; handler: (r: unknown) => Promise }); + }, + }, + }; + const emitClaimed = (agentId: string, inboxItemId: string, turn: number) => + listeners.get("agent/inbox/claimed")?.forEach(l => + l({ agent: { id: agentId, followup: () => {} }, message: { id: inboxItemId }, turn }), + ); + const emitSessionEvent = (sessionId: string, event: DshSessionEvent) => + listeners.get("session/event")?.forEach(l => l({ header: { id: sessionId } }, event)); + return { ctx, listeners, storage, routes, emitClaimed, emitSessionEvent }; +} + +const TOKEN = "probe-token"; +const authHeaders = { [TOKEN_HEADER]: TOKEN }; + +describe("dsh chatgpt-bridge host plugin", () => { + test("registers the control route on apply", () => { + const { ctx, routes } = makeCtx(new Map()); + const plugin = createBridgePlugin(ctx, { controlToken: TOKEN }); + plugin.apply(); + expect(routes).toHaveLength(1); + expect(routes[0]!.path).toBe("/chatgpt-bridge"); + }); + + test("deliver enqueues followup and reports SUBMITTED_UNVERIFIED (enqueue ≠ completion)", async () => { + const followups: DshUserMessage[] = []; + const { ctx } = makeCtx(new Map([["session-1", { followups }]])); + const plugin = createBridgePlugin(ctx, { controlToken: TOKEN }); + const response = await plugin.handler({ + method: "POST", + path: "/chatgpt-bridge/session-1/deliver", + headers: authHeaders, + body: { message: "please inspect this", chatConversationId: "12345678-90ab-4cde-8f01-234567890abc", inboxItemId: "item-1" }, + }); + expect(response.status).toBe(202); + expect((response.body as { state: string }).state).toBe("SUBMITTED_UNVERIFIED"); + expect(followups).toHaveLength(1); + expect(followups[0]!.content).toBe("please inspect this"); + }); + + test("missing/unknown session is TARGET_NOT_FOUND, never fabricated", async () => { + const { ctx } = makeCtx(new Map()); + const plugin = createBridgePlugin(ctx, { controlToken: TOKEN }); + const response = await plugin.handler({ + method: "POST", + path: "/chatgpt-bridge/session-x/deliver", + headers: authHeaders, + body: { message: "hi" }, + }); + expect(response.status).toBe(404); + expect((response.body as { code: string }).code).toBe("TARGET_NOT_FOUND"); + }); + + test("bad or missing control token is AUTH_REQUIRED", async () => { + const { ctx } = makeCtx(new Map([["session-1", { followups: [] }]])); + const plugin = createBridgePlugin(ctx, { controlToken: TOKEN }); + const response = await plugin.handler({ + method: "POST", + path: "/chatgpt-bridge/session-1/deliver", + headers: { "x-bridge-token": "wrong" }, + body: { message: "hi" }, + }); + expect(response.status).toBe(401); + }); + + test("claimed turn + assistant message + turn end correlate into the binding state", async () => { + const { ctx, emitClaimed, emitSessionEvent } = makeCtx(new Map([["session-1", { followups: [] }]])); + const plugin = createBridgePlugin(ctx, { controlToken: TOKEN }); + await plugin.handler({ + method: "POST", + path: "/chatgpt-bridge/session-1/deliver", + headers: authHeaders, + body: { message: "go", inboxItemId: "item-9", chatConversationId: "12345678-90ab-4cde-8f01-234567890abc" }, + }); + emitClaimed("session-1", "item-9", 7); + emitSessionEvent("session-1", { type: "assistant/message", turn: 7, step: 1, message: { id: "assistant-7" } }); + emitSessionEvent("session-1", { type: "turn/end", turn: 7, reason: "end_turn" }); + + const view = await plugin.handler({ + method: "GET", + path: "/chatgpt-bridge/session-1/binding", + headers: authHeaders, + }); + expect(view.status).toBe(200); + const binding = (view.body as { binding: Record }).binding; + expect(binding.lastClaimedTurn).toBe(7); + expect(binding.lastAssistantMessageId).toBe("assistant-7"); + expect(binding.lastTurnEnded).toBe(7); + }); + + test("an in-flight turn (claimed, not ended) makes the target active", async () => { + const { ctx, emitClaimed } = makeCtx(new Map([["session-1", { followups: [] }]])); + const plugin = createBridgePlugin(ctx, { controlToken: TOKEN }); + await plugin.handler({ + method: "POST", + path: "/chatgpt-bridge/session-1/deliver", + headers: authHeaders, + body: { message: "first", inboxItemId: "item-1" }, + }); + emitClaimed("session-1", "item-1", 3); + const second = await plugin.handler({ + method: "POST", + path: "/chatgpt-bridge/session-1/deliver", + headers: authHeaders, + body: { message: "second" }, + }); + expect(second.status).toBe(409); + expect((second.body as { code: string }).code).toBe("TARGET_ACTIVE"); + }); + + test("binding to a different chat is rejected with BINDING_CHANGED", async () => { + const { ctx } = makeCtx(new Map([["session-1", { followups: [] }]])); + const plugin = createBridgePlugin(ctx, { controlToken: TOKEN }); + await plugin.handler({ + method: "POST", + path: "/chatgpt-bridge/session-1/deliver", + headers: authHeaders, + body: { message: "first", chatConversationId: "12345678-90ab-4cde-8f01-234567890abc" }, + }); + const swapped = await plugin.handler({ + method: "POST", + path: "/chatgpt-bridge/session-1/deliver", + headers: authHeaders, + body: { message: "second", chatConversationId: "99999999-90ab-4cde-8f01-234567890abc" }, + }); + expect(swapped.status).toBe(409); + expect((swapped.body as { code: string }).code).toBe("BINDING_CHANGED"); + }); + + test("subagent-owned sessions are rejected on lineage even when the parent is gone", () => { + const { ctx } = makeCtx(new Map()); + const plugin = createBridgePlugin(ctx, { controlToken: TOKEN }); + expect(plugin.isSubagentOwned({ id: "s1", origin: "subagent" })).toBe(true); + expect(plugin.isSubagentOwned({ id: "s2", parentSession: "parent-1" })).toBe(true); + expect(plugin.isSubagentOwned({ id: "s3" })).toBe(false); + }); +}); diff --git a/tests/chatgpt-bridge/chatgpt-bridge-provider-adapter.test.ts b/tests/chatgpt-bridge/chatgpt-bridge-provider-adapter.test.ts new file mode 100644 index 00000000000..1cb90bc9348 --- /dev/null +++ b/tests/chatgpt-bridge/chatgpt-bridge-provider-adapter.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, test } from "bun:test"; +import { createRegisteredAdapter, getAdapterDefinition } from "../../src/adapters/registry"; +import { createChatGptWebAdapter } from "../../src/chatgpt-bridge/provider/adapter"; +import type { AdapterEvent, OcxProviderConfig } from "../../src/types"; +import type { IncomingMeta } from "../../src/adapters/base"; + +function provider(): OcxProviderConfig { + return { adapter: "chatgpt-web", baseUrl: "https://chatgpt.com" } as OcxProviderConfig; +} + +function incoming(): IncomingMeta { + return { headers: new Headers(), translatorBudget: {} as IncomingMeta["translatorBudget"] }; +} + +describe("chatgpt-web adapter", () => { + test("registry resolves the chatgpt-web wire", () => { + expect(getAdapterDefinition("chatgpt-web")).toBeDefined(); + const adapter = createRegisteredAdapter(provider(), {} as never); + expect(adapter.name).toBe("chatgpt-web"); + }); + + test("without a transport every turn fails with CHATGPT_WEB_TRANSPORT_UNAVAILABLE and no fabricated output", async () => { + const adapter = createChatGptWebAdapter(provider()); + const events: AdapterEvent[] = []; + await adapter.runTurn!( + { modelId: "chatgpt-web/luna", context: {}, stream: true, options: {} } as never, + incoming(), + event => events.push(event), + ); + expect(events).toHaveLength(1); + const error = events[0] as Extract; + expect(error.code).toBe("CHATGPT_WEB_TRANSPORT_UNAVAILABLE"); + expect(error.retryable).toBe(false); + }); + + test("with a transport events flow through and transport failures stay terminal", async () => { + const emitted: AdapterEvent[] = [ + { type: "text_delta", text: "real model output" }, + { type: "done", endTurn: true }, + ]; + let turnContext: unknown; + const adapter = createChatGptWebAdapter(provider(), { + transport: { + runTurn: async (context, _incoming, emit) => { + turnContext = context; + for (const event of emitted) emit(event); + }, + }, + }); + const events: AdapterEvent[] = []; + await adapter.runTurn!( + { modelId: "chatgpt-web/high", context: { preview: true }, stream: true, options: {} } as never, + incoming(), + event => events.push(event), + ); + expect(events).toEqual(emitted); + expect((turnContext as { modelId: string }).modelId).toBe("chatgpt-web/high"); + + const failing = createChatGptWebAdapter(provider(), { + transport: { + runTurn: async () => { + throw new Error("DELIVERY_UNKNOWN-style transport crash"); + }, + }, + }); + const failures: AdapterEvent[] = []; + await failing.runTurn!( + { modelId: "chatgpt-web/luna", context: {}, stream: true, options: {} } as never, + incoming(), + event => failures.push(event), + ); + const error = failures[0] as Extract; + expect(error.code).toBe("CHATGPT_WEB_TRANSPORT_FAILURE"); + expect(error.retryable).toBe(false); + }); + + test("parseStream path is explicitly disabled", async () => { + const adapter = createChatGptWebAdapter(provider()); + const events: AdapterEvent[] = []; + for await (const event of adapter.parseStream!()) events.push(event); + expect(events).toHaveLength(1); + expect(events[0]!.type).toBe("error"); + }); +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 25bf5372234..c98adf8cf13 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -8,6 +8,10 @@ "server-combo-held-response.test.ts": "server", "key-attribution.test.ts": "usage", "provider-send-path-import.test.ts": "server", + "chatgpt-bridge-core.test.ts": "chatgpt-bridge", + "chatgpt-bridge-codex-host.test.ts": "chatgpt-bridge", + "chatgpt-bridge-dsh-host.test.ts": "chatgpt-bridge", + "chatgpt-bridge-provider-adapter.test.ts": "chatgpt-bridge", "socks5-fetch.test.ts": "lib", "socks5-upload-lifecycle.test.ts": "lib", "provider-egress.test.ts": "lib", diff --git a/tests/test-layout-tooling.test.ts b/tests/test-layout-tooling.test.ts index 792859a3683..5c072149c3b 100644 --- a/tests/test-layout-tooling.test.ts +++ b/tests/test-layout-tooling.test.ts @@ -312,6 +312,12 @@ describe("membership oracle", () => { // Placed under routing/ by its author (#3523, restored by #3530): it exercises the oauth // routing quorum, not the Anthropic adapter, so the anthropic- seed is wrong for it. "anthropic-quorum-cache.test.ts", + // chatgpt-bridge/ is its own domain (2026-09-11): the "chat" oauth seed would grab the + // basename, but the file exercises the chatgpt-bridge core store, not oauth. + "chatgpt-bridge-core.test.ts", + "chatgpt-bridge-codex-host.test.ts", + "chatgpt-bridge-dsh-host.test.ts", + "chatgpt-bridge-provider-adapter.test.ts", ]); const mismatches: string[] = []; let resolved = 0; From 35b5d5be2859643d0eba17ee8117d4743bae7a23 Mon Sep 17 00:00:00 2001 From: Codex Developer Date: Thu, 24 Sep 2026 00:03:14 +0800 Subject: [PATCH 3/7] feat(windows): make manual stops survive a lost proxy The recovery guardian only helps if a hand stop is distinguishable from a crash. Intent is now written by every actor that can stop the proxy, before it dispatches: the tray click handler, `ocx stop` (skipped for a guardian recovery child), and the receipt-less management API stop, which fails closed when an enabled marker cannot be persisted. The intent helper ships as an owned tray asset with rollback and uninstall coverage, and the detached recorder is finally started by the server it describes. The npm package does not ship scripts/, so installing the helper is conditional while the tray script keeps its runtime refusal for a marker with no helper beside it. --- scripts/ocx-recovery-guardian/gateway.cjs | 364 ++++++ scripts/ocx-recovery-guardian/intent.ps1 | 53 + scripts/ocx-recovery-guardian/main.cjs | 360 ++++++ scripts/ocx-recovery-guardian/policy.cjs | 232 ++++ scripts/ocx-recovery-guardian/repair.cjs | 354 ++++++ .../ocx-recovery-guardian/windows-action.ps1 | 580 +++++++++ scripts/windows-visible-log-preload.ts | 35 + scripts/windows-visible-proxy.ps1 | 1048 +++++++++++++++++ src/cli/index.ts | 13 +- src/lib/recovery-intent.ts | 103 ++ src/lib/runtime-diagnostics-child.ts | 143 +++ src/lib/runtime-diagnostics.ts | 384 ++++++ src/server/background-lifecycle.ts | 22 +- src/server/management-api.ts | 12 + src/tray/windows-tray.ps1 | 40 + src/tray/windows.ts | 32 +- .../server/server-runtime-diagnostics.test.ts | 431 +++++++ .../windows/windows-recovery-gateway.test.ts | 338 ++++++ tests/windows/windows-recovery-intent.test.ts | 202 ++++ tests/windows/windows-recovery-main.test.ts | 235 ++++ .../windows-recovery-ownership.test.ts | 213 ++++ tests/windows/windows-recovery-policy.test.ts | 163 +++ tests/windows/windows-recovery-repair.test.ts | 253 ++++ tests/windows/windows-tray.test.ts | 2 +- 24 files changed, 5604 insertions(+), 8 deletions(-) create mode 100644 scripts/ocx-recovery-guardian/gateway.cjs create mode 100644 scripts/ocx-recovery-guardian/intent.ps1 create mode 100644 scripts/ocx-recovery-guardian/main.cjs create mode 100644 scripts/ocx-recovery-guardian/policy.cjs create mode 100644 scripts/ocx-recovery-guardian/repair.cjs create mode 100644 scripts/ocx-recovery-guardian/windows-action.ps1 create mode 100644 scripts/windows-visible-log-preload.ts create mode 100644 scripts/windows-visible-proxy.ps1 create mode 100644 src/lib/recovery-intent.ts create mode 100644 src/lib/runtime-diagnostics-child.ts create mode 100644 src/lib/runtime-diagnostics.ts create mode 100644 tests/server/server-runtime-diagnostics.test.ts create mode 100644 tests/windows/windows-recovery-gateway.test.ts create mode 100644 tests/windows/windows-recovery-intent.test.ts create mode 100644 tests/windows/windows-recovery-main.test.ts create mode 100644 tests/windows/windows-recovery-ownership.test.ts create mode 100644 tests/windows/windows-recovery-policy.test.ts create mode 100644 tests/windows/windows-recovery-repair.test.ts diff --git a/scripts/ocx-recovery-guardian/gateway.cjs b/scripts/ocx-recovery-guardian/gateway.cjs new file mode 100644 index 00000000000..11662a63a73 --- /dev/null +++ b/scripts/ocx-recovery-guardian/gateway.cjs @@ -0,0 +1,364 @@ +"use strict"; + +const http = require("node:http"); +const { URL } = require("node:url"); + +const MAX_BODY_BYTES = 8 * 1024 * 1024; +const BODY_TIMEOUT_MS = 10_000; +const MAX_HEADER_TIMEOUT_MS = 120_000; +const MIN_CONCURRENT_REQUESTS = 64; +const MAX_CONCURRENT_REQUESTS = 128; +const POST_PATHS = new Set([ + "/v1/responses", + "/v1/responses/compact", + "/v1/chat/completions", + "/v1/messages", +]); +const GET_PATHS = new Set(["/v1/models", "/healthz", "/readyz"]); +const HOP_BY_HOP = new Set([ + "connection", "keep-alive", "proxy-authenticate", "proxy-authorization", + "te", "trailer", "transfer-encoding", "upgrade", "host", "content-length", +]); +const FALLBACK_STRIP = new Set([ + "authorization", "x-api-key", "openai-organization", "openai-project", + "cookie", "set-cookie", "proxy-authorization", "proxy-authenticate", +]); + +function parseOrigin(value, name) { + // Do not delegate this decision to DNS: a hosts-file override could make + // `localhost` a remote credential-bearing upstream. Both guardian targets + // have explicit ports, so only accept the canonical numeric form. + const match = typeof value === "string" && /^http:\/\/127\.0\.0\.1:([1-9]\d{0,4})\/?$/.exec(value); + let origin; + try { origin = new URL(value); } catch { throw new Error(`${name} must be an absolute URL`); } + if (!match || Number(match[1]) > 65535 || origin.protocol !== "http:" || origin.hostname !== "127.0.0.1" + || origin.username || origin.password || origin.pathname !== "/" || origin.search || origin.hash) { + throw new Error(`${name} must be a canonical numeric loopback http origin`); + } + return origin; +} + +function writeJson(response, status, body) { + if (response.writableEnded) return; + const encoded = JSON.stringify(body); + response.writeHead(status, { + "content-type": "application/json; charset=utf-8", + "content-length": Buffer.byteLength(encoded), + "cache-control": "no-store", + }); + response.end(encoded); +} + +function safeLog(log, event, detail) { + try { log(event, detail); } catch { /* diagnostics cannot affect routing */ } +} + +function filteredHeaders(headers, fallback, fallbackKey) { + const output = {}; + const connectionHeaders = new Set(String(headers.connection || "").toLowerCase() + .split(",").map(value => value.trim()).filter(Boolean)); + for (const [name, value] of Object.entries(headers)) { + const lower = name.toLowerCase(); + if (HOP_BY_HOP.has(lower) || connectionHeaders.has(lower) || lower === "cookie" || lower === "set-cookie" || lower === "origin") continue; + if (!fallback) { + output[lower] = value; + } else if (!FALLBACK_STRIP.has(lower) && (lower === "accept" || lower === "content-type" || lower === "user-agent")) { + output[lower] = value; + } + } + if (fallback) output.authorization = `Bearer ${fallbackKey}`; + return output; +} + +function responseHeaders(headers) { + const output = {}; + for (const [name, value] of Object.entries(headers)) { + const lower = name.toLowerCase(); + if ((HOP_BY_HOP.has(lower) && lower !== "content-length") || lower === "set-cookie" || lower === "cookie" + || lower === "authorization" || lower === "x-api-key" || lower === "proxy-authenticate" || lower === "location" + || /(?:token|secret|credential|account|api[-_]?key)/.test(lower)) continue; + output[lower] = value; + } + return output; +} + +function readBody(request) { + return new Promise((resolve, reject) => { + const chunks = []; + let bytes = 0; + let settled = false; + const timer = setTimeout(() => finish(Object.assign(new Error("request body timed out"), { code: "BODY_TIMEOUT" })), BODY_TIMEOUT_MS); + const finish = (error, body) => { + if (settled) return; + settled = true; + clearTimeout(timer); + request.off("data", onData); + request.off("end", onEnd); + request.off("aborted", onAborted); + request.off("error", onError); + if (error) reject(error); else resolve(body); + }; + const onData = chunk => { + bytes += chunk.length; + if (bytes > MAX_BODY_BYTES) { + request.resume(); + finish(Object.assign(new Error("request body exceeds limit"), { code: "BODY_TOO_LARGE" })); + } else chunks.push(chunk); + }; + const onEnd = () => finish(null, Buffer.concat(chunks)); + const onAborted = () => finish(Object.assign(new Error("client aborted"), { code: "CLIENT_ABORTED" })); + const onError = error => finish(error); + request.on("data", onData); + request.once("end", onEnd); + request.once("aborted", onAborted); + request.once("error", onError); + }); +} + +function hasPreviousResponseId(body) { + try { + const parsed = JSON.parse(body.toString("utf8")); + return typeof parsed.previous_response_id === "string" && parsed.previous_response_id.length > 0; + } catch { + return false; + } +} + +function exactFallbackModel(body, models) { + let parsed; + try { parsed = JSON.parse(body.toString("utf8")); } catch { return null; } + if (!parsed || typeof parsed !== "object" || typeof parsed.model !== "string") return null; + const mapped = Object.prototype.hasOwnProperty.call(models, parsed.model) ? models[parsed.model] : null; + return typeof mapped === "string" && mapped.length > 0 ? mapped : null; +} + +function rewriteFallbackModel(body, mappedModel) { + const parsed = JSON.parse(body.toString("utf8")); + parsed.model = mappedModel; + return Buffer.from(JSON.stringify(parsed)); +} + +function requestUpstream({ origin, method, path, body, headers, headerTimeoutMs, clientRequest, clientResponse, onFailure, log }) { + return new Promise(resolve => { + const upstreamHeaders = { ...headers }; + if (body) upstreamHeaders["content-length"] = String(body.length); + const upstream = http.request({ + protocol: origin.protocol, + hostname: origin.hostname, + port: origin.port, + method, + path, + headers: upstreamHeaders, + }); + let settled = false; + let failureNotified = false; + const notifyFailure = () => { + if (failureNotified) return; + failureNotified = true; + try { onFailure(); } catch { /* recovery notification cannot affect the request */ } + }; + let headerTimer = setTimeout(() => upstream.destroy(Object.assign(new Error("upstream headers timed out"), { code: "UPSTREAM_TIMEOUT" })), headerTimeoutMs); + const settle = result => { + if (settled) return; + settled = true; + clearTimeout(headerTimer); + resolve(result); + }; + upstream.once("response", upstreamResponse => { + clearTimeout(headerTimer); + if (upstreamResponse.statusCode >= 500) notifyFailure(); + clientResponse.writeHead(upstreamResponse.statusCode || 502, responseHeaders(upstreamResponse.headers)); + upstreamResponse.pipe(clientResponse); + upstreamResponse.once("end", () => settle({ ok: true })); + upstreamResponse.once("error", error => { + notifyFailure(); + safeLog(log, "upstream_stream_error", { code: error.code || "stream_error" }); + clientResponse.destroy(error); + settle({ ok: false }); + }); + upstreamResponse.once("aborted", notifyFailure); + }); + upstream.once("error", error => { + notifyFailure(); + safeLog(log, "upstream_request_error", { code: error.code || "request_error" }); + if (!clientResponse.headersSent) writeJson(clientResponse, 502, { error: { code: "upstream_unavailable" } }); + settle({ ok: false }); + }); + clientRequest.once("aborted", () => upstream.destroy()); + clientResponse.once("close", () => { + if (!clientResponse.writableEnded) upstream.destroy(); + }); + if (body && body.length) upstream.end(body); else upstream.end(); + }); +} + +async function createGateway(options) { + const { + port, + primaryOrigin: primaryOriginInput, + fallbackOrigin: fallbackOriginInput, + models = {}, + readFallbackKey, + isPrimaryReady, + isStopped, + onPrimaryFailure, + log = () => {}, + headerTimeoutMs = 90_000, + maxConcurrentRequests = MIN_CONCURRENT_REQUESTS, + } = options || {}; + if (!Number.isInteger(port) || port < 0 || port > 65535) throw new Error("port must be a TCP port"); + if (typeof readFallbackKey !== "function" || typeof isPrimaryReady !== "function" + || typeof isStopped !== "function" || typeof onPrimaryFailure !== "function") throw new Error("gateway callbacks are required"); + const primaryOrigin = parseOrigin(primaryOriginInput, "primaryOrigin"); + const fallbackOrigin = parseOrigin(fallbackOriginInput, "fallbackOrigin"); + if (primaryOrigin.origin === fallbackOrigin.origin) throw new Error("primary and fallback origins must differ"); + if (port !== 0 && (primaryOrigin.port === String(port) || fallbackOrigin.port === String(port))) { + throw new Error("gateway cannot proxy to itself"); + } + const modelMap = Object.freeze({ ...models }); + const boundedHeaderTimeoutMs = Number.isFinite(headerTimeoutMs) + ? Math.min(MAX_HEADER_TIMEOUT_MS, Math.max(1_000, Math.floor(headerTimeoutMs))) + : 90_000; + const boundedMaxConcurrentRequests = Number.isFinite(maxConcurrentRequests) + ? Math.min(MAX_CONCURRENT_REQUESTS, Math.max(MIN_CONCURRENT_REQUESTS, Math.floor(maxConcurrentRequests))) + : MIN_CONCURRENT_REQUESTS; + let activeRequests = 0; + const server = http.createServer(async (request, response) => { + if (activeRequests >= boundedMaxConcurrentRequests) { + writeJson(response, 503, { error: { code: "gateway_busy" } }); + return; + } + activeRequests += 1; + let released = false; + const release = () => { + if (released) return; + released = true; + activeRequests = Math.max(0, activeRequests - 1); + }; + response.once("finish", release); + response.once("close", release); + try { + const localPort = server.address().port; + const allowedHosts = new Set([`127.0.0.1:${localPort}`, `localhost:${localPort}`]); + if (!allowedHosts.has(String(request.headers.host || "").toLowerCase())) { + writeJson(response, 421, { error: { code: "invalid_host" } }); + return; + } + if (request.headers.origin !== undefined || request.headers.cookie !== undefined || request.method === "OPTIONS") { + writeJson(response, 403, { error: { code: "browser_requests_denied" } }); + return; + } + if (!request.url || !request.url.startsWith("/") || request.url.startsWith("//")) { + writeJson(response, 400, { error: { code: "invalid_request_target" } }); + return; + } + const requestUrl = new URL(request.url, "http://127.0.0.1"); + if (requestUrl.search || (!GET_PATHS.has(requestUrl.pathname) && !POST_PATHS.has(requestUrl.pathname))) { + writeJson(response, 404, { error: { code: "route_not_allowed" } }); + return; + } + if (requestUrl.pathname === "/healthz" && request.method === "GET") { + const stopped = Boolean(isStopped()); + const primaryReady = stopped ? false : Boolean(isPrimaryReady()); + writeJson(response, 200, { service: "ocx-recovery-gateway", primaryReady }); + return; + } + if (requestUrl.pathname === "/readyz" && request.method === "GET") { + const stopped = Boolean(isStopped()); + const primaryReady = stopped ? false : Boolean(isPrimaryReady()); + writeJson(response, !stopped && primaryReady ? 200 : 503, { service: "ocx-recovery-gateway", primaryReady }); + return; + } + if (isStopped()) { + writeJson(response, 503, { error: { code: "gateway_stopped" } }); + return; + } + if (GET_PATHS.has(requestUrl.pathname) && request.method !== "GET") { + writeJson(response, 405, { error: { code: "method_not_allowed" } }); + return; + } + if (POST_PATHS.has(requestUrl.pathname) && request.method !== "POST") { + writeJson(response, 405, { error: { code: "method_not_allowed" } }); + return; + } + const primaryReady = Boolean(isPrimaryReady()); + let body = null; + if (request.method === "POST") { + try { body = await readBody(request); } catch (error) { + writeJson(response, error.code === "BODY_TOO_LARGE" ? 413 : error.code === "CLIENT_ABORTED" ? 499 : 408, + { error: { code: error.code === "BODY_TOO_LARGE" ? "body_too_large" : "body_timeout" } }); + return; + } + } + if (primaryReady) { + await requestUpstream({ + origin: primaryOrigin, method: request.method, path: requestUrl.pathname, body, + headers: filteredHeaders(request.headers, false), headerTimeoutMs: boundedHeaderTimeoutMs, clientRequest: request, + clientResponse: response, onFailure: onPrimaryFailure, log, + }); + return; + } + if (request.method === "POST" && hasPreviousResponseId(body)) { + writeJson(response, 503, { error: { code: "fallback_requires_fresh_full_context" } }); + return; + } + let fallbackBody = body; + if (request.method === "POST") { + const mappedModel = exactFallbackModel(body, modelMap); + if (!mappedModel) { + writeJson(response, 503, { error: { code: "fallback_model_unavailable" } }); + return; + } + fallbackBody = rewriteFallbackModel(body, mappedModel); + } + let fallbackKey; + try { fallbackKey = await readFallbackKey(); } catch { + writeJson(response, 503, { error: { code: "fallback_unavailable" } }); + return; + } + if (typeof fallbackKey !== "string" || !fallbackKey) { + writeJson(response, 503, { error: { code: "fallback_unavailable" } }); + return; + } + await requestUpstream({ + origin: fallbackOrigin, method: request.method, path: requestUrl.pathname, body: fallbackBody, + headers: filteredHeaders(request.headers, true, fallbackKey), headerTimeoutMs: boundedHeaderTimeoutMs, clientRequest: request, + clientResponse: response, onFailure: () => safeLog(log, "fallback_failure", { route: requestUrl.pathname }), log, + }); + } catch { + safeLog(log, "gateway_request_error", { code: "gateway_error" }); + try { onPrimaryFailure(); } catch { /* recovery notification cannot affect the request */ } + if (!response.headersSent) writeJson(response, 502, { error: { code: "gateway_unavailable" } }); + else response.destroy(); + } finally { + if (response.writableEnded) release(); + } + }); + server.on("upgrade", (_request, socket) => { + socket.end("HTTP/1.1 426 Upgrade Required\r\nConnection: close\r\nContent-Length: 0\r\n\r\n"); + }); + server.headersTimeout = boundedHeaderTimeoutMs; + server.requestTimeout = boundedHeaderTimeoutMs; + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen({ host: "127.0.0.1", port }, () => { + server.off("error", reject); + resolve(); + }); + }); + const localPort = server.address().port; + if (primaryOrigin.port === String(localPort) || fallbackOrigin.port === String(localPort)) { + await new Promise(resolve => server.close(resolve)); + throw new Error("gateway cannot proxy to itself"); + } + return { + server, + port: localPort, + close: () => new Promise(resolve => { + server.close(() => resolve()); + server.closeAllConnections?.(); + }), + }; +} + +module.exports = { createGateway }; diff --git a/scripts/ocx-recovery-guardian/intent.ps1 b/scripts/ocx-recovery-guardian/intent.ps1 new file mode 100644 index 00000000000..0a688bdf769 --- /dev/null +++ b/scripts/ocx-recovery-guardian/intent.ps1 @@ -0,0 +1,53 @@ +param( + [Parameter(Mandatory = $true)][string]$OpenCodexHome, + [Parameter(Mandatory = $true)][ValidateSet('running', 'stopped', 'maintenance')][string]$Mode, + [long]$Until = 0 +) + +$ErrorActionPreference = 'Stop' +$maxBytes = 16KB + +function Assert-RecoveryIntentPath([string]$Path) { + $item = Get-Item -LiteralPath $Path -Force -ErrorAction Stop + if ((([int]$item.Attributes -band [int][System.IO.FileAttributes]::ReparsePoint) -ne 0) -or -not $item.PSIsContainer -and $item.Length -gt $maxBytes) { + throw 'Recovery guardian marker is malformed; manual lifecycle action was not dispatched.' + } + return $item +} + +function Test-RecoveryGuardianEnabled([string]$OcHome) { + $homeItem = Assert-RecoveryIntentPath $OcHome + if (-not $homeItem.PSIsContainer) { throw 'Recovery guardian marker is malformed; manual lifecycle action was not dispatched.' } + $markerPath = Join-Path $OcHome 'recovery-guardian.json' + if (-not (Test-Path -LiteralPath $markerPath -PathType Leaf)) { return $false } + $markerItem = Assert-RecoveryIntentPath $markerPath + try { $marker = [System.IO.File]::ReadAllText($markerItem.FullName, [System.Text.Encoding]::UTF8) | ConvertFrom-Json -ErrorAction Stop } catch { throw 'Recovery guardian marker is malformed; manual lifecycle action was not dispatched.' } + if ($marker.version -eq 1 -and $marker.enabled -is [bool] -and -not $marker.enabled) { return $false } + if ($marker.version -ne 1 -or -not ($marker.enabled -is [bool]) -or -not $marker.enabled) { throw 'Recovery guardian marker is malformed; manual lifecycle action was not dispatched.' } + return $true +} + +if (Test-RecoveryGuardianEnabled $OpenCodexHome) { + if ($Mode -ne 'maintenance' -and $Until -ne 0) { throw 'Recovery intent maintenance deadline is invalid.' } + if ($Mode -eq 'maintenance' -and $Until -le 0) { throw 'Recovery intent maintenance deadline is invalid.' } + + $intentPath = Join-Path $OpenCodexHome 'recovery-intent.json' + if (Test-Path -LiteralPath $intentPath) { $null = Assert-RecoveryIntentPath $intentPath } + $intent = [ordered]@{ version = 1; mode = $Mode; at = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() } + if ($Mode -eq 'maintenance') { $intent.until = $Until } + $tmpPath = Join-Path $OpenCodexHome ('.recovery-intent.' + $PID + '.' + [Guid]::NewGuid().ToString('N') + '.tmp') + try { + [System.IO.File]::WriteAllText($tmpPath, ($intent | ConvertTo-Json -Compress), (New-Object System.Text.UTF8Encoding($false))) + if (Test-Path -LiteralPath $intentPath) { + # PowerShell converts a literal $null here to an empty String for the + # overloaded .NET call, which Windows rejects as an invalid backup path. + # NullString preserves the native optional-backup sentinel and keeps the + # replacement atomic when a prior intent already exists. + [System.IO.File]::Replace($tmpPath, $intentPath, [System.Management.Automation.Language.NullString]::Value, $true) + } else { + [System.IO.File]::Move($tmpPath, $intentPath) + } + } finally { + if (Test-Path -LiteralPath $tmpPath) { Remove-Item -LiteralPath $tmpPath -Force -ErrorAction SilentlyContinue } + } +} diff --git a/scripts/ocx-recovery-guardian/main.cjs b/scripts/ocx-recovery-guardian/main.cjs new file mode 100644 index 00000000000..b794855a29f --- /dev/null +++ b/scripts/ocx-recovery-guardian/main.cjs @@ -0,0 +1,360 @@ +'use strict'; + +// Desktop companion, deliberately hosted by Node rather than the Bun process it watches. +// No request payload, upstream credential, or arbitrary command is persisted here. +const fs = require('node:fs/promises'); +const path = require('node:path'); +const { spawn } = require('node:child_process'); +const { randomUUID } = require('node:crypto'); +const { createGateway } = require('./gateway.cjs'); +const { RecoveryPolicy } = require('./policy.cjs'); +const { runRepair } = require('./repair.cjs'); + +const pause = ms => new Promise(resolve => setTimeout(resolve, ms)); +const alive = pid => { try { process.kill(pid, 0); return true; } catch { return false; } }; +const writes = new Map(); + +async function jsonFile(file, max = 65536) { + const stat = await fs.lstat(file); + if (!stat.isFile() || stat.isSymbolicLink() || stat.size > max) throw Error('unsafe_state_file'); + return JSON.parse(await fs.readFile(file, 'utf8')); +} +async function atomicJson(file, value) { + const next = (writes.get(file) || Promise.resolve()).catch(() => {}).then(async () => { + const stat = await fs.lstat(file).catch(error => { if (error.code !== 'ENOENT') throw error; }); + if (stat && (!stat.isFile() || stat.isSymbolicLink())) throw Error('unsafe_state_file'); + const temp = `${file}.${process.pid}.${randomUUID()}.tmp`; + try { + await fs.writeFile(temp, JSON.stringify(value) + '\n', { mode: 0o600, flag: 'wx' }); + await fs.rename(temp, file); + } finally { await fs.unlink(temp).catch(error => { if (error.code !== 'ENOENT') throw error; }); } + }); + writes.set(file, next); + try { await next; } finally { if (writes.get(file) === next) writes.delete(file); } +} + +async function loadSettings(file) { + const cfg = await jsonFile(file, 16384); + if (cfg.version !== 1 || cfg.enabled !== true) throw Error('guardian_not_enabled'); + for (const key of ['projectRoot', 'openCodexHome', 'codexHome']) { + if (typeof cfg[key] !== 'string' || !path.isAbsolute(cfg[key])) throw Error('invalid_home'); + cfg[key] = path.resolve(cfg[key]); + } + if (path.resolve(cfg.projectRoot, 'scripts', 'ocx-recovery-guardian') !== __dirname) throw Error('foreign_project'); + if (path.resolve(file) !== path.join(cfg.openCodexHome, 'recovery-guardian.json')) throw Error('foreign_config'); + const home = await fs.lstat(cfg.openCodexHome); + if (!home.isDirectory() || home.isSymbolicLink()) throw Error('unsafe_home'); + for (const key of ['listenPort', 'primaryPort']) { + if (!Number.isInteger(cfg[key]) || cfg[key] < 1 || cfg[key] > 65535) throw Error('invalid_port'); + } + if (cfg.listenPort === cfg.primaryPort) throw Error('gateway_loop'); + if (cfg.fallback?.origin !== 'http://127.0.0.1:20128' || !cfg.fallback.models || Array.isArray(cfg.fallback.models)) throw Error('missing_fallback'); + if (Object.entries(cfg.fallback.models).some(([k, v]) => !k || typeof v !== 'string' || !v)) throw Error('invalid_models'); + if (!['http://127.0.0.1:11434/v1', 'http://127.0.0.1:20128/v1', 'https://api.mnnai.ru/v1'].includes(cfg.repair?.origin)) throw Error('invalid_repair'); + if (cfg.repair.fallbackOrigin !== undefined && cfg.repair.fallbackOrigin !== 'http://127.0.0.1:20128/v1') throw Error('invalid_repair_fallback'); + return cfg; +} + +// Fixed, local-only credential references. Never put a credential in process arguments. +async function readKey(ref, cfg) { + if (ref?.kind === 'ollama-local') return 'ollama-local'; + if (ref?.kind === 'oc-provider' && ref.provider === 'mnn-ai') { + const source = await jsonFile(path.join(cfg.openCodexHome, 'config.json'), 2 * 1024 * 1024); + const row = source.providers?.[ref.provider]; + if (row?.baseUrl !== cfg.repair.origin || typeof row.apiKey !== 'string' || !row.apiKey.trim() || row.apiKey.includes('${')) throw Error('repair_key_unavailable'); + return row.apiKey.trim(); + } + if (ref?.kind === 'or-protected') { + const directory = path.join(cfg.openCodexHome, 'recovery-secrets'); + const info = await fs.lstat(directory); + if (!info.isDirectory() || info.isSymbolicLink()) throw Error('unsafe_key_directory'); + const saved = await jsonFile(path.join(directory, 'or-key.json'), 16384); + if (saved.version !== 1 || saved.baseUrl !== cfg.fallback.origin || typeof saved.key !== 'string' || !saved.key) throw Error('fallback_key_unavailable'); + return saved.key; + } + throw Error('unsupported_key_reference'); +} + +async function boundedCommand(executable, args, { env, timeoutMs = 15000, maxBytes = 32768 } = {}) { + return new Promise(resolve => { + let done = false, output = '', bytes = 0; + const child = spawn(executable, args, { env: env || process.env, windowsHide: true, stdio: ['ignore', 'pipe', 'ignore'] }); + const timer = setTimeout(() => finish({ ok: false, error: 'command_timeout', pid: child.pid, uncertain: true }), timeoutMs); + function finish(value) { if (!done) { done = true; clearTimeout(timer); resolve(value); } } + child.stdout.on('data', chunk => { bytes += chunk.length; if (bytes <= maxBytes) output += chunk; }); + child.on('error', () => finish({ ok: false, error: 'command_spawn_failed' })); + child.on('close', code => { + if (bytes > maxBytes) return finish({ ok: false, error: 'command_output_limit' }); + try { finish({ ok: code === 0, value: JSON.parse(output.replace(/^\uFEFF/, '').trim()), code }); } + catch { finish({ ok: false, error: 'command_receipt_invalid', code }); } + }); + // A timed-out stop may still be acting. It is deliberately NOT killed and no + // replacement is started by the caller until its termination is established. + }); +} + +function parseIntent(value, now) { + if (value?.version !== 1 || !['running', 'stopped', 'maintenance'].includes(value.mode) + || !Number.isSafeInteger(value.at) || value.at < 0 || value.at > now + 5000) return { mode: 'stopped', at: 0, valid: false }; + if (value.mode === 'maintenance' && (!Number.isSafeInteger(value.until) || value.until <= value.at || value.until > value.at + 180000)) return { mode: 'stopped', at: 0, valid: false }; + return { mode: value.mode, at: value.at, until: value.until || 0, valid: true }; +} +function recoveryActionSucceeded(result) { return result?.ok === true && result.value?.action === 'started'; } + +async function createGuardian(configFile, dependencies = {}) { + const cfg = await loadSettings(path.resolve(configFile)); + const now = dependencies.now || Date.now; + const processAlive = dependencies.alive || alive; + const primaryOrigin = `http://127.0.0.1:${cfg.primaryPort}`; + const logDir = path.join(cfg.openCodexHome, 'logs'); + await fs.mkdir(logDir, { recursive: true }); + const logFile = path.join(logDir, 'recovery-guardian.jsonl'); + const statusFile = path.join(cfg.openCodexHome, 'recovery-status.json'); + const budgetFile = path.join(cfg.openCodexHome, 'recovery-budget.json'); + const intentFile = path.join(cfg.openCodexHome, 'recovery-intent.json'); + const policy = new RecoveryPolicy(dependencies.policyOptions); + let primaryReady = false, stopped = true, closing = false, snapshot = null; + let recovering = false, repairRunning = false, currentState = 'starting', lastIntentAt = -1; + let lastIntentSignature = '', recoveryBlocked = null, failureSince = null, lastRecovery = null, lastRepair = null; + let readyIdentity = '', readySince = null, repairController = null; + const pendingActions = new Set(); + let maintenanceUntil = 0, logQueue = Promise.resolve(), pendingLogs = 0; + const log = (event, detail = {}) => { + if (++pendingLogs > 64) { --pendingLogs; return; } + const row = { at: new Date().toISOString(), event, ...detail }; + logQueue = logQueue.then(async () => { + const stat = await fs.stat(logFile).catch(() => null); + if (stat?.size > 2 * 1024 * 1024) await fs.rename(logFile, `${logFile}.1`).catch(() => {}); + await fs.appendFile(logFile, JSON.stringify(row) + '\n', { mode: 0o600 }); + }).catch(() => { console.error('[OCX:ERROR] Recovery log write failed.'); }).finally(() => { --pendingLogs; }); + }; + const powershell = path.join(process.env.SystemRoot || 'C:\\Windows', 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe'); + const actionFile = path.join(__dirname, 'windows-action.ps1'); + const actionArgs = mode => ['-NoProfile', '-NonInteractive', '-File', actionFile, '-Mode', mode, + '-ProjectRoot', cfg.projectRoot, '-OpenCodexHome', cfg.openCodexHome, '-CodexHome', cfg.codexHome, '-Port', String(cfg.primaryPort)]; + const inspect = dependencies.inspect || (() => boundedCommand(powershell, actionArgs('Inspect'))); + const initial = await inspect(); + if (initial.ok && initial.value?.owned === true) snapshot = initial.value; + try { + const saved = await jsonFile(budgetFile); + if (!policy.importSafeState(saved, now())) throw Error('invalid_budget'); + } catch (error) { + if (error.code !== 'ENOENT') throw Error('guardian_budget_invalid'); + } + try { recoveryBlocked = await jsonFile(path.join(cfg.openCodexHome, 'recovery-blocked.json')); } + catch (error) { if (error.code !== 'ENOENT') throw Error('guardian_block_invalid'); } + // A previous companion may have exited while its external stop command was + // still running. A new observer must not replay that uncertain transaction. + if (policy.exportSafeState().recoveryStartedAt !== null && !recoveryBlocked) { + recoveryBlocked = { version: 1, at: now(), reason: 'previous_recovery_uncertain' }; + await atomicJson(path.join(cfg.openCodexHome, 'recovery-blocked.json'), recoveryBlocked); + } + const server = await createGateway({ + port: cfg.listenPort, primaryOrigin, fallbackOrigin: cfg.fallback.origin, + models: cfg.fallback.models, readFallbackKey: () => readKey(cfg.fallback.key, cfg), + isPrimaryReady: () => primaryReady, isStopped: () => stopped, + onPrimaryFailure: () => { primaryReady = false; readySince = null; readyIdentity = ''; }, log, + }); + log('guardian_started', { pid: process.pid, port: cfg.listenPort, primaryPort: cfg.primaryPort }); + + async function probe(route) { + if (dependencies.probe) return dependencies.probe(route); + try { + const response = await fetch(primaryOrigin + route, { redirect: 'error', signal: AbortSignal.timeout(1200) }); + const reader = response.body.getReader(); + const chunks = []; let size = 0; + for (;;) { + const next = await reader.read(); if (next.done) break; + size += next.value.byteLength; + if (size > 8192) { await reader.cancel(); return { ok: false }; } + chunks.push(next.value); + } + const body = JSON.parse(Buffer.concat(chunks).toString('utf8')); + return { ok: response.ok && body.service === 'opencodex' && (route !== '/readyz' || body.status === 'ready'), pid: body.pid }; + } catch { return { ok: false }; } + } + + function dispatch(work) { + const pending = work().catch(() => log('guardian_action_failed', { reason: 'local_state_failure' })); + pendingActions.add(pending); + pending.finally(() => pendingActions.delete(pending)); + } + + async function diagnose(reason) { + if (repairRunning || stopped || closing) return; + repairRunning = true; + const controller = new AbortController(); + repairController = controller; + const expectedIntentAt = lastIntentAt; + try { + const claimAt = now(); + const repairBudgetFile = path.join(cfg.openCodexHome, 'recovery-repair-budget.json'); + let budget = { attempts: [] }; + try { budget = await jsonFile(repairBudgetFile); } catch (error) { if (error.code !== 'ENOENT') throw error; } + if (!Array.isArray(budget.attempts) || budget.attempts.some(at => !Number.isSafeInteger(at))) throw Error('invalid_repair_budget'); + const attempts = budget.attempts.filter(at => at >= claimAt - 3600000); + if (attempts.length >= 2 || attempts.some(at => at >= (failureSince ?? claimAt))) return; + await atomicJson(repairBudgetFile, { version: 1, attempts: [...attempts, claimAt] }); + const incidentDir = path.join(cfg.openCodexHome, 'recovery-incidents', new Date(claimAt).toISOString().replace(/[:.]/g, '-') + '-' + process.pid); + await fs.mkdir(incidentDir, { recursive: true, mode: 0o700 }); + const incident = { reason: snapshot?.pid && !processAlive(snapshot.pid) ? 'unexpected_exit' : 'health_not_ready', + healthReady: primaryReady, pid: snapshot?.pid, attempts: policy.exportSafeState().attempts.length, + timing: { durationMs: Math.max(0, claimAt - (failureSince ?? claimAt)), observedAtMs: claimAt } }; + await atomicJson(path.join(incidentDir, 'incident.json'), incident); + if (dependencies.beforeRepairDispatch) await dependencies.beforeRepairDispatch(); + const freshIntent = parseIntent(await jsonFile(intentFile, 8192), now()); + if (closing || stopped || controller.signal.aborted || !freshIntent.valid + || freshIntent.mode !== 'running' || freshIntent.at !== expectedIntentAt) return; + log('glm_repair_started', { reason }); + const result = await (dependencies.repair || runRepair)({ + incident, signal: controller.signal, + projectRoot: cfg.projectRoot, incidentDir, endpoint: cfg.repair.origin, + model: 'glm-5.3-flash', readKey: () => readKey(cfg.repair.key, cfg), + ...(cfg.repair.fallbackOrigin ? { fallbackEndpoint: cfg.repair.fallbackOrigin, readFallbackKey: () => readKey(cfg.repair.fallbackKey, cfg) } : {}), + }); + await atomicJson(path.join(incidentDir, 'receipt.json'), result); + lastRepair = { at: now(), outcome: result.outcome, failureClass: result.failureClass || null, candidateCount: result.candidateCount || 0 }; + log('glm_repair_finished', lastRepair); + } catch { lastRepair = { at: now(), outcome: 'failed', failureClass: 'LOCAL_OR_NETWORK' }; log('glm_repair_failed', lastRepair); } + finally { repairRunning = false; if (repairController === controller) repairController = null; } + } + + async function recover() { + if (recovering || stopped || closing || recoveryBlocked || !snapshot?.owned) return; + if (policy.markRecoveryStarted(now()).reason !== 'recovery-started') return; + recovering = true; + const expected = { ...snapshot }; + log('recovery_attempt_started', { pid: expected.pid }); + try { + await atomicJson(budgetFile, policy.exportSafeState()); + const freshIntent = parseIntent(await jsonFile(intentFile, 8192), now()); + if (!freshIntent.valid || freshIntent.mode !== 'running' || freshIntent.at !== lastIntentAt) throw Error('intent_changed'); + const result = dependencies.action ? await dependencies.action(expected) : await boundedCommand(powershell, [...actionArgs('Recover'), + '-ExpectedPid', String(expected.pid || 0), '-ExpectedStart', String(expected.start || ''), + '-ExpectedLauncherPid', String(expected.launcherPid || 0), '-ExpectedLauncherStart', String(expected.launcherStart || '')], + { timeoutMs: 145000 }); + // An OS start receipt is not readiness. The normal probe must observe and + // identity-check the new process before the policy can leave recovery. + lastRecovery = { at: now(), ok: recoveryActionSucceeded(result), reason: result.value?.reason || (result.uncertain ? 'stop_result_uncertain' : 'action_failed') }; + if (!recoveryActionSucceeded(result)) { + policy.markRecoveryFinished({ ok: false }, now()); + log('recovery_attempt_failed', { reason: result.uncertain ? 'stop_result_uncertain' : 'recovery_action_failed' }); + if (result.uncertain || ['stop-uncertain', 'stop-not-confirmed'].includes(result.value?.reason)) { + recoveryBlocked = { version: 1, at: now(), intentAt: lastIntentAt, reason: 'stop_result_uncertain' }; + await atomicJson(path.join(cfg.openCodexHome, 'recovery-blocked.json'), recoveryBlocked); + } + dispatch(() => diagnose('recovery_action_failed')); + } else { + policy.markRecoveryFinished({ ok: true }, now()); + log('recovery_start_received', { confirmedReady: false }); + const updated = await inspect(); + if (updated.ok && updated.value?.owned === true) snapshot = updated.value; + } + } catch { policy.markRecoveryFinished({ ok: false }, now()); log('recovery_aborted', { reason: 'intent_or_identity_changed' }); } + finally { recovering = false; await atomicJson(budgetFile, policy.exportSafeState()).catch(() => {}); } + } + + async function observe() { + let rawIntent; + try { rawIntent = await jsonFile(intentFile, 8192); } catch { rawIntent = null; } + const intent = parseIntent(rawIntent, now()); + const signature = `${intent.mode}|${intent.at}|${intent.until}`; + if (signature !== lastIntentSignature) { + const wasStopped = stopped; + const hadIntent = lastIntentAt >= 0; + lastIntentSignature = signature; + lastIntentAt = intent.at; + primaryReady = false; + readyIdentity = ''; readySince = null; + repairController?.abort(); + stopped = intent.mode === 'stopped'; + maintenanceUntil = intent.mode === 'maintenance' ? intent.until : 0; + if (hadIntent && !stopped && (wasStopped || currentState === 'foreign')) policy.reset({ now: now() }); + log('intent_observed', { mode: intent.mode, valid: intent.valid }); + } + stopped = intent.mode === 'stopped'; + if (stopped || intent.mode === 'maintenance') repairController?.abort(); + const [health, ready] = await Promise.all([probe('/healthz'), probe('/readyz')]); + const healthPid = Number.isInteger(health.pid) && health.pid > 0 ? health.pid : null; + if (healthPid) { + const updated = await inspect(); + if (updated.ok && updated.value?.owned === true && updated.value.pid === healthPid) { + snapshot = updated.value; + if (currentState === 'foreign') policy.reset({ now: now() }); + } else snapshot = null; + } + const observedReady = !stopped && health.ok && ready.ok && healthPid === ready.pid && snapshot?.owned === true && snapshot.pid === healthPid; + if (!observedReady && !stopped && failureSince === null) failureSince = now(); + const sample = { + ready: observedReady, health: health.ok, alive: !!snapshot?.pid && processAlive(snapshot.pid), owned: snapshot?.owned === true, + manualStop: stopped, launcherAlive: snapshot?.launcherPid ? processAlive(snapshot.launcherPid) : false, + }; + let decision; + if (stopped) decision = policy.observe(sample, now()); + else if (recovering) decision = { state: 'recovering', useFallback: true, action: 'none' }; + else if (now() < maintenanceUntil) decision = { state: 'maintenance', useFallback: !observedReady, action: 'none' }; + else if (!snapshot && healthPid === null) decision = { state: 'waiting_for_owner', useFallback: true, action: 'none' }; + else decision = policy.observe(sample, now()); + // Admission is per verified process generation and durable running intent. + // Policy reset/import must never allow a fresh process to skip this window. + const identity = observedReady ? `${snapshot.pid}|${snapshot.start}|${snapshot.launcherPid}|${snapshot.launcherStart}` : ''; + if (!identity || identity !== readyIdentity) { readyIdentity = identity; readySince = identity ? now() : null; } + const stableReady = readySince !== null && now() - readySince >= policy.options.recoveryStableMs; + primaryReady = observedReady && stableReady && !decision.useFallback; + if (decision.state === 'healthy' && !primaryReady) decision = { ...decision, state: 'stabilizing', useFallback: true }; + if (primaryReady) failureSince = null; + if (decision.state !== currentState) { currentState = decision.state; log('state_changed', { state: currentState, ready: primaryReady }); } + if (decision.state === 'stopped') stopped = true; + await atomicJson(statusFile, state()); + await atomicJson(budgetFile, policy.exportSafeState()); + if (decision.action === 'recover' && !recoveryBlocked) dispatch(recover); + if (decision.action === 'diagnose') dispatch(() => diagnose(decision.reason || 'recovery_budget_exhausted')); + if (recoveryBlocked && !observedReady && !stopped && !recovering) dispatch(() => diagnose('recovery_result_uncertain')); + } + async function tick() { + try { await observe(); } + catch (error) { + primaryReady = false; readyIdentity = ''; readySince = null; + currentState = 'observation_failed'; + repairController?.abort(); + throw error; + } + } + function state() { + return { version: 1, at: now(), pid: process.pid, primaryPid: snapshot?.pid || null, state: currentState, + primaryReady, recovering, repairRunning, recoveryBlocked: Boolean(recoveryBlocked), listenPort: cfg.listenPort, + fallbackModels: Object.keys(cfg.fallback.models), fallbackConfigured: Object.keys(cfg.fallback.models).length > 0, + lastRecovery, lastRepair }; + } + const close = async () => { + closing = true; + repairController?.abort(); + await server.close(); + await Promise.allSettled([...pendingActions]); + log('guardian_stopping', { pid: process.pid }); + await logQueue; + }; + return { tick, close, state, server, drain: () => Promise.allSettled([...pendingActions]) }; +} + +async function startGuardian(configFile) { + const guardian = await createGuardian(configFile); + let closing = false; + const stop = () => { closing = true; }; + process.once('SIGINT', stop); + process.once('SIGTERM', stop); + try { + while (!closing) { + const start = Date.now(); + try { await guardian.tick(); } catch { console.error('[OCX:WARN] Guardian observation failed; inspect recovery state.'); } + await pause(Math.max(50, 2000 - (Date.now() - start))); + } + } finally { process.off('SIGINT', stop); process.off('SIGTERM', stop); await guardian.close(); } +} + +if (require.main === module) { + const args = process.argv.slice(2); + if (args.length !== 2 || args[0] !== '--config') { console.error('Usage: node main.cjs --config '); process.exitCode = 2; } + else startGuardian(args[1]).catch(() => { console.error('[OCX:ERROR] Recovery guardian failed; inspect its local state and configuration.'); process.exitCode = 1; }); +} +module.exports = { startGuardian, createGuardian, loadSettings, readKey, jsonFile, atomicJson, boundedCommand, parseIntent, recoveryActionSucceeded }; diff --git a/scripts/ocx-recovery-guardian/policy.cjs b/scripts/ocx-recovery-guardian/policy.cjs new file mode 100644 index 00000000000..ffef6bf4df1 --- /dev/null +++ b/scripts/ocx-recovery-guardian/policy.cjs @@ -0,0 +1,232 @@ +"use strict"; + +const DEFAULTS = Object.freeze({ pollMs: 2_000, failureThreshold: 3, recoverAfterMs: 20_000, startupGraceMs: 45_000, maxAttempts: 2, attemptWindowMs: 900_000, recoveryStableMs: 30_000 }); +const STATE_VERSION = 1; +const STATE_KEYS = Object.freeze(["attempts", "awaitingReady", "consecutiveFailures", "diagnoseIssued", "failureSince", "foreignLatch", "healthySince", "incidentActive", "lastFailedAttemptAt", "readySince", "recoveryDeadline", "recoveryStartedAt", "startedAt", "stoppedLatch", "version"]); + +function nonNegativeInteger(value, name) { + if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${name} must be a non-negative safe integer`); + return value; +} +function positiveInteger(value, name) { + const normalized = nonNegativeInteger(value, name); + if (normalized === 0) throw new Error(`${name} must be positive`); + return normalized; +} +function nullableInteger(value) { return value === null || (Number.isSafeInteger(value) && value >= 0); } +function decision(state, useFallback, action, reason) { return { state, useFallback, action, reason }; } + +class RecoveryPolicy { + constructor(options = {}) { + if (!options || typeof options !== "object" || Array.isArray(options)) throw new Error("options must be an object"); + const merged = { ...DEFAULTS }; + for (const [name, value] of Object.entries(options)) { + if (!Object.prototype.hasOwnProperty.call(DEFAULTS, name)) throw new Error(`unknown recovery policy option: ${name}`); + merged[name] = name === "startupGraceMs" ? nonNegativeInteger(value, name) : positiveInteger(value, name); + } + this.options = Object.freeze(merged); + this.attempts = []; + this.startedAt = null; + this.lastNow = null; + this.stoppedLatch = false; + this.foreignLatch = false; + this.incidentActive = false; + this.consecutiveFailures = 0; + this.failureSince = null; + this.healthySince = null; + this.lastFailedAttemptAt = null; + this.recoveryStartedAt = null; + this.recoveryDeadline = null; + this.awaitingReady = false; + this.readySince = null; + this.diagnoseIssued = false; + } + + observe(sample, now) { + const current = this.#now(now); + this.#pruneAttempts(current); + if (this.stoppedLatch) return decision("stopped", false, "none", "stopped-latched"); + if (this.foreignLatch) return decision("foreign", true, "none", "foreign-or-unknown-identity"); + if (!sample || typeof sample !== "object") return this.#foreign(); + if (sample.manualStop === true) return this.#stop("manual-stop"); + if (sample.owned === true && sample.launcherAlive === false) return this.#stop("launcher-stopped"); + if (sample.owned !== true || sample.identityChanged === true) return this.#foreign(); + const ready = sample.ready === true && sample.health === true && sample.alive === true; + if (this.recoveryStartedAt !== null || this.awaitingReady) return this.#observeRecovery(ready, current); + if (ready) return this.#observeReady(current); + return this.#observeFailure(sample, current); + } + + markRecoveryStarted(now) { + const current = this.#now(now); + this.#pruneAttempts(current); + if (this.stoppedLatch) return decision("stopped", false, "none", "stopped-latched"); + if (this.foreignLatch) return decision("foreign", true, "none", "foreign-or-unknown-identity"); + if (this.recoveryStartedAt !== null || this.awaitingReady) return decision("recovering", true, "none", "recovery-in-progress"); + if (this.attempts.length >= this.options.maxAttempts) return this.#exhausted(); + if (this.#isBackoff(current)) return decision("failed", true, "none", "recovery-backoff"); + this.attempts.push(current); + this.recoveryStartedAt = current; + this.recoveryDeadline = current + this.options.recoverAfterMs; + this.readySince = null; + this.awaitingReady = false; + return decision("recovering", true, "none", "recovery-started"); + } + + markRecoveryFinished(result, now) { + const current = this.#now(now); + if (this.recoveryStartedAt === null) return this.stoppedLatch ? decision("stopped", false, "none", "stopped-latched") : this.foreignLatch ? decision("foreign", true, "none", "foreign-or-unknown-identity") : decision("failed", true, "none", "recovery-not-started"); + this.recoveryStartedAt = null; + if (!result || result.ok !== true) return this.#recordFailedAttempt(current, "recovery-failed"); + this.awaitingReady = true; + this.readySince = null; + this.recoveryDeadline = current + this.options.startupGraceMs + this.options.recoveryStableMs; + return decision("recovering", true, "none", "awaiting-stable-ready"); + } + + reset({ resetBudget = false, now = this.lastNow === null ? 0 : this.lastNow } = {}) { + const current = this.#now(now); + this.#pruneAttempts(current); + this.startedAt = current; + this.stoppedLatch = false; + this.foreignLatch = false; + this.incidentActive = false; + this.consecutiveFailures = 0; + this.failureSince = null; + this.healthySince = null; + this.lastFailedAttemptAt = null; + this.recoveryStartedAt = null; + this.recoveryDeadline = null; + this.awaitingReady = false; + this.readySince = null; + if (resetBudget === true) { this.attempts = []; this.diagnoseIssued = false; } + } + + exportSafeState(now = this.lastNow === null ? Date.now() : this.lastNow) { + const current = nonNegativeInteger(now, "now"); + this.#pruneAttempts(current); + return { version: STATE_VERSION, attempts: [...this.attempts], startedAt: this.startedAt, stoppedLatch: this.stoppedLatch, foreignLatch: this.foreignLatch, incidentActive: this.incidentActive, consecutiveFailures: this.consecutiveFailures, failureSince: this.failureSince, healthySince: this.healthySince, lastFailedAttemptAt: this.lastFailedAttemptAt, recoveryStartedAt: this.recoveryStartedAt, recoveryDeadline: this.recoveryDeadline, awaitingReady: this.awaitingReady, readySince: this.readySince, diagnoseIssued: this.diagnoseIssued }; + } + + importSafeState(value, now) { + const current = this.#now(now); + if (!this.#isSafeState(value)) { + this.foreignLatch = true; + this.stoppedLatch = false; + this.incidentActive = true; + this.attempts = []; + return false; + } + this.attempts = value.attempts.filter(attempt => attempt >= current - this.options.attemptWindowMs).slice(-this.options.maxAttempts); + this.startedAt = value.startedAt; + this.stoppedLatch = value.stoppedLatch; + this.foreignLatch = value.foreignLatch; + this.incidentActive = value.incidentActive; + this.consecutiveFailures = value.consecutiveFailures; + this.failureSince = value.failureSince; + this.healthySince = value.healthySince; + this.lastFailedAttemptAt = value.lastFailedAttemptAt; + this.recoveryStartedAt = value.recoveryStartedAt; + this.recoveryDeadline = value.recoveryDeadline; + this.awaitingReady = value.awaitingReady; + this.readySince = value.readySince; + this.diagnoseIssued = value.diagnoseIssued; + return true; + } + + #observeRecovery(ready, now) { + if (this.recoveryStartedAt !== null) { + if (now >= this.recoveryDeadline) return this.#recordFailedAttempt(now, "recovery-timeout"); + return decision("recovering", true, "none", "recovery-in-progress"); + } + if (ready) { + if (this.readySince === null) this.readySince = now; + if (now - this.readySince >= this.options.recoveryStableMs) return this.#endIncident(now); + } else this.readySince = null; + // A start receipt that cannot become ready is a failed normal recovery, not + // a reason to wait for a second replacement attempt before diagnostics. + // The caller's repair budget still deduplicates the bounded GLM dispatch. + if (now >= this.recoveryDeadline) return this.#recordFailedAttempt(now, "recovery-not-stable", true); + return decision("recovering", true, "none", "awaiting-stable-ready"); + } + + #observeReady(now) { + this.consecutiveFailures = 0; + this.failureSince = null; + if (!this.incidentActive) return decision("healthy", false, "none", "ready"); + if (this.healthySince === null) this.healthySince = now; + if (now - this.healthySince >= this.options.recoveryStableMs) return this.#endIncident(now); + return decision("fallback", true, "none", "awaiting-stable-ready"); + } + + #observeFailure(sample, now) { + this.incidentActive = true; + this.healthySince = null; + this.readySince = null; + this.consecutiveFailures += 1; + if (this.failureSince === null) this.failureSince = now; + const startupGrace = now - this.startedAt < this.options.startupGraceMs; + const childDead = sample.alive === false; + if (startupGrace && !childDead) return decision("suspect", false, "none", "startup-grace"); + if (childDead) return this.#recoverDecision(now, "owned-child-dead"); + if (this.consecutiveFailures < this.options.failureThreshold) return decision("suspect", false, "none", "transient-failure"); + if (now - this.failureSince < this.options.recoverAfterMs) return decision("fallback", true, "none", "failure-threshold"); + return this.#recoverDecision(now, "failure-duration"); + } + + #recoverDecision(now, reason) { + this.#pruneAttempts(now); + if (this.attempts.length >= this.options.maxAttempts) return this.#exhausted(); + if (this.#isBackoff(now)) return decision("failed", true, "none", "recovery-backoff"); + return decision("fallback", true, "recover", reason); + } + + #recordFailedAttempt(now, reason, diagnoseImmediately = false) { + this.recoveryStartedAt = null; + this.awaitingReady = false; + this.readySince = null; + this.recoveryDeadline = null; + this.lastFailedAttemptAt = now; + this.incidentActive = true; + this.healthySince = null; + if (this.attempts.length >= this.options.maxAttempts) return this.#exhausted(); + if (diagnoseImmediately) return decision("failed", true, "diagnose", reason); + return decision("failed", true, "none", reason); + } + + #exhausted() { + if (!this.diagnoseIssued) { this.diagnoseIssued = true; return decision("failed", true, "diagnose", "attempt-budget-exhausted"); } + return decision("failed", true, "none", "attempt-budget-exhausted"); + } + + #endIncident(now) { + this.incidentActive = false; + this.consecutiveFailures = 0; + this.failureSince = null; + this.healthySince = now; + this.lastFailedAttemptAt = null; + this.recoveryStartedAt = null; + this.recoveryDeadline = null; + this.awaitingReady = false; + this.readySince = null; + this.diagnoseIssued = false; + return decision("healthy", false, "none", "stable-ready"); + } + + #stop(reason) { this.stoppedLatch = true; return decision("stopped", false, "none", reason); } + #foreign() { this.foreignLatch = true; return decision("foreign", true, "none", "foreign-or-unknown-identity"); } + #isBackoff(now) { if (this.lastFailedAttemptAt === null) return false; return now - this.lastFailedAttemptAt < (this.attempts.length <= 1 ? 5_000 : 15_000); } + #pruneAttempts(now) { const floor = now - this.options.attemptWindowMs; this.attempts = this.attempts.filter(attempt => attempt >= floor).slice(-this.options.maxAttempts); } + #now(now) { const current = nonNegativeInteger(now, "now"); this.lastNow = current; if (this.startedAt === null) this.startedAt = current; return current; } + + #isSafeState(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const keys = Object.keys(value).sort(); + if (keys.length !== STATE_KEYS.length || keys.some((key, index) => key !== STATE_KEYS[index])) return false; + if (value.version !== STATE_VERSION || !Array.isArray(value.attempts) || value.attempts.length > this.options.maxAttempts || !value.attempts.every(attempt => Number.isSafeInteger(attempt) && attempt >= 0)) return false; + if (!nullableInteger(value.startedAt) || !nullableInteger(value.failureSince) || !nullableInteger(value.healthySince) || !nullableInteger(value.lastFailedAttemptAt) || !nullableInteger(value.recoveryStartedAt) || !nullableInteger(value.recoveryDeadline) || !nullableInteger(value.readySince) || !Number.isSafeInteger(value.consecutiveFailures) || value.consecutiveFailures < 0) return false; + return typeof value.stoppedLatch === "boolean" && typeof value.foreignLatch === "boolean" && typeof value.incidentActive === "boolean" && typeof value.awaitingReady === "boolean" && typeof value.diagnoseIssued === "boolean"; + } +} + +module.exports = { RecoveryPolicy }; diff --git a/scripts/ocx-recovery-guardian/repair.cjs b/scripts/ocx-recovery-guardian/repair.cjs new file mode 100644 index 00000000000..17356699c70 --- /dev/null +++ b/scripts/ocx-recovery-guardian/repair.cjs @@ -0,0 +1,354 @@ +"use strict"; + +// This module deliberately produces review artifacts only. It never writes a +// suggested replacement back into the repository. +const { createHash } = require("node:crypto"); +const fs = require("node:fs"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); + +const MODEL = "glm-5.3-flash"; +const DEADLINE_MS = 90_000; +const MAX_RESPONSE_BYTES = 128 * 1024; +const MAX_SOURCE_BYTES = 16 * 1024; +const MAX_REPLACEMENT_BYTES = 32 * 1024; +const MAX_PATCHES = 3; +const ALLOWLIST = Object.freeze([ + "src/lib/runtime-diagnostics.ts", + "src/lib/runtime-diagnostics-child.ts", + "src/responses/state.ts", + "src/codex/user-identity.ts", + "scripts/windows-visible-proxy.ps1", + "src/tray/windows-tray.ps1", +]); +const ALLOWED_REASONS = new Set([ + "event_loop_delay", "heartbeat_missing", "health_not_ready", "unexpected_exit", "manual_review", "unknown", +]); +const SENSITIVE = /(?:authorization\s*[:=]|bearer\s+[a-z0-9._~+\-/=]{8,}|(?:api[_-]?key|access[_-]?token|secret|password|credential)\s*[:=]|-----BEGIN(?: [A-Z]+)? PRIVATE KEY-----|\.env\b|userprofile|appdata|home(?:dir)?\s*[:=])/i; +const SOURCE_SENSITIVE_LINE = /(?:authorization|bearer|(?:api[_-]?key|access[_-]?token|secret|password|credential)|\.env\b|userprofile|appdata|profile|home(?:dir)?|(?:[a-z]:\\|\/)users[\\/])/i; + +function sha256(input) { + return createHash("sha256").update(input).digest("hex"); +} + +function safeInteger(value) { + return Number.isSafeInteger(value) && value >= 0 ? value : undefined; +} + +function sanitizeIncident(incident) { + const input = incident && typeof incident === "object" ? incident : {}; + const reason = typeof input.reason === "string" && ALLOWED_REASONS.has(input.reason) ? input.reason : "unknown"; + const timing = {}; + const rawTiming = input.timing && typeof input.timing === "object" && !Array.isArray(input.timing) ? input.timing : {}; + for (const name of ["delayMs", "heartbeatGapMs", "elapsedMs", "durationMs", "observedAtMs"]) { + const value = safeInteger(rawTiming[name]); + if (value !== undefined) timing[name] = value; + } + const output = { reason, healthReady: input.healthReady === true, attempts: safeInteger(input.attempts) ?? 0, timing }; + const pid = safeInteger(input.pid); + if (pid !== undefined) output.pid = pid; + return output; +} + +function normalizeEndpoint(value) { + let parsed; + try { parsed = new URL(value); } catch { return null; } + const normalizedPath = parsed.pathname.replace(/\/+$/, ""); + if (parsed.username || parsed.password || parsed.search || parsed.hash) return null; + const loopback = parsed.protocol === "http:" && parsed.hostname === "127.0.0.1" && parsed.port === "20128"; + const ollama = parsed.protocol === "http:" && parsed.hostname === "127.0.0.1" && parsed.port === "11434"; + const mnn = parsed.protocol === "https:" && parsed.hostname === "api.mnnai.ru" && (parsed.port === "" || parsed.port === "443"); + if ((loopback || ollama || mnn) && normalizedPath === "/v1") return `${parsed.protocol}//${parsed.host}/v1`; + return null; +} + +function wireModelFor(endpoint) { + if (endpoint === "http://127.0.0.1:11434/v1") return "glm-5.3-flash:cloud"; + if (endpoint === "http://127.0.0.1:20128/v1") return "ollama-local/glm-5.3-flash:cloud"; + return MODEL; +} + +function isWithin(base, target) { + const relative = path.relative(base, target); + return relative !== "" && !relative.startsWith(`..${path.sep}`) && relative !== ".." && !path.isAbsolute(relative); +} + +function assertNoSymlinks(base, target) { + if (!isWithin(base, target)) throw new Error("candidate escapes incident directory"); + let current = base; + const pieces = path.relative(base, target).split(path.sep); + for (const piece of pieces) { + current = path.join(current, piece); + if (!fs.existsSync(current)) continue; + if (fs.lstatSync(current).isSymbolicLink()) throw new Error("candidate path contains symlink"); + } +} + +function assertNoSymlinksAlong(target) { + const absolute = path.resolve(target); + const parsed = path.parse(absolute); + let current = parsed.root; + for (const piece of absolute.slice(parsed.root.length).split(path.sep).filter(Boolean)) { + current = path.join(current, piece); + if (!fs.existsSync(current)) continue; + if (fs.lstatSync(current).isSymbolicLink()) throw new Error("path contains symlink"); + } +} + +function redactSource(text) { + const lines = text.split(/(?<=\n)/u); + return lines.map(line => SOURCE_SENSITIVE_LINE.test(line) ? "[REDACTED_SENSITIVE_SOURCE_LINE]\n" : line).join(""); +} + +function truncateUtf8(text, limit) { + if (Buffer.byteLength(text) <= limit) return text; + let truncated = Buffer.from(text, "utf8").subarray(0, limit).toString("utf8"); + while (Buffer.byteLength(truncated) > limit) truncated = truncated.slice(0, -1); + return truncated; +} + +function sourceSnapshot(projectRoot) { + const root = fs.realpathSync(projectRoot); + const entries = []; + const perSourceBytes = Math.floor(MAX_SOURCE_BYTES / ALLOWLIST.length); + for (const relativePath of ALLOWLIST) { + const absolute = path.resolve(root, relativePath); + if (!isWithin(root, absolute) || !fs.existsSync(absolute) || fs.lstatSync(absolute).isSymbolicLink()) continue; + const original = fs.readFileSync(absolute, "utf8"); + const sanitized = redactSource(original); + // Every supplied file gets context; no large first file can consume the + // complete disclosure budget. + const snippet = truncateUtf8(sanitized, perSourceBytes); + entries.push({ relativePath, absolute, original, hash: sha256(original), snippet }); + } + return { root, entries }; +} + +function promptFor(incident, sources) { + return [ + "You are producing a review-only candidate for a local recovery incident.", + "Return one JSON object only (no markdown/fences): {\"diagnosis\":string,\"patches\":[{\"path\":string,\"find\":string,\"replace\":string}] }.", + "If bounded evidence does not justify a patch, return {\"diagnosis\":\"brief bounded explanation\",\"patches\":[]} exactly in that shape.", + "Do not emit markdown, tools, shell commands, credentials, paths outside the supplied repository-relative allowlist, or an executable instruction.", + "Use only exact model glm-5.3-flash. Patches are candidates only and will not be applied automatically.", + `Incident: ${JSON.stringify(incident)}`, + "Approved source snippets (some sensitive lines were redacted):", + JSON.stringify(sources.map(entry => ({ path: entry.relativePath, content: entry.snippet }))), + ].join("\n"); +} + +async function readBoundedBody(response, signal) { + if (!response || !response.body) throw new Error("missing response body"); + const reader = response.body.getReader(); + const chunks = []; + let bytes = 0; + const abort = () => { void reader.cancel(); }; + if (signal) signal.addEventListener("abort", abort, { once: true }); + try { + while (true) { + if (signal && signal.aborted) throw Object.assign(new Error("response cancelled"), { name: "AbortError" }); + const next = await reader.read(); + if (next.done) break; + bytes += next.value.byteLength; + if (bytes > MAX_RESPONSE_BYTES) { + try { await reader.cancel(); } catch { /* bounded cancellation only */ } + const error = new Error("response too large"); + error.code = "RESPONSE_TOO_LARGE"; + throw error; + } + chunks.push(next.value); + } + } finally { + if (signal) signal.removeEventListener("abort", abort); + } + return Buffer.concat(chunks).toString("utf8"); +} + +function responseContent(text) { + let outer; + try { outer = JSON.parse(text); } catch { throw Object.assign(new Error("response JSON invalid"), { code: "RESPONSE_JSON" }); } + const content = outer && outer.choices && outer.choices[0] && outer.choices[0].message && outer.choices[0].message.content; + if (typeof content !== "string" || Buffer.byteLength(content) > MAX_RESPONSE_BYTES) { + throw Object.assign(new Error("model content invalid"), { code: "MODEL_OUTPUT_INVALID" }); + } + let candidate; + try { candidate = JSON.parse(content); } catch { throw Object.assign(new Error("model JSON invalid"), { code: "MODEL_OUTPUT_INVALID" }); } + return candidate; +} + +function uniqueIndex(text, needle) { + if (!needle) return -1; + const first = text.indexOf(needle); + return first >= 0 && text.indexOf(needle, first + needle.length) < 0 ? first : -1; +} + +function validateModelOutput(output, snapshots) { + if (!output || typeof output !== "object" || Array.isArray(output) || typeof output.diagnosis !== "string" + || !Array.isArray(output.patches) || output.patches.length > MAX_PATCHES || SENSITIVE.test(output.diagnosis) + || Object.keys(output).length !== 2 || !Object.prototype.hasOwnProperty.call(output, "diagnosis") || !Object.prototype.hasOwnProperty.call(output, "patches")) { + throw Object.assign(new Error("model output shape invalid"), { code: "MODEL_OUTPUT_INVALID" }); + } + const sourceByPath = new Map(snapshots.map(item => [item.relativePath, item])); + const seen = new Set(); + const patches = []; + for (const patch of output.patches) { + if (!patch || typeof patch !== "object" || typeof patch.path !== "string" || typeof patch.find !== "string" || typeof patch.replace !== "string" + || !sourceByPath.has(patch.path) || seen.has(patch.path) || !patch.find || Buffer.byteLength(patch.replace) > MAX_REPLACEMENT_BYTES + || SENSITIVE.test(patch.find) || SENSITIVE.test(patch.replace) || Object.keys(patch).length !== 3 + || !Object.prototype.hasOwnProperty.call(patch, "path") || !Object.prototype.hasOwnProperty.call(patch, "find") || !Object.prototype.hasOwnProperty.call(patch, "replace")) { + throw Object.assign(new Error("model patch invalid"), { code: "MODEL_OUTPUT_INVALID" }); + } + const snapshot = sourceByPath.get(patch.path); + const index = uniqueIndex(snapshot.original, patch.find); + if (index < 0) throw Object.assign(new Error("model find not unique"), { code: "MODEL_OUTPUT_INVALID" }); + seen.add(patch.path); + patches.push({ ...patch, snapshot, index, candidate: snapshot.original.slice(0, index) + patch.replace + snapshot.original.slice(index + patch.find.length) }); + } + return patches; +} + +function bundledBunPath(projectRoot) { + const executable = process.platform === "win32" ? "bun.exe" : "bun"; + const candidate = path.resolve(projectRoot, "node_modules", "bun", "bin", executable); + if (!isWithin(projectRoot, candidate) || !fs.existsSync(candidate)) return null; + try { + assertNoSymlinksAlong(candidate); + return fs.lstatSync(candidate).isFile() ? candidate : null; + } catch { return null; } +} + +function verifySyntax(candidatePath, relativePath, projectRoot) { + if (relativePath.endsWith(".ts")) { + const bunPath = bundledBunPath(projectRoot); + if (!bunPath) return { patchStatus: "needs_review", verifier: "bundled_bun_unavailable" }; + const program = "const fs=require('node:fs'); const input=fs.readFileSync(process.argv[1],'utf8'); new Bun.Transpiler({loader:'ts'}).transformSync(input);"; + const result = spawnSync(bunPath, ["-e", program, candidatePath], { stdio: "ignore", timeout: 10_000, windowsHide: true }); + return result.error ? { patchStatus: "needs_review", verifier: "bundled_bun_unavailable" } + : result.status === 0 ? { patchStatus: "syntax_ok", verifier: "bun_transpiler" } + : { patchStatus: "syntax_invalid", verifier: "bun_transpiler" }; + } + if (relativePath.endsWith(".ps1")) { + const program = "$t=$null;$e=$null;[System.Management.Automation.Language.Parser]::ParseFile($args[0],[ref]$t,[ref]$e)|Out-Null;if($e.Count){exit 1};exit 0"; + const result = spawnSync("powershell.exe", ["-NoProfile", "-NonInteractive", "-Command", program, candidatePath], { stdio: "ignore", timeout: 10_000, windowsHide: true }); + return result.error ? { patchStatus: "needs_review", verifier: "powershell_parser_unavailable" } + : result.status === 0 ? { patchStatus: "syntax_ok", verifier: "powershell_parser" } + : { patchStatus: "syntax_invalid", verifier: "powershell_parser" }; + } + return { patchStatus: "needs_review", verifier: "unsupported_extension" }; +} + +function writeCandidates(incidentDir, projectRoot, patches) { + const incidentRoot = path.resolve(incidentDir); + assertNoSymlinksAlong(incidentRoot); + if (!fs.statSync(incidentRoot).isDirectory()) throw Object.assign(new Error("incident directory invalid"), { code: "CANDIDATE_WRITE" }); + const candidateRoot = path.join(incidentRoot, "candidate"); + fs.mkdirSync(candidateRoot, { recursive: true, mode: 0o700 }); + assertNoSymlinks(incidentRoot, candidateRoot); + const metadata = []; + for (const patch of patches) { + if (sha256(fs.readFileSync(patch.snapshot.absolute, "utf8")) !== patch.snapshot.hash) { + throw Object.assign(new Error("snapshot changed"), { code: "SNAPSHOT_CHANGED" }); + } + const outputPath = path.resolve(candidateRoot, patch.path); + if (!isWithin(candidateRoot, outputPath)) throw Object.assign(new Error("candidate escapes root"), { code: "CANDIDATE_WRITE" }); + fs.mkdirSync(path.dirname(outputPath), { recursive: true, mode: 0o700 }); + assertNoSymlinks(incidentRoot, outputPath); + try { fs.writeFileSync(outputPath, patch.candidate, { encoding: "utf8", mode: 0o600, flag: "wx" }); } + catch { throw Object.assign(new Error("candidate write rejected"), { code: "CANDIDATE_WRITE" }); } + const verification = verifySyntax(outputPath, patch.path, projectRoot); + metadata.push({ path: patch.path, snapshotSha256: patch.snapshot.hash, candidateSha256: sha256(patch.candidate), patchStatus: verification.patchStatus, verifier: verification.verifier }); + } + const metadataPath = path.join(candidateRoot, "metadata.json"); + assertNoSymlinks(incidentRoot, metadataPath); + try { fs.writeFileSync(metadataPath, JSON.stringify({ patches: metadata }), { encoding: "utf8", mode: 0o600, flag: "wx" }); } + catch { throw Object.assign(new Error("metadata write rejected"), { code: "CANDIDATE_WRITE" }); } + return metadata; +} + +function receipt(incident, fields) { + return { version: 1, incidentSha256: sha256(JSON.stringify(incident)), model: MODEL, requestCount: fields.requestCount ?? 0, ...fields }; +} + +async function runRepair({ incident, projectRoot, incidentDir, endpoint, fallbackEndpoint, model = MODEL, readKey, readFallbackKey, fetchFn = fetch, signal } = {}) { + const sanitized = sanitizeIncident(incident); + if (model !== MODEL || typeof readKey !== "function" || typeof fetchFn !== "function" || typeof projectRoot !== "string" || typeof incidentDir !== "string") { + return receipt(sanitized, { outcome: "failed", failureClass: "INPUT_INVALID" }); + } + if (signal !== undefined && (!signal || typeof signal.addEventListener !== "function" || typeof signal.removeEventListener !== "function")) { + return receipt(sanitized, { outcome: "failed", failureClass: "INPUT_INVALID" }); + } + if (signal && signal.aborted) return receipt(sanitized, { outcome: "cancelled", failureClass: "CANCELLED" }); + const origin = normalizeEndpoint(endpoint); + if (!origin || !fs.existsSync(projectRoot) || !fs.existsSync(incidentDir)) return receipt(sanitized, { outcome: "failed", failureClass: "INPUT_INVALID" }); + try { + assertNoSymlinksAlong(path.resolve(incidentDir)); + if (!fs.statSync(path.resolve(incidentDir)).isDirectory()) throw new Error("incident directory invalid"); + } catch { return receipt(sanitized, { outcome: "failed", failureClass: "SAFETY_REJECTED" }); } + const fallbackOrigin = fallbackEndpoint === undefined ? null : normalizeEndpoint(fallbackEndpoint); + if (fallbackEndpoint !== undefined && (!fallbackOrigin || fallbackOrigin === origin || typeof readFallbackKey !== "function")) { + return receipt(sanitized, { outcome: "failed", failureClass: "INPUT_INVALID" }); + } + let snapshots; + try { snapshots = sourceSnapshot(projectRoot); } catch { return receipt(sanitized, { outcome: "failed", failureClass: "INPUT_INVALID" }); } + if (snapshots.entries.length !== ALLOWLIST.length) return receipt(sanitized, { outcome: "failed", failureClass: "SAFETY_REJECTED" }); + const endpoints = fallbackOrigin ? [origin, fallbackOrigin] : [origin]; + let lastFailure = "NETWORK"; + let requestCount = 0; + for (let index = 0; index < endpoints.length; index += 1) { + if (signal && signal.aborted) return receipt(sanitized, { outcome: "cancelled", failureClass: "CANCELLED", requestCount }); + let key; + try { key = await (index === 0 ? readKey : readFallbackKey)(); } + catch { return receipt(sanitized, { outcome: "failed", failureClass: "AUTH_UNAVAILABLE", requestCount }); } + if (signal && signal.aborted) return receipt(sanitized, { outcome: "cancelled", failureClass: "CANCELLED", requestCount }); + if (typeof key !== "string" || !key) return receipt(sanitized, { outcome: "failed", failureClass: "AUTH_UNAVAILABLE", requestCount }); + const controller = new AbortController(); + const forwardAbort = () => controller.abort(); + if (signal) { + signal.addEventListener("abort", forwardAbort, { once: true }); + if (signal.aborted) { + forwardAbort(); + signal.removeEventListener("abort", forwardAbort); + return receipt(sanitized, { outcome: "cancelled", failureClass: "CANCELLED", requestCount }); + } + } + let timer; + const deadline = new Promise((_, reject) => { + timer = setTimeout(() => { + controller.abort(); + reject(Object.assign(new Error("repair request timed out"), { name: "AbortError" })); + }, DEADLINE_MS); + }); + try { + requestCount += 1; + const response = await Promise.race([fetchFn(`${endpoints[index]}/chat/completions`, { + method: "POST", + redirect: "error", + headers: { "content-type": "application/json", authorization: `Bearer ${key}` }, + body: JSON.stringify({ model: wireModelFor(endpoints[index]), stream: false, max_tokens: 4096, temperature: 0, response_format: { type: "json_object" }, messages: [{ role: "user", content: promptFor(sanitized, snapshots.entries) }] }), + signal: controller.signal, + }), deadline]); + if (!response || !response.ok) { + lastFailure = "HTTP_STATUS"; + if (index + 1 < endpoints.length) continue; + return receipt(sanitized, { outcome: "failed", failureClass: lastFailure, requestCount }); + } + const output = responseContent(await Promise.race([readBoundedBody(response, controller.signal), deadline])); + const patches = validateModelOutput(output, snapshots.entries); + if (patches.length === 0) return receipt(sanitized, { outcome: "no_candidate", diagnosis: output.diagnosis, candidateCount: 0, requestCount }); + const metadata = writeCandidates(incidentDir, snapshots.root, patches); + return receipt(sanitized, { outcome: "candidate_ready", diagnosis: output.diagnosis, candidateCount: metadata.length, patches: metadata, requestCount }); + } catch (error) { + if (signal && signal.aborted) return receipt(sanitized, { outcome: "cancelled", failureClass: "CANCELLED", requestCount }); + lastFailure = controller.signal.aborted || (error && error.name === "AbortError") ? "TIMEOUT" : error && error.code ? error.code : "NETWORK"; + if ((lastFailure === "NETWORK" || lastFailure === "TIMEOUT" || lastFailure === "RESPONSE_TOO_LARGE" || lastFailure === "RESPONSE_JSON") + && index + 1 < endpoints.length) continue; + return receipt(sanitized, { outcome: "failed", failureClass: lastFailure, requestCount }); + } finally { + clearTimeout(timer); + if (signal) signal.removeEventListener("abort", forwardAbort); + } + } + return receipt(sanitized, { outcome: "failed", failureClass: lastFailure, requestCount }); +} + +module.exports = { runRepair }; diff --git a/scripts/ocx-recovery-guardian/windows-action.ps1 b/scripts/ocx-recovery-guardian/windows-action.ps1 new file mode 100644 index 00000000000..1c7625d0335 --- /dev/null +++ b/scripts/ocx-recovery-guardian/windows-action.ps1 @@ -0,0 +1,580 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory)] + [ValidateSet('Inspect', 'Recover')] + [string]$Mode, + [Parameter(Mandatory)][string]$ProjectRoot, + [Parameter(Mandatory)][string]$OpenCodexHome, + [Parameter(Mandatory)][string]$CodexHome, + [Parameter(Mandatory)][ValidateRange(1, 65535)][int]$Port, + [ValidateRange(0, [int]::MaxValue)][int]$ExpectedPid = 0, + [string]$ExpectedStart = '', + [ValidateRange(0, [int]::MaxValue)][int]$ExpectedLauncherPid = 0, + [string]$ExpectedLauncherStart = '' +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$RuntimePortMaxBytes = 32768 +$IntentMaxBytes = 16384 +$StopTimeoutMs = 120000 +$OutputCounterCapBytes = 65536 + +function New-ActionResult { + param( + [string]$Action = 'inspect', + [string]$Reason = 'inspect', + [bool]$Owned = $false, + [bool]$Alive = $false, + [int]$ListenerPid = 0, + [int]$ProcessId = $ExpectedPid, + [string]$Start = '', + [int]$LauncherPid = $ExpectedLauncherPid, + [string]$LauncherStart = '', + [bool]$LauncherAlive = $false, + [string]$StopStatus = 'not-attempted' + ) + # This is intentionally a scalar-only envelope. Process command lines, + # environment values, paths, CLI output, request data, and error text are + # never returned to the guardian or an external diagnostic channel. + return [ordered]@{ + action = $Action; reason = $Reason; owned = $Owned; alive = $Alive + listenerPid = $ListenerPid; pid = $ProcessId; start = $Start + launcherPid = $LauncherPid; launcherStart = $LauncherStart + launcherAlive = $LauncherAlive; stopStatus = $StopStatus + } +} + +function Write-ActionResult { + param([System.Collections.IDictionary]$Result) + # Bypass the PowerShell success-output pipeline: this helper's contract is + # exactly one JSON record and a caller may be draining it with a bounded + # pipe while deciding whether a recovery is safe. + [Console]::Out.WriteLine(($Result | ConvertTo-Json -Compress)) +} + +function Get-FullPath { + param([Parameter(Mandatory)][string]$Path) + if ([string]::IsNullOrWhiteSpace($Path) -or -not [System.IO.Path]::IsPathRooted($Path)) { + throw 'invalid-path' + } + return [System.IO.Path]::GetFullPath($Path) +} + +function Assert-NonReparsePath { + param([Parameter(Mandatory)][string]$Path, [Parameter(Mandatory)][bool]$Leaf) + $full = Get-FullPath $Path + if ($Leaf) { + if (-not (Test-Path -LiteralPath $full -PathType Leaf)) { throw 'missing-file' } + } elseif (-not (Test-Path -LiteralPath $full -PathType Container)) { + throw 'missing-directory' + } + $root = [System.IO.Path]::GetPathRoot($full) + $relative = $full.Substring($root.Length).TrimStart([System.IO.Path]::DirectorySeparatorChar, [System.IO.Path]::AltDirectorySeparatorChar) + $current = $root + foreach ($part in $relative.Split(@([System.IO.Path]::DirectorySeparatorChar, [System.IO.Path]::AltDirectorySeparatorChar), [System.StringSplitOptions]::RemoveEmptyEntries)) { + $current = Join-Path $current $part + $entry = Get-Item -LiteralPath $current -Force -ErrorAction Stop + if (([int]$entry.Attributes -band [int][System.IO.FileAttributes]::ReparsePoint) -ne 0) { throw 'reparse-path' } + } + return $full +} + +function Initialize-CanonicalDirectoryType { + if ($null -ne ('OcxGuardianCanonicalDirectory' -as [type])) { return } + Add-Type -TypeDefinition @' +using System; +using System.ComponentModel; +using System.IO; +using System.Runtime.InteropServices; +using Microsoft.Win32.SafeHandles; +public static class OcxGuardianCanonicalDirectory { + const uint FILE_FLAG_BACKUP_SEMANTICS = 0x02000000; + const uint FILE_SHARE_READ = 1, FILE_SHARE_WRITE = 2, FILE_SHARE_DELETE = 4; + [DllImport("kernel32.dll", SetLastError=true, CharSet=CharSet.Unicode)] + static extern SafeFileHandle CreateFile(string name, uint access, uint share, IntPtr sec, uint disposition, uint flags, IntPtr template); + [DllImport("kernel32.dll", SetLastError=true, CharSet=CharSet.Unicode)] + static extern uint GetFinalPathNameByHandle(SafeFileHandle handle, System.Text.StringBuilder text, uint size, uint flags); + public static string Resolve(string path) { + using (var handle = CreateFile(path, 0, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, IntPtr.Zero, 3, FILE_FLAG_BACKUP_SEMANTICS, IntPtr.Zero)) { + if (handle.IsInvalid) throw new Win32Exception(Marshal.GetLastWin32Error()); + var text = new System.Text.StringBuilder(32768); var length = GetFinalPathNameByHandle(handle, text, (uint)text.Capacity, 0); + if (length == 0 || length >= text.Capacity) throw new Win32Exception(Marshal.GetLastWin32Error()); + var result = text.ToString(); return result.StartsWith("\\\\?\\") ? result.Substring(4) : result; + } + } +} +'@ | Out-Null +} + +function Get-StableCodexHome { + param([Parameter(Mandatory)][string]$Path) + $configured = Get-FullPath $Path + if (-not (Test-Path -LiteralPath $configured -PathType Container)) { throw 'missing-directory' } + # A normal directory needs no Win32 handle-resolution call. That keeps + # Inspect bounded even on hosts where the WMI provider is contended; only + # the explicitly supported user junction/symlink route needs canonicality. + $configuredEntry = Get-Item -LiteralPath $configured -Force -ErrorAction Stop + if ((([int]$configuredEntry.Attributes -band [int][System.IO.FileAttributes]::ReparsePoint) -eq 0) -and $configuredEntry.PSIsContainer) { + return [pscustomobject]@{ configured = $configured; canonical = $configured } + } + Initialize-CanonicalDirectoryType + $canonical = [OcxGuardianCanonicalDirectory]::Resolve($configured) + $entry = Get-Item -LiteralPath $canonical -Force -ErrorAction Stop + if (-not $entry.PSIsContainer -or (([int]$entry.Attributes -band [int][System.IO.FileAttributes]::ReparsePoint) -ne 0)) { throw 'unsafe-codex-home' } + return [pscustomobject]@{ configured = $configured; canonical = $canonical } +} + +function Test-ExactPath { + param([AllowNull()][string]$Left, [Parameter(Mandatory)][string]$Right) + if ([string]::IsNullOrWhiteSpace($Left)) { return $false } + try { + return [string]::Equals([System.IO.Path]::GetFullPath($Left), $Right, [System.StringComparison]::OrdinalIgnoreCase) + } catch { return $false } +} + +function Convert-ToTicksString { + param([AllowNull()]$CreationDate) + if ($null -eq $CreationDate) { return '' } + try { + if ($CreationDate -is [DateTime]) { return $CreationDate.ToUniversalTime().Ticks.ToString([Globalization.CultureInfo]::InvariantCulture) } + if ($CreationDate -is [DateTimeOffset]) { return $CreationDate.UtcDateTime.Ticks.ToString([Globalization.CultureInfo]::InvariantCulture) } + if ($CreationDate -isnot [string] -or [string]::IsNullOrWhiteSpace($CreationDate)) { return '' } + return [System.Management.ManagementDateTimeConverter]::ToDateTime($CreationDate).ToUniversalTime().Ticks.ToString([Globalization.CultureInfo]::InvariantCulture) + } catch { return '' } +} + +function Test-TicksString { + param([AllowNull()][string]$Value) + [long]$parsed = 0 + return [long]::TryParse($Value, [Globalization.NumberStyles]::None, [Globalization.CultureInfo]::InvariantCulture, [ref]$parsed) -and $parsed -gt 0 +} + +function Test-ExpectedIdentity { + param([AllowNull()]$Process, [int]$ProcessId, [string]$Start) + if ($null -eq $Process -or $Process.ProcessId -ne $ProcessId -or [string]::IsNullOrWhiteSpace($Start)) { return $false } + return (Convert-ToTicksString $Process.CreationDate) -ceq $Start +} + +function Get-ProcessExact { + param([int]$ProcessId) + try { return Get-CimInstance Win32_Process -Filter ("ProcessId = {0}" -f $ProcessId) -OperationTimeoutSec 3 -ErrorAction Stop } catch { return $null } +} + +function Test-ArgumentToken { + param([AllowNull()][string]$Text, [Parameter(Mandatory)][AllowEmptyString()][string]$Flag, [Parameter(Mandatory)][string]$Value) + if ([string]::IsNullOrWhiteSpace($Text)) { return $false } + # Win32_Process commonly escapes every backslash in CommandLine. Collapse + # only that representation before comparing fixed local paths; no parsed + # command line is ever emitted. + while ($Text.Contains('\\')) { $Text = $Text.Replace('\\', '\') } + $quoted = [regex]::Escape($Value) + $flag = [regex]::Escape($Flag) + # Launcher and Bun are both started with these exact, bounded argument + # pairs. The quote forms cover Windows' normal ProcessStartInfo rendering. + if ([string]::IsNullOrWhiteSpace($Flag)) { + return [regex]::IsMatch($Text, ('(?i)(?:^|\s)(?:"{0}"|''{0}''|{0})(?=\s|$)' -f $quoted)) + } + return [regex]::IsMatch($Text, ('(?i)(?:^|\s){0}\s+(?:"{1}"|''{1}''|{1})(?=\s|$)' -f $flag, $quoted)) +} + +function Test-OptionalPathArgument { + param([AllowNull()][string]$Text, [string]$Flag, [string]$Configured, [string]$Canonical) + if ([string]::IsNullOrWhiteSpace($Text) -or $Text -notmatch ("(?i)(?:^|\\s)" + [regex]::Escape($Flag) + "(?=\\s|$)")) { return $true } + return (Test-ArgumentToken -Text $Text -Flag $Flag -Value $Configured) -or (Test-ArgumentToken -Text $Text -Flag $Flag -Value $Canonical) +} + +function Test-VisibleLauncherParent { + param([AllowNull()]$Parent, [int]$ProcessId, [string]$Start, [string]$ScriptPath, [string]$Root, [string]$OpenHome, [string]$CodexConfigured, [string]$CodexCanonical) + if (-not (Test-ExpectedIdentity -Process $Parent -ProcessId $ProcessId -Start $Start)) { return $false } + $powershell = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe' + if (-not (Test-ExactPath -Left $Parent.ExecutablePath -Right $powershell)) { return $false } + if (-not (Test-ArgumentToken -Text $Parent.CommandLine -Flag '-File' -Value $ScriptPath)) { return $false } + if (-not (Test-ArgumentToken -Text $Parent.CommandLine -Flag '-ProjectRoot' -Value $Root)) { return $false } + if (-not (Test-OptionalPathArgument -Text $Parent.CommandLine -Flag '-OpenCodexHome' -Configured $OpenHome -Canonical $OpenHome)) { return $false } + return Test-OptionalPathArgument -Text $Parent.CommandLine -Flag '-CodexHome' -Configured $CodexConfigured -Canonical $CodexCanonical +} + +function Test-VisibleLauncherCandidate { + param([AllowNull()]$Parent, [string]$ScriptPath, [string]$Root, [string]$OpenHome, [string]$CodexConfigured, [string]$CodexCanonical) + if ($null -eq $Parent) { return $false } + $powershell = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe' + if (-not (Test-ExactPath -Left $Parent.ExecutablePath -Right $powershell)) { return $false } + if (-not (Test-ArgumentToken -Text $Parent.CommandLine -Flag '-File' -Value $ScriptPath)) { return $false } + if (-not (Test-ArgumentToken -Text $Parent.CommandLine -Flag '-ProjectRoot' -Value $Root)) { return $false } + if (-not (Test-OptionalPathArgument -Text $Parent.CommandLine -Flag '-OpenCodexHome' -Configured $OpenHome -Canonical $OpenHome)) { return $false } + return Test-OptionalPathArgument -Text $Parent.CommandLine -Flag '-CodexHome' -Configured $CodexConfigured -Canonical $CodexCanonical +} + +function Test-ExpectedBunChild { + param([AllowNull()]$Child, [string]$BunPath, [string]$CliPath, [int]$ListenPort) + if ($null -eq $Child -or $Child.ParentProcessId -ne $ExpectedLauncherPid -or -not (Test-ExactPath -Left $Child.ExecutablePath -Right $BunPath)) { return $false } + if (-not (Test-ProjectCliArgument -Text $Child.CommandLine -CliPath $CliPath)) { return $false } + if ($Child.CommandLine -notmatch '(?i)(?:^|\s)start(?=\s|$)') { return $false } + return Test-ArgumentToken -Text $Child.CommandLine -Flag '--port' -Value ([string]$ListenPort) +} + +function Test-InspectBunChild { + param([AllowNull()]$Child, [string]$BunPath, [string]$CliPath, [int]$ListenPort) + if ($null -eq $Child -or -not (Test-ExactPath -Left $Child.ExecutablePath -Right $BunPath)) { return $false } + if (-not (Test-ProjectCliArgument -Text $Child.CommandLine -CliPath $CliPath)) { return $false } + if ($Child.CommandLine -notmatch '(?i)(?:^|\s)start(?=\s|$)') { return $false } + return Test-ArgumentToken -Text $Child.CommandLine -Flag '--port' -Value ([string]$ListenPort) +} + +function Test-ProjectCliArgument { + param([AllowNull()][string]$Text, [string]$CliPath) + if (Test-ArgumentToken -Text $Text -Flag '' -Value $CliPath) { return $true } + # Bun can retain the visible launcher's known-repository CLI as a relative + # token. Its verified parent fixes the working directory to ProjectRoot, + # so admit only this one relative spelling, never an arbitrary script. + return [regex]::IsMatch($Text, '(?i)(?:^|\s)(?:"|''|)?(?:.*\\)?src\\cli\\index\.ts(?:"|'')?(?=\s|$)') +} + +function Get-ListenerPid { + param([int]$ListenPort) + try { + $listeners = @(Get-NetTCPConnection -LocalPort $ListenPort -State Listen -ErrorAction Stop | Select-Object -ExpandProperty OwningProcess -Unique) + if ($listeners.Count -ne 1) { return 0 } + return [int]$listeners[0] + } catch { + # Get-NetTCPConnection reports an absent port as an error on some + # Windows builds. Distinguish that ordinary absence from a failed port + # query without ever treating an unknown occupied port as closed. + try { + $present = @([System.Net.NetworkInformation.IPGlobalProperties]::GetIPGlobalProperties().GetActiveTcpListeners() | Where-Object { $_.Port -eq $ListenPort }) + if ($present.Count -eq 0) { return 0 } + } catch { } + return -1 + } +} + +function Read-BoundedJson { + param([Parameter(Mandatory)][string]$Path, [int]$MaximumBytes) + try { + $item = Get-Item -LiteralPath $Path -Force -ErrorAction Stop + if (([int]$item.Attributes -band [int][System.IO.FileAttributes]::ReparsePoint) -ne 0 -or $item.Length -gt $MaximumBytes) { return $null } + return (Get-Content -LiteralPath $Path -Raw -ErrorAction Stop | ConvertFrom-Json -ErrorAction Stop) + } catch { return $null } +} + +function Read-RuntimePortSnapshot { + param([string]$OpenCodexDirectory) + $record = Read-BoundedJson -Path (Join-Path $OpenCodexDirectory 'runtime-port.json') -MaximumBytes $RuntimePortMaxBytes + if ($null -eq $record) { return [pscustomobject]@{ port = 0; pid = 0 } } + $port = if ($record.port -is [int] -or $record.port -is [long]) { [int]$record.port } else { 0 } + $runtimePid = if ($record.pid -is [int] -or $record.pid -is [long]) { [int]$record.pid } else { 0 } + return [pscustomobject]@{ port = $port; pid = $runtimePid } +} + +function Get-OwnershipSnapshot { + param([string]$BunPath, [string]$CliPath, [string]$ScriptPath, [string]$Root, [string]$OpenCodexDirectory, [string]$CodexConfigured, [string]$CodexCanonical) + $target = Get-ProcessExact $ExpectedPid + $parent = if ($ExpectedLauncherPid -gt 0) { Get-ProcessExact $ExpectedLauncherPid } else { $null } + $listenerPid = Get-ListenerPid $Port + $targetAlive = $null -ne $target + $launcherAlive = $null -ne $parent + $targetStart = if ($targetAlive) { Convert-ToTicksString $target.CreationDate } else { '' } + $launcherStart = if ($launcherAlive) { Convert-ToTicksString $parent.CreationDate } else { '' } + $expectedChild = (Test-ExpectedIdentity -Process $target -ProcessId $ExpectedPid -Start $ExpectedStart) -and + (Test-ExpectedBunChild -Child $target -BunPath $BunPath -CliPath $CliPath -ListenPort $Port) + $expectedParent = Test-VisibleLauncherParent -Parent $parent -ProcessId $ExpectedLauncherPid -Start $ExpectedLauncherStart -ScriptPath $ScriptPath -Root $Root -OpenHome $OpenCodexDirectory -CodexConfigured $CodexConfigured -CodexCanonical $CodexCanonical + $runtime = Read-RuntimePortSnapshot -OpenCodexDirectory $OpenCodexDirectory + # A present runtime record must agree with the same exact identity. An + # absent record is diagnostic absence, not evidence for an arbitrary PID. + $runtimeAgrees = (($runtime.port -eq 0 -or $runtime.port -eq $Port) -and ($runtime.pid -eq 0 -or $runtime.pid -eq $ExpectedPid)) + $owned = $expectedChild -and $expectedParent -and $runtimeAgrees -and ($listenerPid -eq 0 -or $listenerPid -eq $ExpectedPid) + return [pscustomobject]@{ + owned = $owned; alive = $targetAlive; listenerPid = $listenerPid; pid = $ExpectedPid; start = $targetStart + launcherPid = $ExpectedLauncherPid; launcherStart = $launcherStart; launcherAlive = $launcherAlive; launcherOwned = $expectedParent + } +} + +function Get-InspectSnapshot { + param([string]$BunPath, [string]$CliPath, [string]$ScriptPath, [string]$Root, [string]$OpenCodexDirectory, [string]$CodexConfigured, [string]$CodexCanonical) + $runtime = Read-RuntimePortSnapshot -OpenCodexDirectory $OpenCodexDirectory + $listenerPid = Get-ListenerPid $Port + $candidateIds = @(@($runtime.pid, $listenerPid) | Where-Object { $_ -gt 0 } | Select-Object -Unique) + if ($candidateIds.Count -ne 1 -or ($runtime.port -ne 0 -and $runtime.port -ne $Port)) { + return [pscustomobject]@{ owned = $false; alive = $false; listenerPid = $listenerPid; pid = 0; start = ''; launcherPid = 0; launcherStart = ''; launcherAlive = $false; launcherOwned = $false } + } + $target = Get-ProcessExact ([int]$candidateIds[0]) + $targetAlive = $null -ne $target + $targetStart = if ($targetAlive) { Convert-ToTicksString $target.CreationDate } else { '' } + $parent = if ($targetAlive -and $target.ParentProcessId -gt 0) { Get-ProcessExact ([int]$target.ParentProcessId) } else { $null } + $launcherAlive = $null -ne $parent + $launcherStart = if ($launcherAlive) { Convert-ToTicksString $parent.CreationDate } else { '' } + $launcherPid = if ($launcherAlive) { [int]$parent.ProcessId } else { 0 } + $launcherOwned = Test-VisibleLauncherCandidate -Parent $parent -ScriptPath $ScriptPath -Root $Root -OpenHome $OpenCodexDirectory -CodexConfigured $CodexConfigured -CodexCanonical $CodexCanonical + $childOwned = Test-InspectBunChild -Child $target -BunPath $BunPath -CliPath $CliPath -ListenPort $Port + $runtimeAgrees = (($runtime.port -eq 0 -or $runtime.port -eq $Port) -and ($runtime.pid -eq 0 -or $runtime.pid -eq $target.ProcessId)) + $owned = $targetAlive -and $childOwned -and $launcherOwned -and $runtimeAgrees -and ($listenerPid -eq 0 -or $listenerPid -eq $target.ProcessId) + return [pscustomobject]@{ owned = $owned; alive = $targetAlive; listenerPid = $listenerPid; pid = if ($targetAlive) { [int]$target.ProcessId } else { [int]$candidateIds[0] }; start = $targetStart; launcherPid = $launcherPid; launcherStart = $launcherStart; launcherAlive = $launcherAlive; launcherOwned = $launcherOwned } +} + +function Same-Snapshot { + param($Left, $Right) + return $Left.owned -eq $Right.owned -and $Left.alive -eq $Right.alive -and $Left.listenerPid -eq $Right.listenerPid -and + $Left.start -ceq $Right.start -and $Left.launcherAlive -eq $Right.launcherAlive -and $Left.launcherOwned -eq $Right.launcherOwned -and $Left.launcherStart -ceq $Right.launcherStart +} + +function Read-RecoveryIntent { + param([string]$OpenCodexDirectory) + $path = Join-Path $OpenCodexDirectory 'recovery-intent.json' + if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { return [pscustomobject]@{ valid = $false; reason = 'missing-intent'; mode = ''; at = 0L } } + $intent = Read-BoundedJson -Path $path -MaximumBytes $IntentMaxBytes + if ($null -eq $intent -or -not ($intent.version -is [int] -or $intent.version -is [long]) -or $intent.version -ne 1 -or $intent.mode -isnot [string] -or -not ($intent.at -is [int] -or $intent.at -is [long]) -or $intent.at -lt 0) { + return [pscustomobject]@{ valid = $false; reason = 'invalid-intent'; mode = ''; at = 0L } + } + if ($intent.mode -notin @('stopped', 'running', 'maintenance')) { return [pscustomobject]@{ valid = $false; reason = 'invalid-intent'; mode = ''; at = 0L } } + if ($intent.mode -eq 'stopped') { return [pscustomobject]@{ valid = $false; reason = 'manual-stop'; mode = 'stopped'; at = [long]$intent.at } } + if ($intent.mode -eq 'maintenance') { + try { + if (-not ($intent.until -is [int] -or $intent.until -is [long]) -or $intent.until -le $intent.at) { throw 'invalid' } + $nowMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + if ($intent.until -le $nowMs) { throw 'expired' } + } catch { return [pscustomobject]@{ valid = $false; reason = 'maintenance-expired'; mode = 'maintenance'; at = [long]$intent.at } } + } elseif ($null -ne $intent.PSObject.Properties['until']) { + return [pscustomobject]@{ valid = $false; reason = 'invalid-intent'; mode = ''; at = 0L } + } + return [pscustomobject]@{ valid = $true; reason = 'allowed'; mode = [string]$intent.mode; at = [long]$intent.at } +} + +function Test-CurrentRunningIntent { + param([string]$OpenCodexDirectory, [long]$ExpectedAt) + $current = Read-RecoveryIntent -OpenCodexDirectory $OpenCodexDirectory + if (-not $current.valid) { return $current } + if ($current.mode -ne 'running') { return [pscustomobject]@{ valid = $false; reason = 'intent-not-running'; mode = $current.mode; at = $current.at } } + if ($current.at -ne $ExpectedAt) { return [pscustomobject]@{ valid = $false; reason = 'intent-changed'; mode = $current.mode; at = $current.at } } + return $current +} + +function New-RecoveryBoundaryResult { + param([bool]$Valid, [string]$Reason, $Snapshot) + return [pscustomobject]@{ valid = $Valid; reason = $Reason; snapshot = $Snapshot } +} + +function Test-RecoveryBoundary { + param( + [Parameter(Mandatory)][ValidateSet('before-stop', 'after-stop', 'before-start')][string]$Boundary, + [Parameter(Mandatory)]$ExpectedSnapshot, + [Parameter(Mandatory)][string]$BunPath, + [Parameter(Mandatory)][string]$CliPath, + [Parameter(Mandatory)][string]$ScriptPath, + [Parameter(Mandatory)][string]$Root, + [Parameter(Mandatory)][string]$OpenCodexDirectory, + [Parameter(Mandatory)][string]$CodexConfigured, + [Parameter(Mandatory)][string]$CodexCanonical, + [Parameter(Mandatory)][long]$ExpectedIntentAt + ) + # Intent is read on both sides of each process/port observation. This does + # not claim a cross-process lock, but it makes a user stop/maintenance + # transition during a long CLI stop or its post-stop checks terminal before + # this helper can dispatch a replacement. + $intentBefore = Test-CurrentRunningIntent -OpenCodexDirectory $OpenCodexDirectory -ExpectedAt $ExpectedIntentAt + if (-not $intentBefore.valid) { return New-RecoveryBoundaryResult -Valid $false -Reason $intentBefore.reason -Snapshot $ExpectedSnapshot } + $current = Get-OwnershipSnapshot -BunPath $BunPath -CliPath $CliPath -ScriptPath $ScriptPath -Root $Root -OpenCodexDirectory $OpenCodexDirectory -CodexConfigured $CodexConfigured -CodexCanonical $CodexCanonical + if ($Boundary -eq 'before-stop') { + if (-not (Same-Snapshot $ExpectedSnapshot $current) -or -not $current.alive -or -not $current.owned) { + return New-RecoveryBoundaryResult -Valid $false -Reason 'snapshot-changed' -Snapshot $current + } + } else { + # The old child must remain dead and the old, exact visible launcher + # generation must remain present. A new listener, a PID reuse, or a + # manually closed owner window is never authorization to start another. + if ($current.alive -or $current.listenerPid -ne 0) { + return New-RecoveryBoundaryResult -Valid $false -Reason 'stop-not-confirmed' -Snapshot $current + } + if (-not $current.launcherAlive -or -not $current.launcherOwned) { + return New-RecoveryBoundaryResult -Valid $false -Reason 'launcher-generation-changed' -Snapshot $current + } + if (-not (Test-PortClosedTwice)) { + return New-RecoveryBoundaryResult -Valid $false -Reason 'port-closed-check-failed' -Snapshot $current + } + } + $intentAfter = Test-CurrentRunningIntent -OpenCodexDirectory $OpenCodexDirectory -ExpectedAt $ExpectedIntentAt + if (-not $intentAfter.valid) { return New-RecoveryBoundaryResult -Valid $false -Reason $intentAfter.reason -Snapshot $current } + return New-RecoveryBoundaryResult -Valid $true -Reason 'allowed' -Snapshot $current +} + +function Initialize-DiscardDrainType { + if ($null -ne ('OcxGuardianDiscardDrain' -as [type])) { return } + Add-Type -TypeDefinition @' +using System; +using System.IO; +using System.Threading.Tasks; +public sealed class OcxGuardianDrainResult { public long Count; public bool Capped; } +public static class OcxGuardianDiscardDrain { + public static async Task DrainAsync(Stream stream, long cap) { + var result = new OcxGuardianDrainResult(); var buffer = new byte[4096]; int read; + while ((read = await stream.ReadAsync(buffer, 0, buffer.Length)) > 0) { + if (result.Count < cap) result.Count = Math.Min(cap, result.Count + read); + if (read > 0 && result.Count >= cap) result.Capped = true; + } + return result; + } +} +'@ | Out-Null +} + +function Invoke-GracefulProjectStop { + param([string]$BunPath, [string]$CliPath, [string]$OpenCodexDirectory, [string]$CodexDirectory) + Initialize-DiscardDrainType + $info = New-Object System.Diagnostics.ProcessStartInfo + $info.FileName = $BunPath + $info.Arguments = ('"{0}" stop' -f $CliPath.Replace('"', '""')) + $info.WorkingDirectory = $ProjectRoot + $info.UseShellExecute = $false + $info.CreateNoWindow = $true + $info.RedirectStandardOutput = $true + $info.RedirectStandardError = $true + $info.EnvironmentVariables['OPENCODEX_HOME'] = $OpenCodexDirectory + $info.EnvironmentVariables['CODEX_HOME'] = $CodexDirectory + $info.EnvironmentVariables['OPENCODEX_GUARDIAN_RECOVERY'] = '1' + $child = New-Object System.Diagnostics.Process + $child.StartInfo = $info + if (-not $child.Start()) { return 'stop-start-failed' } + $stdout = [OcxGuardianDiscardDrain]::DrainAsync($child.StandardOutput.BaseStream, $OutputCounterCapBytes) + $stderr = [OcxGuardianDiscardDrain]::DrainAsync($child.StandardError.BaseStream, $OutputCounterCapBytes) + try { + if (-not $child.WaitForExit($StopTimeoutMs)) { return 'stop-timeout' } + [Threading.Tasks.Task]::WaitAll(@($stdout, $stderr), 5000) + if ($child.ExitCode -ne 0) { return 'stop-exit-nonzero' } + return 'stop-exit-zero' + } finally { $child.Dispose() } +} + +function Test-PortClosedTwice { + if ((Get-ListenerPid $Port) -ne 0) { return $false } + Start-Sleep -Milliseconds 250 + return (Get-ListenerPid $Port) -eq 0 +} + +function Start-VisibleLauncher { + param([string]$ScriptPath, [string]$CodexHomeArgument) + $powershell = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe' + if (-not (Test-Path -LiteralPath $powershell -PathType Leaf)) { return $false } + function ConvertTo-NativeArgument([string]$Value) { return '"' + $Value.Replace('"', '\"') + '"' } + $launchArguments = @('-NoProfile', '-File', (ConvertTo-NativeArgument $ScriptPath), '-ProjectRoot', (ConvertTo-NativeArgument $ProjectRoot), '-OpenCodexHome', (ConvertTo-NativeArgument $OpenCodexHome), '-CodexHome', (ConvertTo-NativeArgument $CodexHomeArgument), '-Port', ([string]$Port), '-ConsoleLevel', 'Warn') + $launcher = Start-Process -FilePath $powershell -ArgumentList ($launchArguments -join ' ') -WorkingDirectory $ProjectRoot -WindowStyle Normal -PassThru + return $null -ne $launcher +} + +# A malformed/missing durable intent is a pure refusal. Check it before any +# project inspection so this boundary remains harmless for isolated fixtures. +if ($Mode -eq 'Recover') { + if ($ExpectedPid -le 0 -or $ExpectedLauncherPid -le 0 -or -not (Test-TicksString $ExpectedStart) -or -not (Test-TicksString $ExpectedLauncherStart)) { + Write-ActionResult (New-ActionResult -Action 'refused' -Reason 'invalid-expected-identity' -ProcessId $ExpectedPid -LauncherPid $ExpectedLauncherPid) + exit 0 + } + try { $intentHome = Assert-NonReparsePath -Path $OpenCodexHome -Leaf $false } catch { + Write-ActionResult (New-ActionResult -Action 'refused' -Reason 'invalid-input' -ProcessId $ExpectedPid -LauncherPid $ExpectedLauncherPid) + exit 0 + } + $earlyIntent = Read-RecoveryIntent -OpenCodexDirectory $intentHome + if (-not $earlyIntent.valid) { + Write-ActionResult (New-ActionResult -Action 'refused' -Reason $earlyIntent.reason -ProcessId $ExpectedPid -LauncherPid $ExpectedLauncherPid) + exit 0 + } +} + +try { + $ProjectRoot = Assert-NonReparsePath -Path $ProjectRoot -Leaf $false + $OpenCodexHome = Assert-NonReparsePath -Path $OpenCodexHome -Leaf $false + $codexHomePaths = Get-StableCodexHome -Path $CodexHome + $CodexHome = $codexHomePaths.configured + # Re-read after trusted path validation: a valid intent must remain valid + # at the point where it could authorize an action. + $intent = if ($Mode -eq 'Recover') { Read-RecoveryIntent -OpenCodexDirectory $OpenCodexHome } else { $null } + if ($Mode -eq 'Recover' -and -not $intent.valid) { + Write-ActionResult (New-ActionResult -Action 'refused' -Reason $intent.reason -ProcessId $ExpectedPid -LauncherPid $ExpectedLauncherPid) + exit 0 + } + if ($Mode -eq 'Recover' -and $intent.mode -ne 'running') { + Write-ActionResult (New-ActionResult -Action 'refused' -Reason 'intent-not-running' -ProcessId $ExpectedPid -LauncherPid $ExpectedLauncherPid) + exit 0 + } + $intentAt = if ($Mode -eq 'Recover') { [long]$intent.at } else { 0L } + $repositoryRoot = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..\..')) + if (-not [string]::Equals($ProjectRoot, $repositoryRoot, [System.StringComparison]::OrdinalIgnoreCase)) { throw 'unexpected-project-root' } + $bunPath = Assert-NonReparsePath -Path (Join-Path $ProjectRoot 'node_modules\bun\bin\bun.exe') -Leaf $true + $cliPath = Assert-NonReparsePath -Path (Join-Path $ProjectRoot 'src\cli\index.ts') -Leaf $true + $visibleScriptPath = Assert-NonReparsePath -Path (Join-Path $ProjectRoot 'scripts\windows-visible-proxy.ps1') -Leaf $true + + if ($Mode -eq 'Inspect') { + $inspect = Get-InspectSnapshot -BunPath $bunPath -CliPath $cliPath -ScriptPath $visibleScriptPath -Root $ProjectRoot -OpenCodexDirectory $OpenCodexHome -CodexConfigured $codexHomePaths.configured -CodexCanonical $codexHomePaths.canonical + Write-ActionResult (New-ActionResult -Action 'inspect' -Reason 'inspect' -Owned $inspect.owned -Alive $inspect.alive -ListenerPid $inspect.listenerPid -ProcessId $inspect.pid -Start $inspect.start -LauncherPid $inspect.launcherPid -LauncherStart $inspect.launcherStart -LauncherAlive $inspect.launcherAlive) + exit 0 + } + + $first = Get-OwnershipSnapshot -BunPath $bunPath -CliPath $cliPath -ScriptPath $visibleScriptPath -Root $ProjectRoot -OpenCodexDirectory $OpenCodexHome -CodexConfigured $codexHomePaths.configured -CodexCanonical $codexHomePaths.canonical + + Start-Sleep -Milliseconds 200 + $second = Get-OwnershipSnapshot -BunPath $bunPath -CliPath $cliPath -ScriptPath $visibleScriptPath -Root $ProjectRoot -OpenCodexDirectory $OpenCodexHome -CodexConfigured $codexHomePaths.configured -CodexCanonical $codexHomePaths.canonical + if (-not (Same-Snapshot $first $second)) { + Write-ActionResult (New-ActionResult -Action 'refused' -Reason 'snapshot-changed' -Owned $second.owned -Alive $second.alive -ListenerPid $second.listenerPid -ProcessId $second.pid -Start $second.start -LauncherPid $second.launcherPid -LauncherStart $second.launcherStart -LauncherAlive $second.launcherAlive) + exit 0 + } + + if ($second.alive) { + if (-not $second.owned) { + Write-ActionResult (New-ActionResult -Action 'refused' -Reason 'ownership-lost' -Owned $second.owned -Alive $second.alive -ListenerPid $second.listenerPid -ProcessId $second.pid -Start $second.start -LauncherPid $second.launcherPid -LauncherStart $second.launcherStart -LauncherAlive $second.launcherAlive) + exit 0 + } + $beforeStopBoundary = Test-RecoveryBoundary -Boundary 'before-stop' -ExpectedSnapshot $second -BunPath $bunPath -CliPath $cliPath -ScriptPath $visibleScriptPath -Root $ProjectRoot -OpenCodexDirectory $OpenCodexHome -CodexConfigured $codexHomePaths.configured -CodexCanonical $codexHomePaths.canonical -ExpectedIntentAt $intentAt + if (-not $beforeStopBoundary.valid) { + $failed = $beforeStopBoundary.snapshot + Write-ActionResult (New-ActionResult -Action 'refused' -Reason $beforeStopBoundary.reason -Owned $failed.owned -Alive $failed.alive -ListenerPid $failed.listenerPid -ProcessId $failed.pid -Start $failed.start -LauncherPid $failed.launcherPid -LauncherStart $failed.launcherStart -LauncherAlive $failed.launcherAlive) + exit 0 + } + $stopStatus = Invoke-GracefulProjectStop -BunPath $bunPath -CliPath $cliPath -OpenCodexDirectory $OpenCodexHome -CodexDirectory $codexHomePaths.canonical + if ($stopStatus -ne 'stop-exit-zero') { + Write-ActionResult (New-ActionResult -Action 'refused' -Reason 'stop-uncertain' -Owned $second.owned -Alive $second.alive -ListenerPid $second.listenerPid -ProcessId $second.pid -Start $second.start -LauncherPid $second.launcherPid -LauncherStart $second.launcherStart -LauncherAlive $second.launcherAlive -StopStatus $stopStatus) + exit 0 + } + $afterStopBoundary = Test-RecoveryBoundary -Boundary 'after-stop' -ExpectedSnapshot $second -BunPath $bunPath -CliPath $cliPath -ScriptPath $visibleScriptPath -Root $ProjectRoot -OpenCodexDirectory $OpenCodexHome -CodexConfigured $codexHomePaths.configured -CodexCanonical $codexHomePaths.canonical -ExpectedIntentAt $intentAt + if (-not $afterStopBoundary.valid) { + $failed = $afterStopBoundary.snapshot + Write-ActionResult (New-ActionResult -Action 'refused' -Reason $afterStopBoundary.reason -Owned $failed.owned -Alive $failed.alive -ListenerPid $failed.listenerPid -ProcessId $failed.pid -Start $failed.start -LauncherPid $failed.launcherPid -LauncherStart $failed.launcherStart -LauncherAlive $failed.launcherAlive -StopStatus $stopStatus) + exit 0 + } + } else { + if ($intent.mode -ne 'running') { + Write-ActionResult (New-ActionResult -Action 'refused' -Reason 'dead-pid-requires-running-intent' -Owned $second.owned -Alive $second.alive -ListenerPid $second.listenerPid -ProcessId $second.pid -Start $second.start -LauncherPid $second.launcherPid -LauncherStart $second.launcherStart -LauncherAlive $second.launcherAlive) + exit 0 + } + if (-not $second.launcherAlive -or -not $second.launcherOwned) { + Write-ActionResult (New-ActionResult -Action 'refused' -Reason 'launcher-not-alive' -Owned $second.owned -Alive $second.alive -ListenerPid $second.listenerPid -ProcessId $second.pid -Start $second.start -LauncherPid $second.launcherPid -LauncherStart $second.launcherStart -LauncherAlive $second.launcherAlive) + exit 0 + } + if ($second.listenerPid -ne 0 -or -not (Test-PortClosedTwice)) { + Write-ActionResult (New-ActionResult -Action 'refused' -Reason 'foreign-listener' -Owned $second.owned -Alive $second.alive -ListenerPid $second.listenerPid -ProcessId $second.pid -Start $second.start -LauncherPid $second.launcherPid -LauncherStart $second.launcherStart -LauncherAlive $second.launcherAlive) + exit 0 + } + } + + $codexHomeAfter = Get-StableCodexHome -Path $CodexHome + if (-not [string]::Equals($codexHomePaths.canonical, $codexHomeAfter.canonical, [System.StringComparison]::OrdinalIgnoreCase)) { + Write-ActionResult (New-ActionResult -Action 'refused' -Reason 'codex-home-changed' -Owned $false -Alive $false -ListenerPid (Get-ListenerPid $Port) -ProcessId $ExpectedPid -LauncherPid $ExpectedLauncherPid) + exit 0 + } + # Configuration canonicality is another mutation boundary. Re-validate the + # old process generation and the unchanged running intent immediately + # before dispatching a new visible launcher. + $beforeStartBoundary = Test-RecoveryBoundary -Boundary 'before-start' -ExpectedSnapshot $second -BunPath $bunPath -CliPath $cliPath -ScriptPath $visibleScriptPath -Root $ProjectRoot -OpenCodexDirectory $OpenCodexHome -CodexConfigured $codexHomePaths.configured -CodexCanonical $codexHomePaths.canonical -ExpectedIntentAt $intentAt + if (-not $beforeStartBoundary.valid) { + $failed = $beforeStartBoundary.snapshot + Write-ActionResult (New-ActionResult -Action 'refused' -Reason $beforeStartBoundary.reason -Owned $failed.owned -Alive $failed.alive -ListenerPid $failed.listenerPid -ProcessId $failed.pid -Start $failed.start -LauncherPid $failed.launcherPid -LauncherStart $failed.launcherStart -LauncherAlive $failed.launcherAlive) + exit 0 + } + if (-not (Start-VisibleLauncher -ScriptPath $visibleScriptPath -CodexHomeArgument $codexHomePaths.configured)) { + Write-ActionResult (New-ActionResult -Action 'refused' -Reason 'visible-launch-failed' -Owned $false -Alive $false -ListenerPid 0 -ProcessId $ExpectedPid -LauncherPid $ExpectedLauncherPid) + exit 0 + } + Write-ActionResult (New-ActionResult -Action 'started' -Reason 'visible-launcher-dispatched' -Owned $false -Alive $false -ListenerPid 0 -ProcessId $ExpectedPid -LauncherPid $ExpectedLauncherPid -StopStatus 'confirmed') + exit 0 +} catch { + Write-ActionResult (New-ActionResult -Action 'refused' -Reason 'invalid-input' -ProcessId $ExpectedPid -LauncherPid $ExpectedLauncherPid) + exit 0 +} diff --git a/scripts/windows-visible-log-preload.ts b/scripts/windows-visible-log-preload.ts new file mode 100644 index 00000000000..6d9eee7a5fe --- /dev/null +++ b/scripts/windows-visible-log-preload.ts @@ -0,0 +1,35 @@ +/** Desktop console only: tag severity without dropping the full file transcript. */ +const originalWarn = console.warn.bind(console); +const originalError = console.error.bind(console); + +console.warn = (...args: unknown[]) => { + // This scalar-only timing event is instrumentation, not an operational warning. + // Keep it in the transcript, but do not flood the warning-only console. + let level = "WARN"; + if (args.length === 1 && typeof args[0] === "string" && args[0].startsWith("{")) { + try { + const row = JSON.parse(args[0]); + if (!row?.level && !row?.error) { + if (row?.event === "codex-account-list-timing") level = "INFO"; + // Successful hardening is diagnostics, not a permission failure. Unknown + // and failed outcomes remain visible; never suppress all ACL events. + if (row?.event === "management-token-acl-hardening" && row.ok === true && row.errorCode === "none") level = "INFO"; + } + } catch { /* ordinary warning text */ } + } + originalWarn(`[OCX:${level}]`, ...args); +}; +console.error = (...args: unknown[]) => { + // Some CLI notices intentionally use stderr, but explicitly say WARNING. + // Only recognize a leading marker; an exception mentioning a warning stays red. + const message = typeof args[0] === "string" ? args[0].replace(/\x1b\[[0-?]*[ -/]*[@-~]/g, "").trimStart() : ""; + originalError(message.startsWith("WARNING:") ? "[OCX:WARN]" : "[OCX:ERROR]", ...args); +}; + +// The launcher records the authoritative PID and OS exit status after it has +// drained both pipes. This marker is the child-side counterpart: it confirms +// that ordinary Bun shutdown reached its exit hook without serializing an +// exception, command line, environment, or request data. +process.on("exit", (code) => { + originalWarn(`[OCX:INFO] child-process-exit code=${code}`); +}); diff --git a/scripts/windows-visible-proxy.ps1 b/scripts/windows-visible-proxy.ps1 new file mode 100644 index 00000000000..372044cff92 --- /dev/null +++ b/scripts/windows-visible-proxy.ps1 @@ -0,0 +1,1048 @@ +[CmdletBinding()] +param( + [string]$ProjectRoot, + [string]$OpenCodexHome, + [string]$CodexHome, + [ValidateRange(1, 65535)] + [int]$Port = 10100, + [switch]$CheckOnly, + [switch]$Restart, + [ValidateSet('Warn', 'Error', 'All')] + [string]$ConsoleLevel = 'Warn', + # Keeps automated, isolated checks from waiting for keyboard input after the fake child exits. + [switch]$NoPause +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +if ([string]::IsNullOrWhiteSpace($ProjectRoot)) { + $ProjectRoot = Split-Path -Parent $PSScriptRoot +} +if ([string]::IsNullOrWhiteSpace($OpenCodexHome)) { + $OpenCodexHome = Join-Path ([Environment]::GetFolderPath("UserProfile")) ".opencodex" +} +if ([string]::IsNullOrWhiteSpace($CodexHome)) { + $CodexHome = Join-Path ([Environment]::GetFolderPath("UserProfile")) ".codex" +} + +$LogFileName = "windows-visible-proxy.log" +$MaxLogBytes = 2MB +$MaxChildLineCharacters = 16384 +$StopCommandWaitMs = 120000 +$script:VisibleProxyMutex = $null +$script:OwnsVisibleProxyMutex = $false +$script:RestartMutex = $null +$script:OwnsRestartMutex = $false +$script:LauncherExitCode = 1 +$script:LauncherFailurePhase = 'initialization' +$script:LastConsoleLevel = @{ stdout = 'INFO'; stderr = 'ERROR'; launcher = 'INFO' } + +function Get-VisibleLogRecord { + param([string]$Channel, [AllowEmptyString()][string]$Message) + # The preload tags console methods before Bun merges warn/error into stderr. + # Remove child ANSI controls: the viewer, not arbitrary output, owns the colors. + $clean = [regex]::Replace($Message, "\x1B\[[0-?]*[ -/]*[@-~]", '') + $level = if ($Channel -eq 'stderr') { 'ERROR' } else { 'INFO' } + if ($clean -match '^\[OCX:(INFO|WARN|ERROR)\]\s?(.*)$') { + $level = $Matches[1] + $clean = $Matches[2] + } elseif ($clean -match '^\s+\S' -and $script:LastConsoleLevel.ContainsKey($Channel)) { + $level = $script:LastConsoleLevel[$Channel] + } elseif ($clean -match '^\s*\[(WARN|WARNING|ERROR|FATAL)\]') { + $level = if ($Matches[1] -in @('WARN','WARNING')) { 'WARN' } else { 'ERROR' } + } + $script:LastConsoleLevel[$Channel] = $level + return [pscustomobject]@{ Level = $level; Message = $clean } +} + +function Write-VisibleConsole { + param([string]$Channel, [AllowEmptyString()][string]$Message, [string]$Timestamp = (Get-Date -Format 'HH:mm:ss')) + $record = Get-VisibleLogRecord -Channel $Channel -Message $Message + if ($ConsoleLevel -eq 'Error' -and $record.Level -ne 'ERROR') { return } + if ($ConsoleLevel -eq 'Warn' -and $record.Level -eq 'INFO') { return } + if ([string]::IsNullOrWhiteSpace($record.Message)) { return } + $color = switch ($record.Level) { 'ERROR' { 'Red' }; 'WARN' { 'Yellow' }; default { 'Gray' } } + Write-Host (" {0} " -f $Timestamp) -ForegroundColor DarkGray -NoNewline + Write-Host ("{0,-5} " -f $record.Level) -ForegroundColor $color -NoNewline + Write-Host $record.Message -ForegroundColor $color +} + +function Show-VisibleHeader { + param([string]$Mode, [int]$ListenPort) + try { $Host.UI.RawUI.WindowTitle = "OpenCodex | $Mode | :$ListenPort | $ConsoleLevel" } catch { } + Write-Host '' + Write-Host ' OPENCODEX / PROJECT CONSOLE' -ForegroundColor Cyan + Write-Host (" {0} | http://127.0.0.1:{1} | display: {2}" -f $Mode, $ListenPort, $ConsoleLevel) -ForegroundColor Gray + Write-Host ' WARN = yellow ERROR = red Full output is retained in the rotating log.' -ForegroundColor DarkGray + Write-Host ' Quiet output means no matching messages, not a health/readiness verdict.' -ForegroundColor DarkGray + Write-Host (' ' + ('-' * 76)) -ForegroundColor DarkGray +} + +function Disable-ConsoleQuickEdit { + # Classic conhost pauses synchronous Write-Host while selecting text. If the + # reader then stops draining Bun's pipes, a busy proxy can block on logging. + # Only alter this console's input mode; never change global console settings. + try { + if ($null -eq ('OcxVisibleConsoleMode' -as [type])) { + Add-Type -TypeDefinition @' +using System; +using System.Runtime.InteropServices; +public static class OcxVisibleConsoleMode { + [DllImport("kernel32.dll")] public static extern IntPtr GetStdHandle(int id); + [DllImport("kernel32.dll")] public static extern bool GetConsoleMode(IntPtr handle, out uint mode); + [DllImport("kernel32.dll")] public static extern bool SetConsoleMode(IntPtr handle, uint mode); +} +'@ + } + $inputHandle = [OcxVisibleConsoleMode]::GetStdHandle(-10) + [uint32]$mode = 0 + if ([OcxVisibleConsoleMode]::GetConsoleMode($inputHandle, [ref]$mode)) { + [void][OcxVisibleConsoleMode]::SetConsoleMode($inputHandle, (($mode -bor 0x80) -band (-bnot 0x40))) + } + } catch { } # redirected/test hosts need no console mode +} + +function Resolve-AbsolutePath { + param([Parameter(Mandatory)][string]$Path) + return [System.IO.Path]::GetFullPath($Path) +} + +function Set-RecoveryIntent { + param( + [Parameter(Mandatory)][string]$OpenCodexDirectory, + [Parameter(Mandatory)][ValidateSet('running', 'stopped', 'maintenance')][string]$Mode, + [long]$Until = 0 + ) + $helper = Join-Path $ProjectRoot 'scripts\ocx-recovery-guardian\intent.ps1' + if (-not (Test-Path -LiteralPath $helper -PathType Leaf)) { + throw 'Recovery intent helper is missing; visible launcher did not dispatch a lifecycle action.' + } + # Do not splat an array here: Windows PowerShell treats it as positional + # arguments, so the home path can bind to the helper's -Mode parameter. + $intentArgs = @{ OpenCodexHome = $OpenCodexDirectory; Mode = $Mode } + if ($Mode -eq 'maintenance') { $intentArgs.Until = $Until } + & $helper @intentArgs +} + +function Get-VisibleProxyMutexName { + param( + [Parameter(Mandatory)][string]$Root, + [Parameter(Mandatory)][int]$ListenPort + ) + + $identity = "{0}|{1}" -f $Root.TrimEnd("\").ToUpperInvariant(), $ListenPort + $sha256 = [System.Security.Cryptography.SHA256]::Create() + try { + $bytes = [System.Text.Encoding]::UTF8.GetBytes($identity) + $hash = $sha256.ComputeHash($bytes) + return "Local\OpenCodex.VisibleProxy." + ([System.BitConverter]::ToString($hash).Replace("-", "")) + } finally { + $sha256.Dispose() + } +} + +function Test-OpenCodexHealth { + param([Parameter(Mandatory)][int]$ListenPort) + + $request = $null + $response = $null + $reader = $null + try { + $request = [System.Net.HttpWebRequest]::Create("http://127.0.0.1:$ListenPort/healthz") + $request.Method = "GET" + $request.Timeout = 1500 + $request.ReadWriteTimeout = 1500 + $request.AllowAutoRedirect = $false + $request.Proxy = $null + $response = [System.Net.HttpWebResponse]$request.GetResponse() + if ($response.StatusCode -ne [System.Net.HttpStatusCode]::OK) { return $false } + + $reader = New-Object System.IO.StreamReader($response.GetResponseStream(), [System.Text.Encoding]::UTF8, $true, 4096, $false) + $body = $reader.ReadToEnd() | ConvertFrom-Json + return $body.service -ceq "opencodex" + } catch { + return $false + } finally { + if ($null -ne $reader) { $reader.Dispose() } + if ($null -ne $response) { $response.Dispose() } + } +} + +function Get-RecoveryGatewayState { + param([Parameter(Mandatory)][int]$ListenPort) + $request = $null + $response = $null + $reader = $null + try { + $request = [System.Net.HttpWebRequest]::Create("http://127.0.0.1:$ListenPort/healthz") + $request.Method = 'GET' + $request.Timeout = 1200 + $request.ReadWriteTimeout = 1200 + $request.AllowAutoRedirect = $false + $request.Proxy = $null + $response = [System.Net.HttpWebResponse]$request.GetResponse() + $reader = New-Object System.IO.StreamReader($response.GetResponseStream(), [System.Text.Encoding]::UTF8, $true, 8192, $false) + $body = $reader.ReadToEnd() | ConvertFrom-Json -ErrorAction Stop + if ($response.StatusCode -eq [System.Net.HttpStatusCode]::OK -and $body.service -ceq 'ocx-recovery-gateway') { return 'running' } + return 'foreign' + } catch [System.Net.WebException] { + if ($null -ne $_.Exception.Response) { return 'foreign' } + return 'absent' + } catch { + return 'foreign' + } finally { + if ($null -ne $reader) { $reader.Dispose() } + if ($null -ne $response) { $response.Dispose() } + } +} + +function Test-RecoveryGuardianProcessIdentity { + param( + [Parameter(Mandatory)][int]$ListenPort, + [Parameter(Mandatory)][string]$ExpectedNode, + [Parameter(Mandatory)][string]$MainPath, + [Parameter(Mandatory)][string]$MarkerPath + ) + try { + $owners = @( + Get-NetTCPConnection -State Listen -LocalPort $ListenPort -ErrorAction Stop | + Select-Object -ExpandProperty OwningProcess -Unique + ) + if ($owners.Count -ne 1 -or [int]$owners[0] -le 0) { return $false } + $process = Get-CimInstance Win32_Process -Filter ("ProcessId = {0}" -f [int]$owners[0]) -OperationTimeoutSec 3 -ErrorAction Stop + if ($null -eq $process -or [string]::IsNullOrWhiteSpace($process.ExecutablePath) -or [string]::IsNullOrWhiteSpace($process.CommandLine)) { return $false } + if ([System.IO.Path]::GetFullPath($process.ExecutablePath) -cne $ExpectedNode) { return $false } + # WMI can render native argument backslashes escaped. Normalize that + # representation before matching the fixed local argv contract. + $commandLine = $process.CommandLine + while ($commandLine.Contains('\\')) { $commandLine = $commandLine.Replace('\\', '\') } + $main = [regex]::Escape($MainPath) + $marker = [regex]::Escape($MarkerPath) + if (-not [regex]::IsMatch($commandLine, ('(?i)(?:^|\s)(?:"{0}"|''{0}''|{0})(?=\s|$)' -f $main))) { return $false } + return [regex]::IsMatch($commandLine, ('(?i)(?:^|\s)--config\s+(?:"{0}"|''{0}''|{0})(?=\s|$)' -f $marker)) + } catch { return $false } +} + +function ConvertTo-RecoveryGuardianArgument { + param([Parameter(Mandatory)][string]$Value) + + if ([string]::IsNullOrWhiteSpace($Value) -or $Value.IndexOf('"') -ge 0 -or $Value -match '[\x00-\x1F]') { + throw 'Recovery guardian launch path contains an unsafe quote or control character.' + } + + # Windows PowerShell combines ArgumentList elements into a native command + # line. Quote every path and double trailing backslashes so the closing + # quote cannot be consumed by the Windows command-line parser. + return '"' + ($Value -replace '(\\+)$', '$1$1') + '"' +} + +function Ensure-RecoveryGuardian { + param( + [Parameter(Mandatory)][string]$OpenCodexDirectory, + [Parameter(Mandatory)][string]$Root, + [Parameter(Mandatory)][string]$EffectiveCodexHome, + [Parameter(Mandatory)][int]$PrimaryPort + ) + $markerPath = Join-Path $OpenCodexDirectory 'recovery-guardian.json' + if (-not (Test-Path -LiteralPath $markerPath -PathType Leaf)) { return } + try { + $markerInfo = Get-Item -LiteralPath $markerPath -Force -ErrorAction Stop + if ((([int]$markerInfo.Attributes -band [int][System.IO.FileAttributes]::ReparsePoint) -ne 0) -or $markerInfo.Length -gt 16KB) { throw 'unsafe' } + $marker = [System.IO.File]::ReadAllText($markerInfo.FullName, [System.Text.Encoding]::UTF8) | ConvertFrom-Json -ErrorAction Stop + } catch { + throw 'Recovery guardian marker is malformed; visible launcher did not start the proxy.' + } + if ($marker.version -eq 1 -and $marker.enabled -is [bool] -and -not $marker.enabled) { return } + if ($marker.version -ne 1 -or -not ($marker.enabled -is [bool]) -or -not $marker.enabled) { + throw 'Recovery guardian marker is malformed; visible launcher did not start the proxy.' + } + $expectedNode = Join-Path ${env:ProgramFiles} 'nodejs\node.exe' + $mainPath = Join-Path $Root 'scripts\ocx-recovery-guardian\main.cjs' + try { + $approvedConfig = $marker.projectRoot -is [string] -and $marker.openCodexHome -is [string] -and $marker.codexHome -is [string] -and + [System.IO.Path]::GetFullPath($marker.projectRoot) -ceq $Root -and + [System.IO.Path]::GetFullPath($marker.openCodexHome) -ceq $OpenCodexDirectory -and + [System.IO.Path]::GetFullPath($marker.codexHome) -ceq $EffectiveCodexHome -and + $marker.nodePath -is [string] -and [System.IO.Path]::GetFullPath($marker.nodePath) -ceq $expectedNode -and + ($marker.listenPort -is [int] -or $marker.listenPort -is [long]) -and $marker.listenPort -ge 1 -and $marker.listenPort -le 65535 -and + ($marker.primaryPort -is [int] -or $marker.primaryPort -is [long]) -and $marker.primaryPort -eq $PrimaryPort -and + $marker.primaryPort -ne $marker.listenPort -and $null -ne $marker.fallback -and $null -ne $marker.fallback.models -and $null -ne $marker.repair -and + (Test-Path -LiteralPath $expectedNode -PathType Leaf) -and (Test-Path -LiteralPath $mainPath -PathType Leaf) + } catch { $approvedConfig = $false } + if (-not $approvedConfig) { + throw 'Recovery guardian marker is not an approved local companion configuration; visible launcher did not start the proxy.' + } + $nodeInfo = Get-Item -LiteralPath $expectedNode -Force -ErrorAction Stop + $mainInfo = Get-Item -LiteralPath $mainPath -Force -ErrorAction Stop + if ((([int]$nodeInfo.Attributes -band [int][System.IO.FileAttributes]::ReparsePoint) -ne 0) -or (([int]$mainInfo.Attributes -band [int][System.IO.FileAttributes]::ReparsePoint) -ne 0)) { + throw 'Recovery guardian executable identity is unsafe; visible launcher did not start the proxy.' + } + $gateway = Get-RecoveryGatewayState -ListenPort ([int]$marker.listenPort) + if ($gateway -eq 'running') { + if (Test-RecoveryGuardianProcessIdentity -ListenPort ([int]$marker.listenPort) -ExpectedNode $expectedNode -MainPath $mainPath -MarkerPath $markerPath) { return } + throw 'Recovery guardian port is occupied by an unrecognized listener; visible launcher did not start the proxy.' + } + if ($gateway -ne 'absent') { throw 'Recovery guardian port is occupied by an unrecognized listener; visible launcher did not start the proxy.' } + $guardianArgs = @( + (ConvertTo-RecoveryGuardianArgument -Value $mainPath), + '--config', + (ConvertTo-RecoveryGuardianArgument -Value $markerPath) + ) + $guardian = Start-Process -FilePath $expectedNode -ArgumentList $guardianArgs -WindowStyle Hidden -PassThru + if ($null -eq $guardian) { throw 'Recovery guardian did not start; visible launcher did not start the proxy.' } + $guardian.Dispose() +} + +function Write-VisibleLog { + param( + [Parameter(Mandatory)][string]$LogPath, + [Parameter(Mandatory)][string]$Channel, + [Parameter(Mandatory)][AllowEmptyString()][string]$Message + ) + + $line = "[{0:yyyy-MM-dd HH:mm:ss.fff K}] [{1}] {2}" -f (Get-Date), $Channel, $Message + Write-VisibleConsole -Channel $Channel -Message $Message + $directory = Split-Path -Parent $LogPath + [System.IO.Directory]::CreateDirectory($directory) | Out-Null + $bytes = [System.Text.Encoding]::UTF8.GetBytes($line + [Environment]::NewLine) + + if (Test-Path -LiteralPath $LogPath -PathType Leaf) { + $length = (Get-Item -LiteralPath $LogPath).Length + if ($length -gt 0 -and $length + $bytes.Length -gt $MaxLogBytes) { + $previous = "$LogPath.1" + if (Test-Path -LiteralPath $previous -PathType Leaf) { + Remove-Item -LiteralPath $previous -Force + } + Move-Item -LiteralPath $LogPath -Destination $previous -Force + } + } + + $utf8 = New-Object System.Text.UTF8Encoding($false) + [System.IO.File]::AppendAllText($LogPath, $line + [Environment]::NewLine, $utf8) +} + +function Test-LoopbackPortOccupied { + param([Parameter(Mandatory)][int]$ListenPort) + + foreach ($endpoint in [System.Net.NetworkInformation.IPGlobalProperties]::GetIPGlobalProperties().GetActiveTcpListeners()) { + if ($endpoint.Port -ne $ListenPort) { continue } + if ($endpoint.Address.Equals([System.Net.IPAddress]::Loopback) -or + $endpoint.Address.Equals([System.Net.IPAddress]::IPv6Loopback) -or + $endpoint.Address.Equals([System.Net.IPAddress]::Any) -or + $endpoint.Address.Equals([System.Net.IPAddress]::IPv6Any)) { + return $true + } + } + return $false +} + +function Release-VisibleProxyMutex { + if ($script:OwnsVisibleProxyMutex -and $null -ne $script:VisibleProxyMutex) { + try { [void]$script:VisibleProxyMutex.ReleaseMutex() } catch { } + $script:OwnsVisibleProxyMutex = $false + } +} + +function Release-RestartMutex { + if ($script:OwnsRestartMutex -and $null -ne $script:RestartMutex) { + try { [void]$script:RestartMutex.ReleaseMutex() } catch { } + $script:OwnsRestartMutex = $false + } +} + +function Acquire-VisibleProxyMutexAfterStop { + # The old visible window releases this owner lock before its post-exit prompt. Bound waiting + # lets Restart hand off without requiring the user to close that window. + for ($attempt = 0; $attempt -lt 60; $attempt += 1) { + try { + $script:OwnsVisibleProxyMutex = $script:VisibleProxyMutex.WaitOne(0) + } catch [System.Threading.AbandonedMutexException] { + $script:OwnsVisibleProxyMutex = $true + } + if ($script:OwnsVisibleProxyMutex) { return } + Start-Sleep -Milliseconds 250 + } + throw "The prior visible launcher did not release its owner lock within 15 seconds; no replacement was started." +} + +function Follow-ExistingLog { + param([Parameter(Mandatory)][string]$LogPath) + + if (-not (Test-Path -LiteralPath $LogPath -PathType Leaf)) { + Write-VisibleConsole -Channel stderr -Message '[OCX:WARN] A background OpenCodex instance is serving this port, but no visible console owns it. This window did not start the proxy. Stop its actual manager before switching to desktop mode.' + if (-not $NoPause) { [void](Read-Host "Press Enter to close this window") } + return + } + + Show-VisibleHeader -Mode 'LOG VIEWER (does not own the proxy)' -ListenPort $Port + Write-Host ' Close this viewer without stopping the owner window.' -ForegroundColor DarkGray + # Poll bounded chunks with ReadWrite/Delete sharing; reopen after rotation instead + # of following a renamed .1 forever. Only the owner writes the file. + $offset = 0L + $created = [DateTime]::MinValue + $partial = '' + do { + if (Test-Path -LiteralPath $LogPath -PathType Leaf) { + try { + $file = Get-Item -LiteralPath $LogPath + if ($file.CreationTimeUtc -ne $created -or $file.Length -lt $offset) { + $offset = 0L; $partial = ''; $created = $file.CreationTimeUtc + } + $stream = [System.IO.File]::Open($LogPath, 'Open', 'Read', ([System.IO.FileShare]::ReadWrite -bor [System.IO.FileShare]::Delete)) + try { + [void]$stream.Seek($offset, 'Begin') + $reader = New-Object System.IO.StreamReader($stream, [System.Text.Encoding]::UTF8) + try { $chunk = $reader.ReadToEnd(); $offset = $stream.Position } finally { $reader.Dispose() } + } finally { $stream.Dispose() } + $lines = ($partial + $chunk).Split("`n") + $partial = $lines[-1] + for ($i = 0; $i -lt $lines.Length - 1; $i++) { + if ($lines[$i] -match '^\[\d{4}-\d{2}-\d{2} (\d{2}:\d{2}:\d{2})[^\]]*\] \[(stdout|stderr|launcher)\] (.*)') { + Write-VisibleConsole -Timestamp $Matches[1] -Channel $Matches[2] -Message $Matches[3].TrimEnd("`r") + } + } + } catch [System.IO.IOException] { } # owner may be rotating between open and read + } + if (-not $NoPause) { Start-Sleep -Milliseconds 500 } + } while (-not $NoPause) +} + +function Write-LauncherLifecycleEvent { + param( + [Parameter(Mandatory)][string]$OpenCodexDirectory, + [Parameter(Mandatory)][string]$EventName, + [int]$ChildPid = 0, + [AllowNull()][Nullable[int]]$ExitCode = $null, + [string]$FailureKind = '', + [string]$Detail = '' + ) + # Separate from the proxy owner's rotating transcript: two launchers must + # never race to rotate that file. Record scalar lifecycle facts, not CLI + # output, arguments, environment, request contents, or exception text. + $eventMutex = $null + $ownsEventMutex = $false + try { + $directory = Join-Path $OpenCodexDirectory 'logs' + [void][System.IO.Directory]::CreateDirectory($directory) + $eventPath = Join-Path $directory 'windows-visible-launcher-events.log' + $identityBytes = [System.Text.Encoding]::UTF8.GetBytes($eventPath.ToUpperInvariant()) + $hash = [System.Security.Cryptography.SHA256]::Create() + try { $mutexName = 'Local\OpenCodex.VisibleLauncherEvents.' + ([System.BitConverter]::ToString($hash.ComputeHash($identityBytes)).Replace('-', '')) } + finally { $hash.Dispose() } + $eventMutex = New-Object System.Threading.Mutex($false, $mutexName) + try { $ownsEventMutex = $eventMutex.WaitOne(2000) } + catch [System.Threading.AbandonedMutexException] { $ownsEventMutex = $true } + if (-not $ownsEventMutex) { throw 'Timed out waiting to record a launcher lifecycle event.' } + $record = [ordered]@{at=(Get-Date).ToString('o');launcherPid=$PID;event=$EventName;childPid=$ChildPid} + if ($null -ne $ExitCode) { + # Process.ExitCode is signed, whereas Windows crash status is commonly + # reported as an unsigned 32-bit hexadecimal value (for example C0000005). + # Persist both representations so a later investigation need not guess. + $record.exitCode = [int]$ExitCode + $record.exitCodeHex = '0x{0:X8}' -f [BitConverter]::ToUInt32([BitConverter]::GetBytes([int]$ExitCode), 0) + } + if (-not [string]::IsNullOrWhiteSpace($FailureKind)) { + # A stable classifier distinguishes launcher exceptions without + # persisting exception messages, which can carry local paths or data. + $record.failureKind = $FailureKind + } + if (-not [string]::IsNullOrWhiteSpace($Detail)) { + $record.detail = $Detail + } + $line = $record | ConvertTo-Json -Compress + if ((Test-Path -LiteralPath $eventPath) -and (Get-Item -LiteralPath $eventPath).Length -gt 256KB) { + Move-Item -LiteralPath $eventPath -Destination ($eventPath + '.1') -Force + } + [System.IO.File]::AppendAllText($eventPath, $line + [Environment]::NewLine, (New-Object System.Text.UTF8Encoding($false))) + } catch { + Write-VisibleConsole stderr '[OCX:WARN] Launcher lifecycle event could not be saved.' + } finally { + if ($ownsEventMutex -and $null -ne $eventMutex) { + try { [void]$eventMutex.ReleaseMutex() } catch { } + } + if ($null -ne $eventMutex) { $eventMutex.Dispose() } + } +} + +function Invoke-ProjectCliStop { + param( + [Parameter(Mandatory)][string]$BunPath, + [Parameter(Mandatory)][string]$CliPath, + [Parameter(Mandatory)][string]$WorkingDirectory, + [Parameter(Mandatory)][string]$OpenCodexDirectory, + [Parameter(Mandatory)][string]$CodexDirectory + ) + + $startInfo = New-Object System.Diagnostics.ProcessStartInfo + $startInfo.FileName = $BunPath + $startInfo.Arguments = ('"{0}" stop' -f $CliPath) + $startInfo.WorkingDirectory = $WorkingDirectory + $startInfo.UseShellExecute = $false + $startInfo.CreateNoWindow = $false + $startInfo.EnvironmentVariables["OPENCODEX_HOME"] = $OpenCodexDirectory + $startInfo.EnvironmentVariables["CODEX_HOME"] = $CodexDirectory + $startInfo.EnvironmentVariables["HTTP_PROXY"] = "http://127.0.0.1:7890" + $startInfo.EnvironmentVariables["HTTPS_PROXY"] = "http://127.0.0.1:7890" + $startInfo.EnvironmentVariables["ALL_PROXY"] = "socks5://127.0.0.1:7891" + $startInfo.EnvironmentVariables["NO_PROXY"] = "localhost,127.0.0.1,::1" + $startInfo.EnvironmentVariables["OPENCODEX_RUNTIME_DIAGNOSTICS"] = "1" + $startInfo.EnvironmentVariables["OPENCODEX_CODEX_UPSTREAM_TRANSPORT"] = "http-sse" + [void]$startInfo.EnvironmentVariables.Remove("OCX_SERVICE") + $startInfo.EnvironmentVariables["OPENCODEX_GUARDIAN_RECOVERY"] = "1" + + $stopProcess = New-Object System.Diagnostics.Process + $stopProcess.StartInfo = $startInfo + if (-not $stopProcess.Start()) { throw "Unable to start the project CLI stop command." } + Write-LauncherLifecycleEvent -OpenCodexDirectory $OpenCodexDirectory -EventName 'stop-command-started' -ChildPid $stopProcess.Id + try { + # CLI stop includes discovery, drain, port reclamation AND shared-config + # restoration. Abandoning it after 15s left a still-running stop command + # that later shut down the proxy after this launcher had given up. + $stopTimer = [System.Diagnostics.Stopwatch]::StartNew() + $nextNoticeMs = 15000 + while (-not $stopProcess.WaitForExit(1000)) { + if ($stopTimer.ElapsedMilliseconds -ge $StopCommandWaitMs) { + Write-LauncherLifecycleEvent -OpenCodexDirectory $OpenCodexDirectory -EventName 'stop-command-timeout' -ChildPid $stopProcess.Id + throw "Project CLI stop exceeded the bounded 120-second wait; it is still running and no replacement was started." + } + if ($stopTimer.ElapsedMilliseconds -ge $nextNoticeMs) { + Write-VisibleConsole stderr '[OCX:WARN] Still waiting for normal stop and config restoration; replacement has not started yet.' + $nextNoticeMs += 15000 + } + } + Write-LauncherLifecycleEvent -OpenCodexDirectory $OpenCodexDirectory -EventName 'stop-command-exited' -ChildPid $stopProcess.Id -ExitCode $stopProcess.ExitCode + if ($stopProcess.ExitCode -ne 0) { + throw "Project CLI stop exited with code $($stopProcess.ExitCode)." + } + } finally { + $stopProcess.Dispose() + } +} + +function Wait-ForOpenCodexToStop { + param([Parameter(Mandatory)][int]$ListenPort) + + # This is a bounded post-stop confirmation, not a restart loop. Do not launch while the + # identity-checked listener still answers, because that would turn an unknown owner into a port race. + for ($attempt = 0; $attempt -lt 60; $attempt += 1) { + if (-not (Test-OpenCodexHealth -ListenPort $ListenPort)) { return } + Start-Sleep -Milliseconds 250 + } + throw "OpenCodex still answered /healthz 15 seconds after the project CLI stop command; no replacement was started." +} + +function Wait-ForFailureAcknowledgement { + if (-not $NoPause) { + [void](Read-Host "The visible OpenCodex process exited. Press Enter to close this window") + } +} + +function Drain-ChildChunkToLog { + param( + [Parameter(Mandatory)][System.Threading.Tasks.Task[int]]$Task, + [Parameter(Mandatory)][char[]]$Characters, + [Parameter(Mandatory)][hashtable]$State, + [Parameter(Mandatory)][string]$Channel, + [Parameter(Mandatory)][string]$LogPath + ) + + $count = $Task.GetAwaiter().GetResult() + if ($count -eq 0) { + if ($State.Buffer.Length -gt 0 -or $State.Truncated) { + $message = $State.Buffer.ToString() + if ($message.EndsWith("`r")) { $message = $message.Substring(0, $message.Length - 1) } + if ($State.Truncated) { $message += " [truncated]" } + Write-VisibleLog -LogPath $LogPath -Channel $Channel -Message $message + } + return $false + } + + for ($index = 0; $index -lt $count; $index += 1) { + $character = $Characters[$index] + if ($character -eq "`n") { + $message = $State.Buffer.ToString() + if ($message.EndsWith("`r")) { $message = $message.Substring(0, $message.Length - 1) } + if ($State.Truncated) { $message += " [truncated]" } + Write-VisibleLog -LogPath $LogPath -Channel $Channel -Message $message + [void]$State.Buffer.Clear() + $State.Truncated = $false + } elseif ($State.Buffer.Length -lt $MaxChildLineCharacters) { + [void]$State.Buffer.Append($character) + } else { + $State.Truncated = $true + } + } + return $true +} + +function Initialize-VisiblePipePumpType { + # A pipe reader must never wait for a visible console or filesystem write. + # The queues make that boundary explicit: when the disk cannot keep up we + # preserve proxy liveness with bounded memory and emit a durable overflow + # summary as soon as the writer recovers. A permanently unavailable disk + # cannot simultaneously provide lossless logs, bounded memory, and a + # non-blocking child pipe; this deliberately chooses the latter two. + if ($null -eq ('OcxVisiblePipePump' -as [type])) { + Add-Type -TypeDefinition @' +using System; +using System.Collections.Concurrent; +using System.IO; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; + +public sealed class OcxVisiblePipePump { + private sealed class Row { public string Channel; public string Message; public Row(string c, string m) { Channel = c; Message = m; } } + private const int MaxLineCharacters = 16384; + private readonly BlockingCollection records = new BlockingCollection(512); + private readonly BlockingCollection console = new BlockingCollection(256); + private readonly Stream stdout; + private readonly Stream stderr; + private readonly string logPath; + private readonly string displayLevel; + private Thread stdoutReader; + private Thread stderrReader; + private Thread writer; + private Thread consoleWriter; + private long droppedRecords; + private long firstDroppedAt; + private long lastDroppedAt; + private long droppedConsoleRows; + private long consoleFailures; + private string stdoutLevel = "INFO"; + private string stderrLevel = "ERROR"; + private string launcherLevel = "INFO"; + + public OcxVisiblePipePump(Stream standardOutput, Stream standardError, string path, string level) { + stdout = standardOutput; stderr = standardError; logPath = path; displayLevel = level; + } + + public void Start() { + consoleWriter = NewThread(ConsoleLoop, "ocx-visible-console"); + writer = NewThread(WriterLoop, "ocx-visible-writer"); + stdoutReader = NewThread(delegate { ReadLoop(stdout, "stdout"); }, "ocx-visible-stdout"); + stderrReader = NewThread(delegate { ReadLoop(stderr, "stderr"); }, "ocx-visible-stderr"); + } + + public void QueueLauncher(string message) { OfferRecord(new Row("launcher", message)); } + + public string StopAndDescribe(int waitMilliseconds) { + stdoutReader.Join(waitMilliseconds); + stderrReader.Join(waitMilliseconds); + if (!stdoutReader.IsAlive && !stderrReader.IsAlive) records.CompleteAdding(); + writer.Join(waitMilliseconds); + if (!writer.IsAlive) { + console.CompleteAdding(); + // Child pipes and disk writer are already drained; boundedly wait + // for diagnostic rows queued by the writer before PowerShell exits. + consoleWriter.Join(waitMilliseconds); + } + return string.Format("recordDrops={0}; consoleDrops={1}; consoleFailures={2}; writerStopped={3}", Interlocked.Read(ref droppedRecords), Interlocked.Read(ref droppedConsoleRows), Interlocked.Read(ref consoleFailures), !writer.IsAlive); + } + + private static Thread NewThread(ThreadStart action, string name) { + Thread thread = new Thread(action); thread.IsBackground = true; thread.Name = name; thread.Start(); return thread; + } + + private void ReadLoop(Stream stream, string channel) { + try { + StreamReader reader = new StreamReader(stream, new UTF8Encoding(false), true, 4096); + char[] characters = new char[4096]; StringBuilder line = new StringBuilder(); bool truncated = false; int count; + while ((count = reader.Read(characters, 0, characters.Length)) != 0) { + for (int index = 0; index < count; index += 1) { + char character = characters[index]; + if (character == '\n') { FinishLine(channel, line, truncated); line.Length = 0; truncated = false; } + else if (line.Length < MaxLineCharacters) line.Append(character); + else truncated = true; + } + } + if (line.Length > 0 || truncated) FinishLine(channel, line, truncated); + } catch (Exception error) { + OfferRecord(new Row("launcher", "[OCX:ERROR] pipe-reader-failed kind=" + error.GetType().Name)); + } + } + + private void FinishLine(string channel, StringBuilder line, bool truncated) { + string message = line.ToString().TrimEnd('\r'); + if (truncated) message += " [truncated]"; + OfferRecord(new Row(channel, message)); + } + + private void OfferRecord(Row row) { + if (records.IsAddingCompleted || !records.TryAdd(row)) { + long now = DateTime.UtcNow.Ticks; + if (Interlocked.Increment(ref droppedRecords) == 1) Interlocked.CompareExchange(ref firstDroppedAt, now, 0); + Interlocked.Exchange(ref lastDroppedAt, now); + } + } + + private void WriterLoop() { + try { + foreach (Row row in records.GetConsumingEnumerable()) { + try { WriteRecoveredOverflow(); Append(row); OfferConsole(row); } + catch (Exception error) { + CountDroppedRecord(); + OfferConsole(new Row("launcher", "[OCX:ERROR] transcript-writer-failed kind=" + error.GetType().Name)); + Thread.Sleep(100); + } + } + try { WriteRecoveredOverflow(); } + catch (Exception error) { OfferConsole(new Row("launcher", "[OCX:ERROR] transcript-writer-failed kind=" + error.GetType().Name)); } + } catch { } + } + + private void CountDroppedRecord() { + long now = DateTime.UtcNow.Ticks; + if (Interlocked.Increment(ref droppedRecords) == 1) Interlocked.CompareExchange(ref firstDroppedAt, now, 0); + Interlocked.Exchange(ref lastDroppedAt, now); + } + + private void WriteRecoveredOverflow() { + long dropped = Interlocked.Exchange(ref droppedRecords, 0); + if (dropped == 0) return; + long first = Interlocked.Exchange(ref firstDroppedAt, 0); long last = Interlocked.Exchange(ref lastDroppedAt, 0); + string message = string.Format("[OCX:ERROR] transcript-overflow droppedRecords={0}; firstUtcTicks={1}; lastUtcTicks={2}; full transcript has a gap.", dropped, first, last); + try { + Append(new Row("launcher", message)); + OfferConsole(new Row("launcher", message)); + } catch { + RestoreDroppedRecords(dropped, first, last); + throw; + } + } + + private void RestoreDroppedRecords(long dropped, long first, long last) { + Interlocked.Add(ref droppedRecords, dropped); + if (first != 0) { + long observed; + do { + observed = Interlocked.Read(ref firstDroppedAt); + if (observed != 0 && observed <= first) break; + } while (Interlocked.CompareExchange(ref firstDroppedAt, first, observed) != observed); + } + if (last != 0) { + long observed; + do { observed = Interlocked.Read(ref lastDroppedAt); if (observed >= last) break; } + while (Interlocked.CompareExchange(ref lastDroppedAt, last, observed) != observed); + } + } + + private void Append(Row row) { + string directory = Path.GetDirectoryName(logPath); if (!String.IsNullOrEmpty(directory)) Directory.CreateDirectory(directory); + string line = string.Format("[{0:yyyy-MM-dd HH:mm:ss.fff K}] [{1}] {2}{3}", DateTime.Now, row.Channel, row.Message, Environment.NewLine); + const long maxLogBytes = 2L * 1024L * 1024L; + FileInfo file = new FileInfo(logPath); + if (file.Exists && file.Length > 0 && file.Length + Encoding.UTF8.GetByteCount(line) > maxLogBytes) { + string previous = logPath + ".1"; if (File.Exists(previous)) File.Delete(previous); File.Move(logPath, previous); + } + File.AppendAllText(logPath, line, new UTF8Encoding(false)); + } + + private void OfferConsole(Row row) { + if (!console.TryAdd(row)) Interlocked.Increment(ref droppedConsoleRows); + } + + private void ConsoleLoop() { + try { + foreach (Row row in console.GetConsumingEnumerable()) { + string visibleMessage = StripAnsi(row.Message); + string level = Classify(row.Channel, visibleMessage); + long skipped = Interlocked.Exchange(ref droppedConsoleRows, 0); + if (skipped > 0) Render("WARN", "visible-console-overflow droppedRows=" + skipped + "; full transcript remains the source of truth."); + if ((displayLevel == "Error" && level != "ERROR") || (displayLevel == "Warn" && level == "INFO")) continue; + if (!String.IsNullOrWhiteSpace(visibleMessage)) Render(level, StripTag(visibleMessage)); + } + } catch { Interlocked.Increment(ref consoleFailures); } + } + + private string Classify(string channel, string message) { + string level = channel == "stderr" ? "ERROR" : "INFO"; + if (message.StartsWith("[OCX:INFO]")) level = "INFO"; + else if (message.StartsWith("[OCX:WARN]")) level = "WARN"; + else if (message.StartsWith("[OCX:ERROR]")) level = "ERROR"; + else if (message.TrimStart().StartsWith("[WARN") || message.TrimStart().StartsWith("[WARNING")) level = "WARN"; + else if (message.TrimStart().StartsWith("[ERROR") || message.TrimStart().StartsWith("[FATAL")) level = "ERROR"; + else if (message.Length > 0 && Char.IsWhiteSpace(message[0])) { + if (channel == "stdout") level = stdoutLevel; else if (channel == "stderr") level = stderrLevel; else level = launcherLevel; + } + if (channel == "stdout") stdoutLevel = level; else if (channel == "stderr") stderrLevel = level; else launcherLevel = level; + return level; + } + + private static string StripTag(string message) { + if (message.StartsWith("[OCX:INFO]")) return message.Substring(10).TrimStart(); + if (message.StartsWith("[OCX:WARN]")) return message.Substring(10).TrimStart(); + if (message.StartsWith("[OCX:ERROR]")) return message.Substring(11).TrimStart(); + return message; + } + + private static string StripAnsi(string message) { + return Regex.Replace(message, "\\x1B\\[[0-?]*[ -/]*[@-~]", String.Empty); + } + + private static void Render(string level, string message) { + ConsoleColor color = level == "ERROR" ? ConsoleColor.Red : (level == "WARN" ? ConsoleColor.Yellow : ConsoleColor.Gray); + Console.ForegroundColor = ConsoleColor.DarkGray; Console.Write(" " + DateTime.Now.ToString("HH:mm:ss") + " "); + Console.ForegroundColor = color; Console.Write(level.PadRight(5) + " "); Console.WriteLine(message); Console.ResetColor(); + } +} +'@ + } +} + +function Start-VisiblePipePump { + param( + [Parameter(Mandatory)][System.IO.Stream]$StandardOutput, + [Parameter(Mandatory)][System.IO.Stream]$StandardError, + [Parameter(Mandatory)][string]$LogPath, + [Parameter(Mandatory)][string]$DisplayLevel + ) + Initialize-VisiblePipePumpType + $pump = New-Object OcxVisiblePipePump($StandardOutput, $StandardError, $LogPath, $DisplayLevel) + $pump.Start() + return $pump +} + +function Invoke-VisibleProxy { + param( + [Parameter(Mandatory)][string]$BunPath, + [Parameter(Mandatory)][string]$CliPath, + [Parameter(Mandatory)][string]$WorkingDirectory, + [Parameter(Mandatory)][string]$LogPath, + [Parameter(Mandatory)][int]$ListenPort, + [Parameter(Mandatory)][string]$OpenCodexDirectory, + [Parameter(Mandatory)][string]$CodexDirectory, + [scriptblock]$OnStarted + ) + + $startInfo = New-Object System.Diagnostics.ProcessStartInfo + $startInfo.FileName = $BunPath + $preloadPath = Join-Path $WorkingDirectory 'scripts\windows-visible-log-preload.ts' + if (-not (Test-Path -LiteralPath $preloadPath -PathType Leaf)) { throw 'Visible log preload is missing.' } + $startInfo.Arguments = ('--preload "{0}" "{1}" start --port {2}' -f $preloadPath, $CliPath, $ListenPort) + $startInfo.WorkingDirectory = $WorkingDirectory + $startInfo.UseShellExecute = $false + $startInfo.CreateNoWindow = $false + $startInfo.RedirectStandardOutput = $true + $startInfo.RedirectStandardError = $true + $startInfo.StandardOutputEncoding = [System.Text.Encoding]::UTF8 + $startInfo.StandardErrorEncoding = [System.Text.Encoding]::UTF8 + $startInfo.EnvironmentVariables["OPENCODEX_HOME"] = $OpenCodexDirectory + $startInfo.EnvironmentVariables["CODEX_HOME"] = $CodexDirectory + $startInfo.EnvironmentVariables["HTTP_PROXY"] = "http://127.0.0.1:7890" + $startInfo.EnvironmentVariables["HTTPS_PROXY"] = "http://127.0.0.1:7890" + $startInfo.EnvironmentVariables["ALL_PROXY"] = "socks5://127.0.0.1:7891" + $startInfo.EnvironmentVariables["NO_PROXY"] = "localhost,127.0.0.1,::1" + $startInfo.EnvironmentVariables["OPENCODEX_RUNTIME_DIAGNOSTICS"] = "1" + $startInfo.EnvironmentVariables["OPENCODEX_CODEX_UPSTREAM_TRANSPORT"] = "http-sse" + [void]$startInfo.EnvironmentVariables.Remove("OCX_SERVICE") + + $child = New-Object System.Diagnostics.Process + $child.StartInfo = $startInfo + # JIT-compile the reader before the child exists. Otherwise a chatty child + # could fill its pipe while Add-Type is compiling and recreate startup + # backpressure before the dedicated reader thread gets a chance to run. + Initialize-VisiblePipePumpType + if (-not $child.Start()) { + throw "Unable to start the local Bun runtime." + } + + $childPid = $child.Id + # Start the reader immediately after Process.Start. It is separate from the + # visible console and disk writer, so neither can pause pipe draining. + $pump = Start-VisiblePipePump -StandardOutput $child.StandardOutput.BaseStream -StandardError $child.StandardError.BaseStream -LogPath $LogPath -DisplayLevel $ConsoleLevel + Write-LauncherLifecycleEvent -OpenCodexDirectory $OpenCodexDirectory -EventName 'proxy-child-started' -ChildPid $childPid + Show-VisibleHeader -Mode ("OWNER PID {0}" -f $childPid) -ListenPort $ListenPort + # The pipe pump owns this transcript from here until it has drained. Never + # append/rotate on the PowerShell thread concurrently with its writer. + $pump.QueueLauncher(("started PID {0}; port={1}" -f $childPid, $ListenPort)) + if ($null -ne $OnStarted) { & $OnStarted } + $child.WaitForExit() + $exitCode = $child.ExitCode + Write-LauncherLifecycleEvent -OpenCodexDirectory $OpenCodexDirectory -EventName 'proxy-child-exited' -ChildPid $childPid -ExitCode $exitCode + $exitLevel = if ($exitCode -eq 0) { 'WARN' } else { 'ERROR' } + $exitCodeHex = '0x{0:X8}' -f [BitConverter]::ToUInt32([BitConverter]::GetBytes([int]$exitCode), 0) + $pump.QueueLauncher(("[OCX:{0}] PID {1} exited with code {2} ({3}); automatic restart is disabled." -f $exitLevel, $childPid, $exitCode, $exitCodeHex)) + # Once the child has exited, bounded flushing no longer risks its liveness. + # Give the writer a finite grace period for ordinary slow storage; if it + # cannot finish, say so visibly and preserve the child exit in lifecycle. + $pumpState = $pump.StopAndDescribe(5000) + if ($pumpState -match 'recordDrops=([1-9]\d*)') { + Write-LauncherLifecycleEvent -OpenCodexDirectory $OpenCodexDirectory -EventName 'proxy-transcript-overflow-unflushed' -ChildPid $childPid -ExitCode $exitCode -Detail ("droppedRecords=" + $Matches[1]) + } + if ($pumpState -match 'consoleDrops=([1-9]\d*)') { + Write-LauncherLifecycleEvent -OpenCodexDirectory $OpenCodexDirectory -EventName 'proxy-visible-console-overflow' -ChildPid $childPid -ExitCode $exitCode -Detail ("droppedRows=" + $Matches[1]) + Write-VisibleConsole -Channel stderr -Message ("[OCX:WARN] Visible console skipped {0} rows; the transcript is the source of truth." -f $Matches[1]) + } + if ($pumpState -match 'consoleFailures=([1-9]\d*)') { + Write-LauncherLifecycleEvent -OpenCodexDirectory $OpenCodexDirectory -EventName 'proxy-visible-console-failed' -ChildPid $childPid -ExitCode $exitCode -Detail ("failures=" + $Matches[1]) + Write-VisibleConsole -Channel stderr -Message '[OCX:ERROR] Visible console writer failed; consult the transcript and lifecycle event log.' + } + if ($pumpState -match 'writerStopped=False') { + Write-LauncherLifecycleEvent -OpenCodexDirectory $OpenCodexDirectory -EventName 'proxy-transcript-drain-timeout' -ChildPid $childPid -ExitCode $exitCode + Write-VisibleConsole -Channel stderr -Message '[OCX:ERROR] Transcript writer did not drain within 5 seconds after child exit; on-disk transcript may be incomplete.' + } + $child.Dispose() + return $exitCode +} + +try { + $script:LauncherFailurePhase = 'path-validation' + $ProjectRoot = Resolve-AbsolutePath $ProjectRoot + $OpenCodexHome = Resolve-AbsolutePath $OpenCodexHome + $CodexHome = Resolve-AbsolutePath $CodexHome + $bunPath = Join-Path $ProjectRoot "node_modules\\bun\\bin\\bun.exe" + $cliPath = Join-Path $ProjectRoot "src\\cli\\index.ts" + $logPath = Join-Path (Join-Path $OpenCodexHome "logs") $LogFileName + + Write-LauncherLifecycleEvent -OpenCodexDirectory $OpenCodexHome -EventName 'launcher-started' + + foreach ($path in @($ProjectRoot, $bunPath, $cliPath)) { + if (-not (Test-Path -LiteralPath $path -PathType Container) -and -not (Test-Path -LiteralPath $path -PathType Leaf)) { + throw "Required visible-launcher path is missing: $path" + } + } + if (-not (Test-Path -LiteralPath $bunPath -PathType Leaf)) { throw "Bun runtime is not a file: $bunPath" } + if (-not (Test-Path -LiteralPath $cliPath -PathType Leaf)) { throw "OpenCodex CLI entry is not a file: $cliPath" } + + if ($CheckOnly) { + Write-Host "Visible OpenCodex launcher check passed for port $Port." + $script:LauncherExitCode = 0 + exit 0 + } + + $script:LauncherFailurePhase = 'guardian' + Ensure-RecoveryGuardian -OpenCodexDirectory $OpenCodexHome -Root $ProjectRoot -EffectiveCodexHome $CodexHome -PrimaryPort $Port + + Disable-ConsoleQuickEdit + + $script:LauncherFailurePhase = 'ownership' + $mutexName = Get-VisibleProxyMutexName -Root $ProjectRoot -ListenPort $Port + $script:VisibleProxyMutex = New-Object System.Threading.Mutex($false, $mutexName) + try { + $script:OwnsVisibleProxyMutex = $script:VisibleProxyMutex.WaitOne(0) + } catch [System.Threading.AbandonedMutexException] { + $script:OwnsVisibleProxyMutex = $true + } + + if ($Restart) { + $script:RestartMutex = New-Object System.Threading.Mutex($false, "$mutexName.Restart") + try { + $script:OwnsRestartMutex = $script:RestartMutex.WaitOne(0) + } catch [System.Threading.AbandonedMutexException] { + $script:OwnsRestartMutex = $true + } + if (-not $script:OwnsRestartMutex) { + throw "Another visible restart is already in progress for this project and port." + } + } + + $maintenanceIntentWritten = $false + $script:LauncherFailurePhase = 'health' + $healthyOpenCodex = Test-OpenCodexHealth -ListenPort $Port + if ($healthyOpenCodex -and -not $Restart) { + # An owner mutex is stronger evidence than a healthy port or an old log. + if ($script:OwnsVisibleProxyMutex) { + Release-VisibleProxyMutex + Show-VisibleHeader -Mode 'BACKGROUND INSTANCE (not this window)' -ListenPort $Port + Write-VisibleConsole -Channel stderr -Message '[OCX:WARN] Port is served outside the visible launcher. No new proxy was started. Check the existing service/manager before desktop migration.' + if (-not $NoPause) { [void](Read-Host 'Press Enter to close this window') } + $script:LauncherExitCode = 0 + exit 0 + } + Release-VisibleProxyMutex + Follow-ExistingLog -LogPath $logPath + $script:LauncherExitCode = 0 + exit 0 + } + + if ($Restart -and $healthyOpenCodex) { + # The maintenance fence is the final reversible preflight before a + # real owner is stopped. Rejected foreign/mutex paths never alter it. + $script:LauncherFailurePhase = 'recovery-intent' + Set-RecoveryIntent -OpenCodexDirectory $OpenCodexHome -Mode "maintenance" -Until ([DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + 180000) + $maintenanceIntentWritten = $true + $script:LauncherFailurePhase = 'stop' + Write-Host "A healthy OpenCodex listener was confirmed on port $Port. Requesting a normal project CLI stop before visible restart..." + Invoke-ProjectCliStop -BunPath $bunPath -CliPath $cliPath -WorkingDirectory $ProjectRoot -OpenCodexDirectory $OpenCodexHome -CodexDirectory $CodexHome + Wait-ForOpenCodexToStop -ListenPort $Port + if (-not $script:OwnsVisibleProxyMutex) { + Acquire-VisibleProxyMutexAfterStop + } + } elseif (-not $healthyOpenCodex) { + if (Test-LoopbackPortOccupied -ListenPort $Port) { + throw "Port $Port is occupied but did not identify as OpenCodex; refusing to launch or select another port." + } + if (-not $script:OwnsVisibleProxyMutex) { + throw "Another visible launcher for this project and port is still active. It did not report a healthy OpenCodex listener, so this launcher will not race it." + } + } + + # A listener can appear after the health probe; reject that exact race rather than relying on + # CLI fallback behavior that could choose a different port. + if (Test-LoopbackPortOccupied -ListenPort $Port) { + throw "Port $Port became occupied before visible launch; no fallback port will be selected." + } + + if ($Restart -and -not $maintenanceIntentWritten) { + # Cold restart reaches here only after every refusal check; fence it + # immediately before the new visible child can be started. + $script:LauncherFailurePhase = 'recovery-intent' + Set-RecoveryIntent -OpenCodexDirectory $OpenCodexHome -Mode "maintenance" -Until ([DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + 180000) + } elseif (-not $Restart) { + $script:LauncherFailurePhase = 'recovery-intent' + Set-RecoveryIntent -OpenCodexDirectory $OpenCodexHome -Mode "running" + } + $onStarted = $null + if ($Restart) { + $onStarted = { + Set-RecoveryIntent -OpenCodexDirectory $OpenCodexHome -Mode "running" + Release-RestartMutex + } + } + $script:LauncherFailurePhase = 'proxy-start' + $exitCode = Invoke-VisibleProxy -BunPath $bunPath -CliPath $cliPath -WorkingDirectory $ProjectRoot -LogPath $logPath -ListenPort $Port -OpenCodexDirectory $OpenCodexHome -CodexDirectory $CodexHome -OnStarted $onStarted + if ($exitCode -ne 0) { + Release-VisibleProxyMutex + Release-RestartMutex + Wait-ForFailureAcknowledgement + $script:LauncherExitCode = $exitCode + exit $exitCode + } + + # Invoke-VisibleProxy already queued the terminal transcript record. Do not + # race a still-draining pump with a second PowerShell append/rotation here. + Write-VisibleConsole -Channel "launcher" -Message "OpenCodex exited normally; automatic restart is disabled." + Release-VisibleProxyMutex + Release-RestartMutex + Wait-ForFailureAcknowledgement + $script:LauncherExitCode = 0 + exit 0 +} catch { + $script:LauncherExitCode = 1 + if (-not [string]::IsNullOrWhiteSpace($OpenCodexHome)) { + $baseType = $_.Exception.GetBaseException().GetType().Name + if ($baseType -notmatch '^[A-Za-z0-9_.]{1,128}$') { $baseType = 'UnknownException' } + $line = $_.InvocationInfo.ScriptLineNumber + if ($line -lt 0 -or $line -gt 1000000) { $line = 0 } + $phase = $script:LauncherFailurePhase + if ($phase -notin @('initialization', 'path-validation', 'guardian', 'ownership', 'health', 'recovery-intent', 'stop', 'proxy-start')) { $phase = 'unknown' } + $hresult = [BitConverter]::ToUInt32([BitConverter]::GetBytes([int]$_.Exception.GetBaseException().HResult), 0) + $detail = 'phase={0};base={1};hresult=0x{2:X8};line={3}' -f $phase, $baseType, $hresult, $line + Write-LauncherLifecycleEvent -OpenCodexDirectory $OpenCodexHome -EventName 'launcher-failed' -ExitCode 1 -FailureKind ($_.Exception.GetType().Name) -Detail $detail + } + Write-VisibleConsole -Channel stderr -Message ("[OCX:ERROR] Visible launcher failed: {0}" -f $_.Exception.Message) + Release-VisibleProxyMutex + Release-RestartMutex + Wait-ForFailureAcknowledgement + exit 1 +} finally { + if (-not [string]::IsNullOrWhiteSpace($OpenCodexHome)) { + Write-LauncherLifecycleEvent -OpenCodexDirectory $OpenCodexHome -EventName 'launcher-exited' -ExitCode $script:LauncherExitCode + } + Release-VisibleProxyMutex + Release-RestartMutex + if ($null -ne $script:VisibleProxyMutex) { $script:VisibleProxyMutex.Dispose() } + if ($null -ne $script:RestartMutex) { $script:RestartMutex.Dispose() } +} diff --git a/src/cli/index.ts b/src/cli/index.ts index 88ab11392d7..d76c7208dd0 100755 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -140,6 +140,7 @@ import { selfLaunchArgv } from "../lib/self-launch-argv"; import { initializeNodeLauncherContext } from "./launcher-context"; import { createLocalAttestationSecret } from "../lib/local-management-attestation"; import { MEMORY_DRAIN_RESTART_MS, REPLACEMENT_READY_TIMEOUT_MS } from "../lib/system-restart-contract"; +import { writeRecoveryIntentIfGuardianEnabled } from "../lib/recovery-intent"; /** * A failed shell-hook reconcile is not cosmetic: a stale hook keeps sourcing @@ -816,7 +817,8 @@ async function handleEnsure(options: { existingIsSuccess?: boolean } = {}): Prom } /** Fixed tray action: start the proxy without depending on codexAutoStart. */ -async function handleTrayProxyStart(existingIsSuccess = true): Promise { +async function handleTrayProxyStart(existingIsSuccess = true, writeRunningIntent = true): Promise { + if (writeRunningIntent) await writeRecoveryIntentIfGuardianEnabled("running"); const ok = await runTrayProxyStart({ findLive: findLiveProxy, existingIsSuccess, @@ -911,7 +913,9 @@ async function handleProxyRestart( } async function handleTrayProxyRestart(): Promise { - await handleProxyRestart(() => handleTrayProxyStart(false)); + await writeRecoveryIntentIfGuardianEnabled("maintenance", { until: Date.now() + 180_000 }); + const restarted = await handleProxyRestart(() => handleTrayProxyStart(false, false)); + if (restarted) await writeRecoveryIntentIfGuardianEnabled("running"); } async function handleRestartStartWhenStopped(): Promise { @@ -991,6 +995,11 @@ async function restoreSharedClientStateAfterStop(): Promise<{ historyOnly: boole } async function handleStop() { + // A guardian-driven recovery child is carrying out a bounded automatic repair, + // not an operator's durable manual-stop instruction. + if (process.env.OPENCODEX_GUARDIAN_RECOVERY !== "1") { + await writeRecoveryIntentIfGuardianEnabled("stopped"); + } const lease = acquireOwnershipMutationLease(serviceStatePaths()); try { return await handleStopUnlocked(); } finally { lease.release(); } diff --git a/src/lib/recovery-intent.ts b/src/lib/recovery-intent.ts new file mode 100644 index 00000000000..1053004532d --- /dev/null +++ b/src/lib/recovery-intent.ts @@ -0,0 +1,103 @@ +import { lstatSync, readFileSync } from "node:fs"; +import { join, resolve } from "node:path"; + +import { atomicWriteFileAsync } from "../config/atomic-write"; +import { getConfigDir } from "../config/paths"; + +export type RecoveryIntentMode = "running" | "stopped" | "maintenance"; + +export interface RecoveryIntent { + version: 1; + mode: RecoveryIntentMode; + at: number; + until?: number; +} + +export interface WriteRecoveryIntentOptions { + home?: string; + at?: number; + until?: number; +} + +const MARKER_MAX_BYTES = 16 * 1024; +const INTENT_MAX_BYTES = 16 * 1024; + +function unsafeMarker(): never { + throw new Error("Recovery guardian marker is malformed; manual lifecycle action was not dispatched."); +} + +function isMissing(error: unknown): boolean { + return typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT"; +} + +function assertSafeExistingFile(path: string, maxBytes: number): ReturnType { + const stat = lstatSync(path); + if (!stat.isFile() || stat.isSymbolicLink() || stat.size > maxBytes) unsafeMarker(); + return stat; +} + +function assertSafeExistingDirectory(path: string): void { + const stat = lstatSync(path); + if (!stat.isDirectory() || stat.isSymbolicLink()) unsafeMarker(); +} + +/** + * Returns true only for a locally enabled v1 recovery guardian. A missing or + * explicitly disabled marker preserves ordinary installs exactly as before; + * every other marker state rejects the lifecycle action before it can mutate + * proxy state. + */ +export function recoveryGuardianEnabled(home: string = getConfigDir()): boolean { + const root = resolve(home); + const marker = join(root, "recovery-guardian.json"); + try { + assertSafeExistingDirectory(root); + assertSafeExistingFile(marker, MARKER_MAX_BYTES); + } catch (error) { + if (isMissing(error)) return false; + throw error; + } + let parsed: unknown; + try { + parsed = JSON.parse(readFileSync(marker, "utf8")); + } catch { + return unsafeMarker(); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return unsafeMarker(); + const markerValue = parsed as { version?: unknown; enabled?: unknown }; + if (markerValue.version === 1 && markerValue.enabled === false) return false; + if (markerValue.version !== 1 || markerValue.enabled !== true) return unsafeMarker(); + return true; +} + +/** Write a non-secret, bounded manual-recovery intent when the guardian opted in. */ +export async function writeRecoveryIntentIfGuardianEnabled( + mode: RecoveryIntentMode, + options: WriteRecoveryIntentOptions = {}, +): Promise { + const home = resolve(options.home ?? getConfigDir()); + if (!recoveryGuardianEnabled(home)) return false; + const at = options.at ?? Date.now(); + if (!Number.isSafeInteger(at) || at < 0) throw new Error("Recovery intent timestamp is invalid."); + if (mode !== "running" && mode !== "stopped" && mode !== "maintenance") { + throw new Error("Recovery intent mode is invalid."); + } + if (options.until !== undefined && (!Number.isSafeInteger(options.until) || options.until <= at)) { + throw new Error("Recovery intent maintenance deadline is invalid."); + } + if (mode !== "maintenance" && options.until !== undefined) { + throw new Error("Only a maintenance intent may carry a deadline."); + } + + const intentPath = join(home, "recovery-intent.json"); + try { + assertSafeExistingFile(intentPath, INTENT_MAX_BYTES); + } catch (error) { + if (!isMissing(error)) throw error; + } + const intent: RecoveryIntent = mode === "maintenance" + ? { version: 1, mode, at, until: options.until } + : { version: 1, mode, at }; + await atomicWriteFileAsync(intentPath, JSON.stringify(intent) + "\n"); + return true; +} diff --git a/src/lib/runtime-diagnostics-child.ts b/src/lib/runtime-diagnostics-child.ts new file mode 100644 index 00000000000..7fc8c31aba0 --- /dev/null +++ b/src/lib/runtime-diagnostics-child.ts @@ -0,0 +1,143 @@ +import { appendFileSync, existsSync, renameSync, statSync, unlinkSync } from "node:fs"; + +// Private child of the managed proxy, not another service or restart controller. +const MAX_LOG_BYTES = 2 * 1024 * 1024; +let path = ""; +let pid = 0; +let heartbeatAt = Date.now(); +let lastLogAt = 0; +let stalled = false; +let parentStoppingAt: number | null = null; +let stallMs = 5000; +let counters: Record = {}; +const operations = new Map(); +const completedSlowOperationSequences = new Set(); +const completedSlowOperationSequenceOrder: number[] = []; +let completedSlowOperationOverflow = 0; +let timer: ReturnType | undefined; + +function rememberCompletedSlowOperation(sequence: number): boolean { + if (completedSlowOperationSequences.has(sequence)) return false; + completedSlowOperationSequences.add(sequence); + completedSlowOperationSequenceOrder.push(sequence); + if (completedSlowOperationSequenceOrder.length > 16) { + const oldest = completedSlowOperationSequenceOrder.shift(); + if (oldest !== undefined) completedSlowOperationSequences.delete(oldest); + } + return true; +} + +function log(kind: string, extra: object = {}): void { + try { + if (existsSync(path) && statSync(path).size >= MAX_LOG_BYTES) { + if (existsSync(path + ".1")) unlinkSync(path + ".1"); + renameSync(path, path + ".1"); + } + appendFileSync(path, JSON.stringify({ + at: new Date().toISOString(), + pid, + recorderPid: process.pid, + kind, + ...extra, + }) + "\n"); + } catch { /* disk/logging failures never signal the managed service */ } +} + +process.on("message", (message: unknown) => { + const m = message as { kind: string; path: string; pid: number; intervalMs: number; + stallMs: number; logEveryMs: number; at: number; id: number; sites: string[]; + timerDelayMs: number; cpuUserDeltaMs: number; cpuSystemDeltaMs: number; + counters: typeof counters; droppedMessages: number; syncOperationsDropped: boolean; + completedSlowOperations?: Array<{ sequence: number; id: number; elapsedMs: number; sites: string[] }>; + completedSlowOperationOverflow?: number }; + if (m.kind === "init" && !path) { + path = m.path; + pid = m.pid; + stallMs = m.stallMs; + heartbeatAt = Date.now(); + log("start"); + let lastTickAt = performance.now(); + timer = setInterval(() => { + const tickAt = performance.now(); + const recorderDelayMs = Math.max(0, tickAt - lastTickAt - m.intervalMs); + lastTickAt = tickAt; + const now = Date.now(); + const gapMs = now - heartbeatAt; + const isStalled = gapMs >= m.stallMs; + if ((isStalled && !stalled) || now - lastLogAt >= m.logEveryMs) { + log(isStalled ? "event-loop-stall" : "sample", { + // Missing IPC alone is not proof that the parent's event loop stalled. + // The parent reports its own monotonic timer delay on the next heartbeat. + observation: isStalled ? "heartbeat-missing" : "heartbeat-current", + heartbeatGapMs: gapMs, recorderDelayMs, counters, operations: [...operations.values()], + }); + lastLogAt = now; + } + stalled = isStalled; + }, m.intervalMs); + process.send?.("ready"); + } else if (m.kind === "parent-stopping") { + parentStoppingAt = Number.isFinite(m.at) ? m.at : Date.now(); + log("parent-stopping"); + } else if (m.kind === "heartbeat") { + const timerDelayMs = Number.isFinite(m.timerDelayMs) ? m.timerDelayMs : null; + const parentDelayConfirmed = timerDelayMs !== null && timerDelayMs >= stallMs; + if (parentDelayConfirmed) log("event-loop-delay", { + intervalMs: m.intervalMs, timerDelayMs, + cpuUserDeltaMs: m.cpuUserDeltaMs, cpuSystemDeltaMs: m.cpuSystemDeltaMs, + counters: m.counters, operations: [...operations.values()], + }); + if (stalled) log("event-loop-recovered", { + gapMs: m.at - heartbeatAt, parentDelayConfirmed, timerDelayMs, + }); + heartbeatAt = m.at; + counters = m.counters; + stalled = false; + if (Array.isArray(m.completedSlowOperations)) { + for (const completed of m.completedSlowOperations) { + if (!Number.isSafeInteger(completed.sequence) || !Number.isSafeInteger(completed.id) + || !Number.isFinite(completed.elapsedMs) || completed.elapsedMs < 250 + || !Array.isArray(completed.sites) || !rememberCompletedSlowOperation(completed.sequence)) continue; + operations.delete(completed.id); + log("slow-sync-operation", { + elapsedMs: completed.elapsedMs, + sites: completed.sites.slice(0, 8), + }); + } + } + const overflow = m.completedSlowOperationOverflow; + if (typeof overflow === "number" && Number.isSafeInteger(overflow) + && overflow > completedSlowOperationOverflow) { + log("completed-slow-operation-overflow", { + droppedCompletedSlowOperations: overflow - completedSlowOperationOverflow, + completedSlowOperationOverflow: overflow, + }); + completedSlowOperationOverflow = overflow; + } + } else if (m.kind === "sync-start") { + if (operations.size >= 16) operations.delete(operations.keys().next().value!); + operations.set(m.id, { at: m.at, sites: m.sites }); + } else if (m.kind === "sync-end") { + const op = operations.get(m.id); + if (op && m.at - op.at >= 250) log("slow-sync-operation", { elapsedMs: m.at - op.at, sites: op.sites }); + operations.delete(m.id); + } else if (m.kind === "observability-gap") { + const clearedOperations = operations.size; + operations.clear(); + log("ipc-observability-gap", { + droppedMessages: Number.isSafeInteger(m.droppedMessages) && m.droppedMessages > 0 ? m.droppedMessages : 0, + syncOperationsDropped: m.syncOperationsDropped === true, + clearedOperations, + }); + } +}); +process.on("disconnect", () => { + if (timer) clearInterval(timer); + if (path) log("parent-disconnected", { + expected: parentStoppingAt !== null, + parentStoppingAt, + heartbeatGapMs: Date.now() - heartbeatAt, + counters, + }); + process.exit(0); +}); diff --git a/src/lib/runtime-diagnostics.ts b/src/lib/runtime-diagnostics.ts new file mode 100644 index 00000000000..630e717456b --- /dev/null +++ b/src/lib/runtime-diagnostics.ts @@ -0,0 +1,384 @@ +import { fork, type ChildProcess, type ForkOptions } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +// Opt-in, scalar-only diagnostics. A separate process keeps writing even when the +// server's event loop is blocked in a synchronous native call. No request data crosses IPC. +export type RuntimeCounters = Record; +let child: ChildProcess | null = null; +type DiagnosticDeliveryCallback = () => void; +type RuntimeDiagnosticSender = (message: object, onDelivered?: DiagnosticDeliveryCallback) => boolean; +let sendToChild: RuntimeDiagnosticSender | null = null; +let sequence = 0; + +const MAX_DIAGNOSTIC_IPC_IN_FLIGHT = 4; +const STOP_DELIVERY_TIMEOUT_MS = 250; +const STATIC_SAMPLE_MIN_INTERVAL_MS = 250; +const SLOW_SYNC_OPERATION_MS = 250; +const COMPLETED_SLOW_OPERATION_CAPACITY = 8; + +type CompletedSlowOperation = { + readonly sequence: number; + readonly id: number; + readonly elapsedMs: number; + readonly sites: readonly string[]; +}; + +type CompletedSlowOperationRing = { + enqueue(operation: CompletedSlowOperation): void; + snapshot(): readonly CompletedSlowOperation[]; + acknowledge(delivered: readonly CompletedSlowOperation[]): void; + readonly overflow: number; +}; + +/** + * A recorder may be briefly IPC-backpressured exactly while the parent event loop + * recovers. Retain completed slow operations until a later heartbeat callback + * confirms delivery, rather than treating `send() === false` as a lost message. + */ +export function createCompletedSlowOperationRingForTests( + capacity = COMPLETED_SLOW_OPERATION_CAPACITY, +): CompletedSlowOperationRing { + const boundedCapacity = Number.isFinite(capacity) + ? Math.max(1, Math.floor(capacity)) + : COMPLETED_SLOW_OPERATION_CAPACITY; + const operations: CompletedSlowOperation[] = []; + let overflow = 0; + return { + enqueue(operation) { + if (operations.length >= boundedCapacity) { + operations.shift(); + overflow += 1; + } + operations.push(operation); + }, + snapshot: () => [...operations], + acknowledge(delivered) { + const deliveredSequences = new Set(delivered.map(operation => operation.sequence)); + for (let index = operations.length - 1; index >= 0; index -= 1) { + if (deliveredSequences.has(operations[index]!.sequence)) operations.splice(index, 1); + } + }, + get overflow() { return overflow; }, + }; +} + +type DiagnosticIpcTarget = { + readonly connected: boolean; + send(message: object, callback: (error: Error | null) => void): boolean; +}; + +type RuntimeDiagnosticStopTarget = DiagnosticIpcTarget & { + disconnect(): void; +}; + +/** + * `ChildProcess.send()` returning false reports IPC backpressure, not a failed + * enqueue. Preserve a queued shutdown marker until its callback, while still + * bounding shutdown if a broken channel never invokes that callback. + */ +export function disconnectAfterRuntimeDiagnosticsStop( + target: RuntimeDiagnosticStopTarget, + at = Date.now(), + timeoutMs = STOP_DELIVERY_TIMEOUT_MS, +): void { + let disconnected = false; + const disconnect = () => { + if (disconnected || !target.connected) return; + disconnected = true; + target.disconnect(); + }; + const fallback = setTimeout(disconnect, timeoutMs); + fallback.unref(); + try { + target.send({ kind: "parent-stopping", at }, () => { + clearTimeout(fallback); + disconnect(); + }); + } catch { + clearTimeout(fallback); + disconnect(); + } +} + +/** + * Diagnostics must never let a blocked recorder turn into retained parent memory. + * A later gap notification tells the child to discard any unmatched sync operation + * rather than attributing an arbitrary duration to it. + */ +export function createBoundedRuntimeDiagnosticSender(target: DiagnosticIpcTarget): RuntimeDiagnosticSender { + let inFlight = 0; + let backpressured = false; + let droppedMessages = 0; + let droppedSyncOperations = false; + + const recordDrop = (message: object): false => { + droppedMessages += 1; + const kind = (message as { kind?: unknown }).kind; + if (kind === "sync-start" || kind === "sync-end") droppedSyncOperations = true; + return false; + }; + const sendOne = (message: object, onDelivered?: DiagnosticDeliveryCallback): boolean => { + if (!target.connected || backpressured || inFlight >= MAX_DIAGNOSTIC_IPC_IN_FLIGHT) return recordDrop(message); + inFlight += 1; + try { + const accepted = target.send(message, error => { + inFlight = Math.max(0, inFlight - 1); + // A callback error means the recorder missed this message. Preserve that + // fact until a later delivered gap clears any partial operation state. + if (error) recordDrop(message); + else onDelivered?.(); + if (inFlight === 0) backpressured = false; + }); + if (!accepted) backpressured = true; + return true; + } catch { + inFlight = Math.max(0, inFlight - 1); + return recordDrop(message); + } + }; + + return (message, onDelivered) => { + if (droppedMessages > 0) { + const gap = { + kind: "observability-gap", + at: Date.now(), + droppedMessages, + syncOperationsDropped: droppedSyncOperations, + }; + if (!sendOne(gap)) return recordDrop(message); + droppedMessages = 0; + droppedSyncOperations = false; + // The gap clears the recorder's incomplete operations before this ordinary + // message is processed; the normal in-flight bound still applies below. + return sendOne(message, onDelivered); + } + return sendOne(message, onDelivered); + }; +} + +let completedSlowOperationSink: ((operation: CompletedSlowOperation) => void) | null = null; + +export function beginRuntimeSyncOperation(): () => void { + const send = sendToChild; + if (!child?.connected || !send) return () => {}; + const id = ++sequence; + const startedAt = performance.now(); + // Keep code locations, never absolute user paths, arguments, or error messages. + const sites = (new Error().stack ?? "").split("\n").flatMap(line => { + const match = /[\\/](src[\\/][\w./\\-]+:\d+:\d+)/.exec(line); + return match ? [match[1]!.replaceAll("\\", "/")] : []; + }).slice(1, 9); + const started = send({ kind: "sync-start", at: Date.now(), id, sites }); + const complete = completedSlowOperationSink; + return () => { + const elapsedMs = Math.max(0, performance.now() - startedAt); + if (elapsedMs >= SLOW_SYNC_OPERATION_MS && complete) { + complete({ sequence: ++sequence, id, elapsedMs, sites }); + } else if (started) { + send({ kind: "sync-end", at: Date.now(), id }); + } + }; +} + +export function startRuntimeDiagnostics( + path: string, + sample: () => RuntimeCounters, + timing = { intervalMs: 1000, stallMs: 5000, logEveryMs: 30_000 }, +): { ready: Promise; closed: Promise; stop(): void } { + const launchOptions: ForkOptions & { windowsHide: boolean } = { + execPath: process.execPath, + execArgv: [], + stdio: ["ignore", "ignore", "ignore", "ipc"], + windowsHide: true, + // A Windows detached recorder can observe the ordinary parent-exit IPC + // boundary. It still cannot escape a launcher Job Object or taskkill /T. + detached: process.platform === "win32", + }; + const target = fork(fileURLToPath(new URL("./runtime-diagnostics-child.ts", import.meta.url)), [], launchOptions); + child = target; + const send = createBoundedRuntimeDiagnosticSender(target); + sendToChild = send; + const completedSlowOperations = createCompletedSlowOperationRingForTests(); + const enqueueCompletedSlowOperation = (operation: CompletedSlowOperation) => { + completedSlowOperations.enqueue(operation); + }; + completedSlowOperationSink = enqueueCompletedSlowOperation; + let lastSampleAt = performance.now(); + let lastCpu = process.cpuUsage(); + let sampleFailures = 0; + let memoryUsageFailures = 0; + let samplerActive = false; + let sampling = false; + const beginStaticSampleOperation = (phase: string): (() => void) => { + // Keep the sampler's own operation static and scalar-only: collecting a + // stack here would add work exactly where a native runtime call may stall. + // Before recorder readiness, avoid queuing a startup operation that cannot + // describe a real sample. + // A begin/end pair adds two bounded IPC messages. Keep the normal 1s + // diagnostic cadence attributable without turning deliberately rapid test + // or debug cadences into their own recorder backpressure source. + if (!samplerActive || timing.intervalMs < STATIC_SAMPLE_MIN_INTERVAL_MS || !target.connected) return () => {}; + const id = ++sequence; + const startedAt = performance.now(); + const started = send({ kind: "sync-start", at: Date.now(), id, sites: [phase] }); + return () => { + const elapsedMs = Math.max(0, performance.now() - startedAt); + if (elapsedMs >= SLOW_SYNC_OPERATION_MS) { + enqueueCompletedSlowOperation({ sequence: ++sequence, id, elapsedMs, sites: [phase] }); + } else if (started) { + send({ kind: "sync-end", at: Date.now(), id }); + } + }; + }; + const sampleCounters = (cpu: NodeJS.CpuUsage): RuntimeCounters => { + let usage: ReturnType | null = null; + const finishMemoryUsage = beginStaticSampleOperation("src/lib/runtime-diagnostics.ts:process.memoryUsage"); + try { + usage = process.memoryUsage(); + } catch { + // Native-call failures stay distinguishable from the supplied business + // counters below, without retaining an error object or its message. + memoryUsageFailures += 1; + } finally { + finishMemoryUsage(); + } + let sampled: RuntimeCounters = {}; + try { sampled = sample(); } catch { sampleFailures += 1; } + return { + ...sampled, + uptime: process.uptime(), + rss: usage?.rss ?? null, + heapUsed: usage?.heapUsed ?? null, + external: usage?.external ?? null, + cpuUserMs: cpu.user / 1000, + cpuSystemMs: cpu.system / 1000, + diagnosticsSampleFailures: sampleFailures, + diagnosticsMemoryUsageFailures: memoryUsageFailures, + }; + }; + const sendHeartbeat = (heartbeat: object): void => { + const completed = completedSlowOperations.snapshot(); + send({ + ...heartbeat, + completedSlowOperations: completed, + completedSlowOperationOverflow: completedSlowOperations.overflow, + }, () => { + completedSlowOperations.acknowledge(completed); + }); + }; + let stopRequested = false; + let readySettled = false; + let resolveReady!: () => void; + let rejectReady!: () => void; + const ready = new Promise((resolve, reject) => { + resolveReady = () => { + if (readySettled) return; + readySettled = true; + resolve(); + }; + rejectReady = () => { + if (readySettled) return; + readySettled = true; + reject(new Error("runtime diagnostics recorder closed before ready")); + }; + }); + // Callers can still observe rejection; this prevents an optional recorder + // startup failure from becoming an unhandled-rejection process failure. + void ready.catch(() => {}); + let timer: ReturnType | undefined; + let closedSettled = false; + let resolveClosed!: () => void; + const closed = new Promise(resolve => { resolveClosed = resolve; }); + const settleClosed = () => { + if (closedSettled) return; + closedSettled = true; + if (timer) clearInterval(timer); + if (child === target) child = null; + if (sendToChild === send) sendToChild = null; + if (completedSlowOperationSink === enqueueCompletedSlowOperation) completedSlowOperationSink = null; + rejectReady(); + resolveClosed(); + }; + target.once("close", settleClosed); + target.once("exit", settleClosed); + const logRecorderEvent = (kind: "recorder-error" | "recorder-exit", extra: object = {}) => { + if (!stopRequested) console.warn("[runtime-diagnostics] recorder-event", JSON.stringify({ + kind, + recorderPid: target.pid ?? null, + ...extra, + })); + }; + target.on("error", () => { + logRecorderEvent("recorder-error"); + rejectReady(); + const failureCloseTimer = setTimeout(() => { + if (target.exitCode === null) target.kill(); + settleClosed(); + }, 2000); + failureCloseTimer.unref(); + void closed.then(() => clearTimeout(failureCloseTimer)); + }); + target.once("exit", (exitCode, signalCode) => logRecorderEvent("recorder-exit", { + exitCode: exitCode ?? null, + signalCode: signalCode ?? null, + })); + target.on("message", message => { + if (message !== "ready") return; + // The first heartbeat begins from readiness, not recorder process launch. + // This keeps startup IPC/bootstrap time out of the timer-delay baseline. + lastSampleAt = performance.now(); + lastCpu = process.cpuUsage(); + samplerActive = true; + resolveReady(); + }); + send({ kind: "init", path, pid: process.pid, ...timing }); + timer = setInterval(() => { + // setInterval callbacks normally serialize on one event loop, but retain a + // guard so a future re-entrant sample seam cannot nest sync operations. + if (!samplerActive || sampling) return; + sampling = true; + try { + const now = performance.now(); + const intervalMs = now - lastSampleAt; + const cpu = process.cpuUsage(); + const cpuUserDeltaMs = (cpu.user - lastCpu.user) / 1000; + const cpuSystemDeltaMs = (cpu.system - lastCpu.system) / 1000; + const counters = sampleCounters(cpu); + // Advance the baseline regardless of a business-counter failure. Otherwise a + // failed sample manufactures a parent timer delay on the next good sample. + lastSampleAt = now; + lastCpu = cpu; + sendHeartbeat({ kind: "heartbeat", at: Date.now(), + intervalMs, timerDelayMs: Math.max(0, intervalMs - timing.intervalMs), + cpuUserDeltaMs, + cpuSystemDeltaMs, + counters, + }); + } finally { + sampling = false; + } + }, timing.intervalMs); + timer.unref(); + target.unref(); + target.channel?.unref?.(); + return { ready, closed, stop() { + stopRequested = true; + clearInterval(timer); + if (child === target) child = null; + if (sendToChild === send) sendToChild = null; + if (target.connected) { + // Do not extend shutdown, but make one final bounded attempt to carry a + // completed slow operation that ended between ordinary heartbeats. + if (completedSlowOperations.snapshot().length > 0) { + sendHeartbeat({ kind: "heartbeat", at: Date.now(), intervalMs: timing.intervalMs, + timerDelayMs: 0, cpuUserDeltaMs: 0, cpuSystemDeltaMs: 0, counters: {} }); + } + // A requested stop is a lifecycle fact, not a recurring diagnostic. Let + // the recorder receive it before its IPC channel closes where possible. + disconnectAfterRuntimeDiagnosticsStop(target); + } + const killTimer = setTimeout(() => { if (target.exitCode === null) target.kill(); }, 2000); + killTimer.unref(); + void closed.then(() => clearTimeout(killTimer)); + } }; +} diff --git a/src/server/background-lifecycle.ts b/src/server/background-lifecycle.ts index a2cf4da136e..5125e3d1885 100644 --- a/src/server/background-lifecycle.ts +++ b/src/server/background-lifecycle.ts @@ -1,4 +1,9 @@ import type { StorageCleanupPolicy } from "../types"; +import { join } from "node:path"; +import { getConfigDir } from "../config/paths"; +import { startRuntimeDiagnostics } from "../lib/runtime-diagnostics"; +import { getActiveTurnCount } from "./lifecycle"; +import { responseStateMetrics } from "../responses/state"; import { startStateStoreSweeper } from "../lib/state-store-sweeper"; import { abortStorageCleanupPolicyJobAsync, @@ -32,6 +37,7 @@ import { type PolicyApply = (policy: StorageCleanupPolicy) => void; type ProcessLoops = { + diagnostics: ReturnType | null; memoryWatchdog: MemoryWatchdog; stateStoreSweeper: ReturnType; }; @@ -58,10 +64,22 @@ function setLivePolicyOwner(applyPolicy: PolicyApply | null): void { } function startProcessLoops(applyPolicy: PolicyApply): ProcessLoops { + let diagnostics: ReturnType | null = null; let memoryWatchdog: MemoryWatchdog | null = null; let stateStoreSweeper: ReturnType | null = null; try { memoryWatchdog = startMemoryWatchdog(); + if (process.env.OPENCODEX_RUNTIME_DIAGNOSTICS === "1") { + try { + diagnostics = startRuntimeDiagnostics(join(getConfigDir(), "runtime-diagnostics.jsonl"), () => { + const turns = getActiveTurnCount(); + const state = turns > 0 ? responseStateMetrics() : null; + return { activeTurns: turns, responseBytes: state?.totalBytes ?? null, + spillWrites: state?.spillWrites ?? null, spillFailures: state?.spillWriteFailures ?? null, + spillTimeoutRefusals: state?.spillAclTimeoutMemoRefusals ?? null }; + }); + } catch { console.warn("[runtime-diagnostics] recorder could not start"); } + } stateStoreSweeper = startStateStoreSweeper(); setLivePolicyOwner(applyPolicy); startStorageCleanupScheduler(); @@ -95,8 +113,9 @@ function startProcessLoops(applyPolicy: PolicyApply): ProcessLoops { .catch(() => { // The next poll tick retries. }); - return { memoryWatchdog, stateStoreSweeper }; + return { memoryWatchdog, stateStoreSweeper, diagnostics }; } catch (error) { + diagnostics?.stop(); memoryWatchdog?.stop(); stateStoreSweeper?.stop(); stopStorageCleanupScheduler(); @@ -110,6 +129,7 @@ function startProcessLoops(applyPolicy: PolicyApply): ProcessLoops { function stopProcessLoops(): void { const loops = processLoops; processLoops = null; + loops?.diagnostics?.stop(); loops?.memoryWatchdog.stop(); loops?.stateStoreSweeper.stop(); stopStorageCleanupScheduler(); diff --git a/src/server/management-api.ts b/src/server/management-api.ts index 63733717d9b..1e5018d6842 100644 --- a/src/server/management-api.ts +++ b/src/server/management-api.ts @@ -85,6 +85,7 @@ import type { CatalogDisposition, ConvergeCodex } from "../codex/convergence-typ import { normalizeCatalogDisposition } from "../codex/catalog-refresh-status"; import { managementBodyTooLargeResponse } from "./management/body"; import { handleSessionRoutes } from "./management/session-routes"; +import { writeRecoveryIntentIfGuardianEnabled } from "../lib/recovery-intent"; import { packageVersion } from "../lib/package-version"; // installed npm version instead of a stale hardcode. @@ -328,6 +329,17 @@ export async function handleManagementAPI( const { deferralMatchesReceipt } = await import("../config/pending-teardown"); const { deferralHonored, performStopTeardown } = await import("./stop-teardown"); const holdsReceipt = deferralHonored(url, deferralMatchesReceipt); + if (!holdsReceipt) { + try { + await writeRecoveryIntentIfGuardianEnabled("stopped"); + } catch { + return jsonResponse({ + success: false, + code: "recovery_intent_unavailable", + message: "The enabled recovery guardian marker could not be validated, so the stop was not dispatched.", + }, 503, req, config); + } + } const respawnRisk = holdsReceipt ? "none" : installedServiceRespawnRisk(); if (respawnRisk === "respawnable") { return jsonResponse({ diff --git a/src/tray/windows-tray.ps1 b/src/tray/windows-tray.ps1 index e2ee8727c7c..bdaaa661987 100644 --- a/src/tray/windows-tray.ps1 +++ b/src/tray/windows-tray.ps1 @@ -310,6 +310,43 @@ function Complete-PendingAction([bool]$Success) { } } +function Test-RecoveryGuardianIntentEnabled([string]$OpenCodexHome) { + # A missing marker means this home predates the guardian, or the guardian was + # removed with it: the legacy tray actions stand on their own. Any marker that + # exists is load-bearing, so an unreadable one must fail closed and let the + # helper supply its own bounded error rather than dispatching a silent action. + $markerPath = Join-Path $OpenCodexHome 'recovery-guardian.json' + if (-not (Test-Path -LiteralPath $markerPath -PathType Leaf)) { return $false } + try { + $markerInfo = Get-Item -LiteralPath $markerPath -Force -ErrorAction Stop + if ((([int]$markerInfo.Attributes -band [int][System.IO.FileAttributes]::ReparsePoint) -ne 0) -or $markerInfo.Length -gt 16KB) { return $true } + $marker = [System.IO.File]::ReadAllText($markerInfo.FullName, [System.Text.Encoding]::UTF8) | ConvertFrom-Json -ErrorAction Stop + # Only a complete explicit disabled marker preserves legacy tray actions + # without requiring the new installed helper. Every other marker state is + # fail-closed and lets the helper supply its bounded error. + return -not ($marker.version -eq 1 -and $marker.enabled -is [bool] -and -not $marker.enabled) + } catch { return $true } +} + +function Set-RecoveryIntent( + [Parameter(Mandatory)][string]$OpenCodexHome, + [Parameter(Mandatory)][ValidateSet('running', 'stopped', 'maintenance')][string]$Mode, + [long]$Until = 0 +) { + # The installed tray must write before dispatching its child: an older running + # proxy cannot be trusted to persist a manual Stop on the tray's behalf. + $helper = Join-Path $PSScriptRoot 'opencodex-recovery-intent.ps1' + if (-not (Test-Path -LiteralPath $helper -PathType Leaf)) { + if (-not (Test-RecoveryGuardianIntentEnabled $OpenCodexHome)) { return } + throw 'Recovery intent helper is missing; lifecycle action was not dispatched.' + } + # Array splatting is positional in Windows PowerShell. Keep the helper's + # named lifecycle arguments in a hashtable so a home path never binds -Mode. + $intentArgs = @{ OpenCodexHome = $OpenCodexHome; Mode = $Mode } + if ($Mode -eq 'maintenance') { $intentArgs.Until = $Until } + & $helper @intentArgs +} + function Update-TrayState { $target = Read-ListenTarget $script:port = [int]$target.port @@ -428,6 +465,7 @@ function Update-TrayState { $openItem.add_Click({ Start-OcxCommand @("gui") }) $startItem.add_Click({ if (-not (Set-PendingAction "Start Proxy" 75)) { return } + Set-RecoveryIntent -OpenCodexHome $OpenCodexHome -Mode "running" $statusItem.Text = "Proxy: Starting..." # service start can spend 20s and the CLI then observes health for another 40s. $startProcess = Start-OcxCommand @("__tray-start") -TrackExit @@ -439,6 +477,7 @@ $startItem.add_Click({ }) $stopItem.add_Click({ if (-not (Set-PendingAction "Stop Proxy" 15)) { return } + Set-RecoveryIntent -OpenCodexHome $OpenCodexHome -Mode "stopped" $statusItem.Text = "Proxy: Stopping..." $stopProcess = Start-OcxCommand @("stop") -TrackExit if ($stopProcess -is [System.Diagnostics.Process]) { @@ -449,6 +488,7 @@ $stopItem.add_Click({ }) $restartItem.add_Click({ if (-not (Set-PendingAction "Restart Proxy" 160)) { return } + Set-RecoveryIntent -OpenCodexHome $OpenCodexHome -Mode "maintenance" -Until ([DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + 180000) $statusItem.Text = "Proxy: Restarting..." # /api/system/restart may drain active work for 60s and then spend up to 70s # handing off to an identity-verified replacement. The tray observes health/PID diff --git a/src/tray/windows.ts b/src/tray/windows.ts index c20e0491d17..565bbf95680 100644 --- a/src/tray/windows.ts +++ b/src/tray/windows.ts @@ -20,6 +20,9 @@ const TRAY_ICON_FILES = [ "opencodex-tray-warning.ico", "opencodex-tray-offline.ico", ] as const; +// The guardian intent helper keeps its scripts/ namespace in the repository and +// installs into the private home alongside the tray script it is called from. +const INSTALLED_TRAY_RECOVERY_INTENT_FILE = "opencodex-recovery-intent.ps1"; export interface WindowsTrayEntry { bun: string; @@ -68,6 +71,10 @@ function installedTrayIconPaths(): string[] { return TRAY_ICON_FILES.map(name => join(getConfigDir(), name)); } +function installedTrayRecoveryIntentPath(): string { + return join(getConfigDir(), INSTALLED_TRAY_RECOVERY_INTENT_FILE); +} + export function windowsTrayStatePathsOwned( state: Pick & { launcherPath?: string }, configDir = getConfigDir(), @@ -86,6 +93,10 @@ function sourceTrayIconPaths(): string[] { return TRAY_ICON_FILES.map(name => join(import.meta.dir, "assets", name)); } +function sourceTrayRecoveryIntentPath(): string { + return join(import.meta.dir, "..", "..", "scripts", "ocx-recovery-guardian", "intent.ps1"); +} + function currentCodexHome(): string { const raw = process.env.CODEX_HOME?.trim(); return raw ? resolve(expandUserPath(raw)) : join(homedir(), ".codex"); @@ -469,7 +480,7 @@ function trayStatusFrom(registered: string | null): WindowsTrayStatus { const running = heartbeatProcessAlive(heartbeat); const registrationOwned = state !== null && registered === state.runCommand - && [state.bun, state.cli, state.script, ...(state.launcherPath ? [state.launcherPath] : []), ...installedTrayIconPaths()] + && [state.bun, state.cli, state.script, ...(state.launcherPath ? [state.launcherPath] : []), ...installedTrayIconPaths(), ...(existsSync(sourceTrayRecoveryIntentPath()) ? [installedTrayRecoveryIntentPath()] : [])] .every(path => existsSync(path)); const stale = windowsTrayRegistrationIsStale({ registered: registered !== null, @@ -617,6 +628,8 @@ export function installWindowsTray(startNow = true): WindowsTrayStatus { assertWindows(); const entry = currentEntry(); const sourceScript = sourceTrayScriptPath(); + const sourceRecoveryIntent = sourceTrayRecoveryIntentPath(); + const installedRecoveryIntent = installedTrayRecoveryIntentPath(); const iconPairs = sourceTrayIconPaths().map((source, index) => ({ source, installed: installedTrayIconPaths()[index] })); for (const path of [entry.bun, entry.cli, sourceScript, ...iconPairs.map(pair => pair.source)]) { if (!existsSync(path)) throw new Error(`Cannot install the tray because a required file is missing: ${path}`); @@ -641,8 +654,8 @@ export function installWindowsTray(startNow = true): WindowsTrayStatus { if (existsSync(launcherPath) && (!state?.launcherPath || resolve(state.launcherPath) !== resolve(launcherPath))) { throw new Error(`Refusing to overwrite an unowned tray launcher at ${launcherPath}.`); } - if (!state && iconPairs.some(pair => existsSync(pair.installed))) { - throw new Error("Refusing to overwrite unowned Windows tray icon assets."); + if (!state && [...iconPairs.map(pair => pair.installed), installedRecoveryIntent].some(path => existsSync(path))) { + throw new Error("Refusing to overwrite unowned Windows tray package assets."); } const wasRunning = heartbeatProcessAlive(); if (wasRunning && !state) { @@ -655,6 +668,7 @@ export function installWindowsTray(startNow = true): WindowsTrayStatus { const previousStateBytes = existsSync(trayStatePath()) ? readFileSync(trayStatePath()) : null; const previousScriptBytes = existsSync(entry.script) ? readFileSync(entry.script) : null; + const previousRecoveryIntentBytes = existsSync(installedRecoveryIntent) ? readFileSync(installedRecoveryIntent) : null; const previousLauncherBytes = existsSync(launcherPath) ? readFileSync(launcherPath) : null; const previousIconBytes = new Map(iconPairs.map(pair => [ pair.installed, @@ -665,6 +679,10 @@ export function installWindowsTray(startNow = true): WindowsTrayStatus { if (previousScriptBytes) replaceWindowsTrayOwnedFile(entry.script, previousScriptBytes); else if (existsSync(entry.script)) unlinkSync(entry.script); } catch { /* rollback best-effort */ } + try { + if (previousRecoveryIntentBytes) replaceWindowsTrayOwnedFile(installedRecoveryIntent, previousRecoveryIntentBytes); + else if (existsSync(installedRecoveryIntent)) unlinkSync(installedRecoveryIntent); + } catch { /* rollback best-effort */ } try { if (previousLauncherBytes) replaceWindowsTrayOwnedFile(launcherPath, previousLauncherBytes); else if (existsSync(launcherPath)) unlinkSync(launcherPath); @@ -695,6 +713,12 @@ export function installWindowsTray(startNow = true): WindowsTrayStatus { const hardenedDir = hardenSecretDir(getConfigDir(), { required: true }); if (!hardenedDir.ok) throw new Error("Windows tray directory ACL hardening did not complete; refusing to install persistence."); replaceWindowsTrayOwnedFile(entry.script, readFileSync(sourceScript)); + // `scripts/` is outside the npm package allowlist, so a published install has no + // guardian to carry and gets no helper. The tray script keeps its own fail-closed + // check for a home whose marker exists without the helper beside it. + if (existsSync(sourceRecoveryIntent)) { + replaceWindowsTrayOwnedFile(installedRecoveryIntent, readFileSync(sourceRecoveryIntent)); + } for (const pair of iconPairs) replaceWindowsTrayOwnedFile(pair.installed, readFileSync(pair.source)); replaceWindowsTrayOwnedFile(launcherPath, Buffer.from("\uFEFF" + buildWindowsTrayLauncherScript(entry), "utf16le")); runRegistry(["add", RUN_KEY, "/v", runValue, "/t", "REG_SZ", "/d", runCommand, "/f", "/reg:64"]); @@ -747,7 +771,7 @@ export function uninstallWindowsTray(): WindowsTrayStatus { if (existing) runRegistry(["delete", RUN_KEY, "/v", state?.runValue ?? windowsTrayRunValue(getConfigDir()), "/f", "/reg:64"]); const ownedPaths = [trayStatePath(), trayHeartbeatPath(), ...(state?.launcherPath ? [state.launcherPath] : [])]; if (state?.script && resolve(state.script) === resolve(installedTrayScriptPath())) ownedPaths.push(state.script); - if (state) ownedPaths.push(...installedTrayIconPaths()); + if (state) ownedPaths.push(...installedTrayIconPaths(), installedTrayRecoveryIntentPath()); for (const path of ownedPaths) { try { if (existsSync(path)) unlinkSync(path); } catch { /* best-effort */ } } diff --git a/tests/server/server-runtime-diagnostics.test.ts b/tests/server/server-runtime-diagnostics.test.ts new file mode 100644 index 00000000000..2eec935c956 --- /dev/null +++ b/tests/server/server-runtime-diagnostics.test.ts @@ -0,0 +1,431 @@ +import { expect, test } from "bun:test"; +import { existsSync, mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fork } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { + beginRuntimeSyncOperation, + createBoundedRuntimeDiagnosticSender, + createCompletedSlowOperationRingForTests, + disconnectAfterRuntimeDiagnosticsStop, + startRuntimeDiagnostics, +} from "../../src/lib/runtime-diagnostics"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +test("independent recorder observes a blocked parent and stops with its owner", async () => { + const dir = mkdtempSync(join(tmpdir(), "ocx-runtime-diagnostics-")); + const path = join(dir, "runtime.jsonl"); + const recorder = startRuntimeDiagnostics(path, () => ({ activeTurns: 12 }), { + intervalMs: 20, stallMs: 100, logEveryMs: 1000, + }); + try { + await Promise.race([recorder.ready, Bun.sleep(3000).then(() => { throw new Error("recorder did not start"); })]); + await Bun.sleep(100); + const end = beginRuntimeSyncOperation(); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 400); + end(); + await Bun.sleep(100); + } finally { + recorder.stop(); + await recorder.closed; + } + try { + const content = readFileSync(path, "utf8"); + const records = content.trim().split("\n").map(line => JSON.parse(line)); + expect(records.some(r => r.kind === "event-loop-stall" && r.heartbeatGapMs >= 100)).toBe(true); + expect(records.some(r => r.kind === "slow-sync-operation" && r.elapsedMs >= 350)).toBe(true); + expect(records.some(r => r.kind === "event-loop-recovered")).toBe(true); + const delay = records.find(r => r.kind === "event-loop-delay"); + expect(delay.timerDelayMs).toBeGreaterThanOrEqual(300); + expect(delay.cpuUserDeltaMs).toBeGreaterThanOrEqual(0); + expect(delay.cpuSystemDeltaMs).toBeGreaterThanOrEqual(0); + expect(records.at(-1).kind).toBe("parent-disconnected"); + expect(content).not.toContain(dir); + } finally { removeTreeWithRetry(dir); } +}); + +test("late IPC without a delayed parent timer is not confirmed as a parent stall", async () => { + const dir = mkdtempSync(join(tmpdir(), "ocx-runtime-diagnostics-ipc-")); + const path = join(dir, "runtime.jsonl"); + const child = fork(fileURLToPath(new URL("../../src/lib/runtime-diagnostics-child.ts", import.meta.url)), [], { + execPath: process.execPath, execArgv: [], stdio: ["ignore", "ignore", "ignore", "ipc"], windowsHide: true, + }); + const closed = new Promise(resolve => child.once("close", () => resolve())); + const ready = new Promise(resolve => child.once("message", () => resolve())); + try { + child.send({ kind: "init", path, pid: process.pid, intervalMs: 20, stallMs: 100, logEveryMs: 1000 }); + await Promise.race([ready, Bun.sleep(3000).then(() => { throw new Error("recorder did not start"); })]); + await Bun.sleep(250); + child.send({ kind: "heartbeat", at: Date.now(), intervalMs: 20, timerDelayMs: 0, + cpuUserDeltaMs: 1, cpuSystemDeltaMs: 0, counters: { activeTurns: 0 } }); + await Bun.sleep(50); + } finally { + if (child.connected) child.disconnect(); + await closed; + } + try { + const records = readFileSync(path, "utf8").trim().split("\n").map(line => JSON.parse(line)); + expect(records.some(r => r.kind === "event-loop-stall" && r.observation === "heartbeat-missing")).toBe(true); + expect(records.some(r => r.kind === "event-loop-delay")).toBe(false); + expect(records.some(r => r.kind === "event-loop-recovered" && r.parentDelayConfirmed === false)).toBe(true); + expect(records.at(-1)).toEqual(expect.objectContaining({ kind: "parent-disconnected", expected: false })); + } finally { removeTreeWithRetry(dir); } +}); + +test("bounded sender drops congested diagnostics and reports a sync observability gap", () => { + const messages: object[] = []; + const callbacks: Array<(error: Error | null) => void> = []; + const sender = createBoundedRuntimeDiagnosticSender({ + connected: true, + send(message, callback) { + messages.push(message); + callbacks.push(callback); + return true; + }, + }); + + for (let index = 0; index < 4; index += 1) expect(sender({ kind: "heartbeat", index })).toBe(true); + expect(sender({ kind: "sync-start", id: 7 })).toBe(false); + expect(messages).toHaveLength(4); + for (const callback of callbacks.splice(0)) callback(null); + + expect(sender({ kind: "heartbeat" })).toBe(true); + expect(messages).toHaveLength(6); + expect(messages[4]).toEqual(expect.objectContaining({ + kind: "observability-gap", + droppedMessages: 1, + syncOperationsDropped: true, + })); +}); + +test("bounded sender reports a failed sync callback as an observability gap", () => { + const messages: object[] = []; + let callback: ((error: Error | null) => void) | undefined; + const sender = createBoundedRuntimeDiagnosticSender({ + connected: true, + send(message, done) { + messages.push(message); + callback = done; + return true; + }, + }); + + expect(sender({ kind: "sync-end", id: 7 })).toBe(true); + callback?.(new Error("recorder disconnected")); + expect(sender({ kind: "heartbeat" })).toBe(true); + expect(messages[1]).toEqual(expect.objectContaining({ + kind: "observability-gap", + droppedMessages: 1, + syncOperationsDropped: true, + })); +}); + +test("completed slow-operation evidence stays bounded until a callback acknowledges its heartbeat", () => { + const ring = createCompletedSlowOperationRingForTests(); + for (let sequence = 1; sequence <= 10; sequence += 1) { + ring.enqueue({ sequence, id: sequence, elapsedMs: 300, sites: ["src/lib/example.ts:1:1"] }); + } + const beforeAck = ring.snapshot(); + expect(beforeAck.map(operation => operation.sequence)).toEqual([3, 4, 5, 6, 7, 8, 9, 10]); + expect(ring.overflow).toBe(2); + + let callback: ((error: Error | null) => void) | undefined; + let acknowledged = 0; + const sender = createBoundedRuntimeDiagnosticSender({ + connected: true, + send(_message, done) { + callback = done; + return false; + }, + }); + expect(sender({ kind: "heartbeat" }, () => { + ring.acknowledge(beforeAck); + acknowledged += 1; + })).toBe(true); + ring.enqueue({ sequence: 11, id: 11, elapsedMs: 300, sites: ["src/lib/new.ts:1:1"] }); + callback?.(new Error("recorder callback failure")); + expect(acknowledged).toBe(0); + expect(ring.snapshot().map(operation => operation.sequence)).toEqual([4, 5, 6, 7, 8, 9, 10, 11]); + + ring.acknowledge(beforeAck); + expect(ring.snapshot().map(operation => operation.sequence)).toEqual([11]); + + const delivered = ring.snapshot(); + let successfulCallback: ((error: Error | null) => void) | undefined; + const queued = createBoundedRuntimeDiagnosticSender({ + connected: true, + send(_message, done) { + successfulCallback = done; + return false; + }, + }); + expect(queued({ kind: "heartbeat" }, () => ring.acknowledge(delivered))).toBe(true); + successfulCallback?.(null); + expect(ring.snapshot()).toEqual([]); +}); + +test("a throwing or suppressed diagnostic send never invokes a delivery acknowledgement", () => { + let acknowledgements = 0; + const throwing = createBoundedRuntimeDiagnosticSender({ + connected: true, + send() { throw new Error("IPC unavailable"); }, + }); + expect(throwing({ kind: "heartbeat" }, () => { acknowledgements += 1; })).toBe(false); + + const suppressed = createBoundedRuntimeDiagnosticSender({ + connected: false, + send() { throw new Error("unreachable"); }, + }); + expect(suppressed({ kind: "heartbeat" }, () => { acknowledgements += 1; })).toBe(false); + expect(acknowledgements).toBe(0); +}); + +test("stop keeps a backpressured parent-stopping marker queued until its callback", async () => { + let callback: ((error: Error | null) => void) | undefined; + let disconnects = 0; + const target = { + connected: true, + send(message: object, done: (error: Error | null) => void) { + expect(message).toEqual(expect.objectContaining({ kind: "parent-stopping" })); + callback = done; + return false; + }, + disconnect() { disconnects += 1; }, + }; + + disconnectAfterRuntimeDiagnosticsStop(target, Date.now(), 100); + await Bun.sleep(20); + expect(disconnects).toBe(0); + callback?.(null); + expect(disconnects).toBe(1); + await Bun.sleep(120); + expect(disconnects).toBe(1); +}); + +test("observability gap clears unmatched sync operations", async () => { + const dir = mkdtempSync(join(tmpdir(), "ocx-runtime-diagnostics-gap-")); + const path = join(dir, "runtime.jsonl"); + const child = fork(fileURLToPath(new URL("../../src/lib/runtime-diagnostics-child.ts", import.meta.url)), [], { + execPath: process.execPath, execArgv: [], stdio: ["ignore", "ignore", "ignore", "ipc"], windowsHide: true, + }); + const closed = new Promise(resolve => child.once("close", () => resolve())); + const ready = new Promise(resolve => child.once("message", () => resolve())); + try { + child.send({ kind: "init", path, pid: process.pid, intervalMs: 1000, stallMs: 5000, logEveryMs: 1000 }); + await Promise.race([ready, Bun.sleep(3000).then(() => { throw new Error("recorder did not start"); })]); + const at = Date.now(); + child.send({ kind: "sync-start", at, id: 7, sites: ["src/lib/example.ts:1:1"] }); + child.send({ kind: "observability-gap", at: at + 1, droppedMessages: 1, syncOperationsDropped: true }); + child.send({ kind: "sync-end", at: at + 1000, id: 7 }); + await Bun.sleep(25); + } finally { + if (child.connected) child.disconnect(); + await closed; + } + try { + const records = readFileSync(path, "utf8").trim().split("\n").map(line => JSON.parse(line)); + expect(records.some(r => r.kind === "ipc-observability-gap" && r.syncOperationsDropped === true)).toBe(true); + expect(records.some(r => r.kind === "slow-sync-operation")).toBe(false); + } finally { removeTreeWithRetry(dir); } +}); + +test("completed slow-operation heartbeat evidence is retained without a delivered start", async () => { + const dir = mkdtempSync(join(tmpdir(), "ocx-runtime-diagnostics-completed-")); + const path = join(dir, "runtime.jsonl"); + const child = fork(fileURLToPath(new URL("../../src/lib/runtime-diagnostics-child.ts", import.meta.url)), [], { + execPath: process.execPath, execArgv: [], stdio: ["ignore", "ignore", "ignore", "ipc"], windowsHide: true, + }); + const closed = new Promise(resolve => child.once("close", () => resolve())); + const ready = new Promise(resolve => child.once("message", () => resolve())); + const completed = { sequence: 77, id: 41, elapsedMs: 360, sites: ["src/lib/runtime-diagnostics.ts:process.memoryUsage"] }; + try { + child.send({ kind: "init", path, pid: process.pid, intervalMs: 20, stallMs: 100, logEveryMs: 1000 }); + await Promise.race([ready, Bun.sleep(3000).then(() => { throw new Error("recorder did not start"); })]); + child.send({ kind: "heartbeat", at: Date.now(), intervalMs: 20, timerDelayMs: 0, + cpuUserDeltaMs: 0, cpuSystemDeltaMs: 0, counters: {}, completedSlowOperations: [completed], + completedSlowOperationOverflow: 2 }); + // A retained snapshot can be retransmitted before the parent sees its IPC + // callback. The child must keep that bounded retry from double-logging it. + child.send({ kind: "heartbeat", at: Date.now(), intervalMs: 20, timerDelayMs: 0, + cpuUserDeltaMs: 0, cpuSystemDeltaMs: 0, counters: {}, completedSlowOperations: [completed], + completedSlowOperationOverflow: 2 }); + await Bun.sleep(50); + } finally { + if (child.connected) child.disconnect(); + await closed; + } + try { + const records = readFileSync(path, "utf8").trim().split("\n").map(line => JSON.parse(line)); + const slow = records.filter(record => record.kind === "slow-sync-operation"); + expect(slow).toHaveLength(1); + expect(slow[0]).toEqual(expect.objectContaining({ elapsedMs: 360, sites: completed.sites })); + expect(records).toContainEqual(expect.objectContaining({ + kind: "completed-slow-operation-overflow", + droppedCompletedSlowOperations: 2, + })); + } finally { removeTreeWithRetry(dir); } +}); + +test("sampling failures retain a bounded base heartbeat and do not manufacture a timer delay", async () => { + const dir = mkdtempSync(join(tmpdir(), "ocx-runtime-diagnostics-sample-")); + const path = join(dir, "runtime.jsonl"); + let samples = 0; + const recorder = startRuntimeDiagnostics(path, () => { + samples += 1; + if (samples === 1) throw new Error("test sample failure"); + return { activeTurns: 0 }; + }, { intervalMs: 20, stallMs: 100, logEveryMs: 20 }); + try { + await Promise.race([recorder.ready, Bun.sleep(3000).then(() => { throw new Error("recorder did not start"); })]); + await Bun.sleep(100); + } finally { + recorder.stop(); + await recorder.closed; + } + try { + const records = readFileSync(path, "utf8").trim().split("\n").map(line => JSON.parse(line)); + expect(records.some(r => r.kind === "sample" && r.counters?.diagnosticsSampleFailures >= 1)).toBe(true); + expect(records.some(r => r.kind === "event-loop-delay")).toBe(false); + } finally { removeTreeWithRetry(dir); } +}); + +test("a slow native memory sample has a fixed diagnostic phase", async () => { + const dir = mkdtempSync(join(tmpdir(), "ocx-runtime-diagnostics-memory-phase-")); + const path = join(dir, "runtime.jsonl"); + const originalMemoryUsage = process.memoryUsage; + let calls = 0; + Object.defineProperty(process, "memoryUsage", { + configurable: true, + value: () => { + calls += 1; + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 300); + return originalMemoryUsage(); + }, + }); + const recorder = startRuntimeDiagnostics(path, () => ({ activeTurns: 0 }), { + intervalMs: 250, stallMs: 100, logEveryMs: 20, + }); + try { + await Promise.race([recorder.ready, Bun.sleep(3000).then(() => { throw new Error("recorder did not start"); })]); + await Bun.sleep(700); + } finally { + Object.defineProperty(process, "memoryUsage", { configurable: true, value: originalMemoryUsage }); + recorder.stop(); + await recorder.closed; + } + try { + expect(calls).toBeGreaterThan(0); + const records = readFileSync(path, "utf8").trim().split("\n").map(line => JSON.parse(line)); + expect(records.some(record => record.kind === "slow-sync-operation" + && record.sites?.includes("src/lib/runtime-diagnostics.ts:process.memoryUsage"))).toBe(true); + } finally { removeTreeWithRetry(dir); } +}); + +test("a throwing native memory sample is counted and leaves no stale phase", async () => { + const dir = mkdtempSync(join(tmpdir(), "ocx-runtime-diagnostics-memory-throw-")); + const path = join(dir, "runtime.jsonl"); + const originalMemoryUsage = process.memoryUsage; + let calls = 0; + Object.defineProperty(process, "memoryUsage", { + configurable: true, + value: () => { + calls += 1; + throw new Error("memory fixture failure"); + }, + }); + const recorder = startRuntimeDiagnostics(path, () => ({ activeTurns: 0 }), { + intervalMs: 250, stallMs: 100, logEveryMs: 20, + }); + try { + await Promise.race([recorder.ready, Bun.sleep(3000).then(() => { throw new Error("recorder did not start"); })]); + await Bun.sleep(300); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 600); + await Bun.sleep(75); + } finally { + Object.defineProperty(process, "memoryUsage", { configurable: true, value: originalMemoryUsage }); + recorder.stop(); + await recorder.closed; + } + try { + expect(calls).toBeGreaterThan(0); + const records = readFileSync(path, "utf8").trim().split("\n").map(line => JSON.parse(line)); + expect(records.some(record => record.counters?.diagnosticsMemoryUsageFailures >= 1)).toBe(true); + const stall = records.find(record => record.kind === "event-loop-stall"); + expect(stall?.operations).toEqual([]); + } finally { removeTreeWithRetry(dir); } +}); + +test("recorder distinguishes an intentional parent shutdown from an unexpected disconnect", async () => { + const dir = mkdtempSync(join(tmpdir(), "ocx-runtime-diagnostics-shutdown-")); + const path = join(dir, "runtime.jsonl"); + const recorder = startRuntimeDiagnostics(path, () => ({ activeTurns: 0 }), { + intervalMs: 20, stallMs: 100, logEveryMs: 1000, + }); + try { + await Promise.race([recorder.ready, Bun.sleep(3000).then(() => { throw new Error("recorder did not start"); })]); + } finally { + recorder.stop(); + await recorder.closed; + } + try { + const records = readFileSync(path, "utf8").trim().split("\n").map(line => JSON.parse(line)); + const started = records.find(r => r.kind === "start"); + expect(started.recorderPid).toEqual(expect.any(Number)); + expect(records.some(r => r.kind === "parent-stopping")).toBe(true); + expect(records.at(-1)).toEqual(expect.objectContaining({ + kind: "parent-disconnected", + expected: true, + })); + } finally { removeTreeWithRetry(dir); } +}); + +test.if(process.platform === "win32")("Windows detached recorder survives a parent exit long enough to record its IPC disconnect", async () => { + const dir = mkdtempSync(join(tmpdir(), "ocx-runtime-diagnostics-detached-")); + const path = join(dir, "runtime.jsonl"); + const probe = Bun.spawn([process.execPath, "-e", ` + import { startRuntimeDiagnostics } from "./src/lib/runtime-diagnostics.ts"; + const recorder = startRuntimeDiagnostics(process.env.OCX_RUNTIME_DIAGNOSTICS_PATH, () => ({ activeTurns: 0 }), { + intervalMs: 20, + stallMs: 100, + logEveryMs: 1000, + }); + await recorder.ready; + process.exit(0); + `], { + cwd: process.cwd(), + env: { ...process.env, OPENCODEX_HOME: join(dir, "home"), OCX_RUNTIME_DIAGNOSTICS_PATH: path }, + stdout: "ignore", + stderr: "ignore", + }); + try { + await Promise.race([probe.exited, Bun.sleep(3000).then(() => { throw new Error("detached parent probe did not exit"); })]); + let content = ""; + for (let index = 0; index < 40; index += 1) { + if (existsSync(path)) { + content = readFileSync(path, "utf8"); + if (content.includes("parent-disconnected")) break; + } + await Bun.sleep(25); + } + const records = content.trim().split("\n").map(line => JSON.parse(line)); + expect(records.at(-1)).toEqual(expect.objectContaining({ + kind: "parent-disconnected", + expected: false, + })); + } finally { + if (probe.exitCode === null) probe.kill(); + removeTreeWithRetry(dir); + } +}); + +test("stopping before child initialization rejects ready and closes without an orphan", async () => { + const dir = mkdtempSync(join(tmpdir(), "ocx-runtime-diagnostics-early-stop-")); + const recorder = startRuntimeDiagnostics(join(dir, "runtime.jsonl"), () => ({ activeTurns: 0 })); + const ready = recorder.ready; + try { + recorder.stop(); + await Promise.race([recorder.closed, Bun.sleep(3000).then(() => { throw new Error("recorder did not close"); })]); + await expect(ready).rejects.toThrow("runtime diagnostics recorder closed before ready"); + } finally { removeTreeWithRetry(dir); } +}); diff --git a/tests/windows/windows-recovery-gateway.test.ts b/tests/windows/windows-recovery-gateway.test.ts new file mode 100644 index 00000000000..d4fde5b9576 --- /dev/null +++ b/tests/windows/windows-recovery-gateway.test.ts @@ -0,0 +1,338 @@ +import { afterEach, expect, test } from "bun:test"; +import { createServer, request as httpRequest, type Server } from "node:http"; +import { spawn } from "node:child_process"; +import { connect } from "node:net"; +import { repoPath } from "../helpers/repo-root"; +import { createGateway } from "../../scripts/ocx-recovery-guardian/gateway.cjs"; + +const closers: Array<() => Promise> = []; +afterEach(async () => { while (closers.length) await closers.pop()!(); }); + +async function listen(handler: Parameters[0]): Promise<{ server: Server; port: number }> { + const server = createServer(handler); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => resolve()); + }); + closers.push(() => new Promise(resolve => { + server.close(() => resolve()); + server.closeAllConnections?.(); + })); + return { server, port: (server.address() as { port: number }).port }; +} + +function call(port: number, options: { method?: string; path?: string; headers?: Record; body?: string } = {}) { + return new Promise<{ status: number; headers: Record; body: string }>((resolve, reject) => { + const req = httpRequest({ host: "127.0.0.1", port, method: options.method ?? "GET", path: options.path ?? "/healthz", + headers: { host: `127.0.0.1:${port}`, ...(options.headers ?? {}) } }, response => { + const chunks: Buffer[] = []; + response.on("data", chunk => chunks.push(Buffer.from(chunk))); + response.on("end", () => resolve({ status: response.statusCode ?? 0, headers: response.headers, body: Buffer.concat(chunks).toString("utf8") })); + }); + req.once("error", reject); + req.end(options.body); + }); +} + +async function gateway(primaryPort: number, fallbackPort: number, primaryReady = true, overrides: Record = {}) { + const instance = await createGateway({ + port: 0, + primaryOrigin: `http://127.0.0.1:${primaryPort}`, + fallbackOrigin: `http://127.0.0.1:${fallbackPort}`, + models: { "codex-test": "or-test" }, + readFallbackKey: async () => "fallback-secret", + isPrimaryReady: () => primaryReady, + isStopped: () => false, + onPrimaryFailure: () => {}, + log: () => {}, + headerTimeoutMs: 500, + ...overrides, + }); + closers.push(instance.close); + return instance; +} + +async function nodeGatewayGetProof() { + const gatewayPath = JSON.stringify(repoPath("scripts", "ocx-recovery-guardian", "gateway.cjs")); + const script = ` + const http = require("node:http"); + const { createGateway } = require(${gatewayPath}); + const listen = handler => new Promise((resolve, reject) => { + const server = http.createServer(handler); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => resolve(server)); + }); + const close = server => new Promise(resolve => { server.close(resolve); server.closeAllConnections?.(); }); + const get = port => new Promise((resolve, reject) => { + const request = http.request({ host: "127.0.0.1", port, path: "/v1/models", headers: { host: "127.0.0.1:" + port } }, response => { + let body = ""; response.setEncoding("utf8"); response.on("data", chunk => body += chunk); response.on("end", () => resolve({ status: response.statusCode, body })); + }); + request.once("error", reject); request.end(); + }); + (async () => { + const primary = await listen((_req, res) => res.end("primary-models")); + const fallback = await listen((_req, res) => res.end("fallback-models")); + let primaryReady = true; + const gateway = await createGateway({ + port: 0, + primaryOrigin: "http://127.0.0.1:" + primary.address().port, + fallbackOrigin: "http://127.0.0.1:" + fallback.address().port, + models: {}, readFallbackKey: async () => "fixed-test-key", + isPrimaryReady: () => primaryReady, isStopped: () => false, onPrimaryFailure: () => {}, log: () => {}, + }); + try { + const primaryResponse = await get(gateway.port); + primaryReady = false; + const fallbackResponse = await get(gateway.port); + if (primaryResponse.status !== 200 || primaryResponse.body !== "primary-models" || fallbackResponse.status !== 200 || fallbackResponse.body !== "fallback-models") throw new Error("GET route result mismatch"); + } finally { await gateway.close(); await close(primary); await close(fallback); } + })().catch(error => { console.error(error && error.stack || error); process.exitCode = 1; }); + `; + await new Promise((resolve, reject) => { + const child = spawn("node.exe", ["-e", script], { stdio: ["ignore", "ignore", "pipe"], windowsHide: true }); + let stderr = ""; + const timer = setTimeout(() => child.kill(), 10_000); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", chunk => { stderr += chunk; }); + child.once("error", error => { clearTimeout(timer); reject(error); }); + child.once("close", code => { + clearTimeout(timer); + if (code === 0) resolve(); else reject(new Error(`Node gateway GET proof failed (${code}): ${stderr}`)); + }); + }); +} + +test("healthy primary receives the supported route", async () => { + let hits = 0; + const primary = await listen((req, res) => { + hits += 1; + expect(req.url).toBe("/v1/responses"); + expect(req.headers["x-opencodex-api-key"]).toBe("ocx-key"); + expect(req.headers["chatgpt-account-id"]).toBe("account"); + expect(req.headers.session_id).toBe("session"); + expect(req.headers["x-codex-trace"]).toBe("trace"); + expect(req.headers["x-connection-nominated"]).toBeUndefined(); + res.writeHead(200, { authorization: "Bearer upstream", "set-cookie": "secret=value", location: "http://secret@example.test", "x-api-key": "upstream-key" }); + res.end("primary"); + }); + const fallback = await listen((_req, res) => res.end("fallback")); + const instance = await gateway(primary.port, fallback.port); + const result = await call(instance.port, { method: "POST", path: "/v1/responses", headers: { + "x-opencodex-api-key": "ocx-key", "chatgpt-account-id": "account", session_id: "session", "x-codex-trace": "trace", + connection: "keep-alive, x-connection-nominated", "x-connection-nominated": "must-not-forward", + }, body: JSON.stringify({ model: "codex-test" }) }); + expect(result.status).toBe(200); + expect(result.body).toBe("primary"); + expect(hits).toBe(1); + expect(result.headers.authorization).toBeUndefined(); + expect(result.headers["set-cookie"]).toBeUndefined(); + expect(result.headers.location).toBeUndefined(); + expect(result.headers["x-api-key"]).toBeUndefined(); +}); + +test("unready primary sends an exact mapped model to fallback with stripped credentials", async () => { + const primary = await listen((_req, res) => res.end("unexpected")); + let seen = ""; + let authorization = ""; + const fallback = await listen((req, res) => { + authorization = String(req.headers.authorization); + expect(req.headers.cookie).toBeUndefined(); + expect(req.headers["x-api-key"]).toBeUndefined(); + req.on("data", chunk => { seen += chunk; }); + req.on("end", () => res.end("fallback")); + }); + const instance = await gateway(primary.port, fallback.port, false); + const result = await call(instance.port, { method: "POST", path: "/v1/responses", headers: { authorization: "Bearer primary", "x-api-key": "primary-key", "openai-project": "primary-project" }, body: JSON.stringify({ model: "codex-test" }) }); + expect(result.body).toBe("fallback"); + expect(authorization).toBe("Bearer fallback-secret"); + expect(JSON.parse(seen)).toMatchObject({ model: "or-test" }); +}); + +test("incompatible fallback models and previous-response continuations do not go outbound", async () => { + const primary = await listen((_req, res) => res.end("unexpected")); + let fallbackHits = 0; + const fallback = await listen((_req, res) => { fallbackHits += 1; res.end("unexpected"); }); + const instance = await gateway(primary.port, fallback.port, false); + const incompatible = await call(instance.port, { method: "POST", path: "/v1/responses", body: JSON.stringify({ model: "other" }) }); + expect(incompatible.status).toBe(503); + expect(incompatible.body).toContain("fallback_model_unavailable"); + const continuation = await call(instance.port, { method: "POST", path: "/v1/responses", body: JSON.stringify({ model: "codex-test", previous_response_id: "resp_1" }) }); + expect(continuation.status).toBe(503); + expect(continuation.body).toContain("fallback_requires_fresh_full_context"); + expect(fallbackHits).toBe(0); +}); + +test("a post-dispatch primary failure is never replayed to fallback", async () => { + let primaryFailures = 0; + const primary = await listen((_req, res) => { primaryFailures += 1; res.writeHead(503); res.end("down"); }); + let fallbackHits = 0; + const fallback = await listen((_req, res) => { fallbackHits += 1; res.end("fallback"); }); + const instance = await gateway(primary.port, fallback.port); + const result = await call(instance.port, { method: "POST", path: "/v1/responses", body: JSON.stringify({ model: "codex-test" }) }); + expect(result.status).toBe(503); + expect(primaryFailures).toBe(1); + expect(fallbackHits).toBe(0); +}); + +test("primary 4xx is relayed without declaring the primary globally failed", async () => { + const primary = await listen((_req, res) => { res.writeHead(400); res.end("client error"); }); + const fallback = await listen((_req, res) => res.end("fallback")); + let failures = 0; + const instance = await gateway(primary.port, fallback.port, true, { onPrimaryFailure: () => { failures += 1; } }); + const result = await call(instance.port, { method: "POST", path: "/v1/responses", body: JSON.stringify({ model: "codex-test" }) }); + expect(result.status).toBe(400); + expect(failures).toBe(0); +}); + +test("handler exceptions become a sterile 502 instead of an unhandled rejection", async () => { + const primary = await listen((_req, res) => res.end("unexpected")); + const fallback = await listen((_req, res) => res.end("unexpected")); + let failures = 0; + const instance = await gateway(primary.port, fallback.port, true, { + isPrimaryReady: () => { throw new Error("fixture"); }, + onPrimaryFailure: () => { failures += 1; }, + }); + const result = await call(instance.port, { path: "/v1/models" }); + expect(result.status).toBe(502); + expect(result.body).toContain("gateway_unavailable"); + expect(result.body).not.toContain("fixture"); + expect(failures).toBe(1); +}); + +test("streaming primary output is passed through once and client close does not retry", async () => { + let primaryHits = 0; + const primary = await listen((_req, res) => { + primaryHits += 1; + res.writeHead(200, { "content-type": "text/event-stream" }); + res.write("data: first\n\n"); + setTimeout(() => res.end("data: [DONE]\n\n"), 25); + }); + let fallbackHits = 0; + const fallback = await listen((_req, res) => { fallbackHits += 1; res.end("fallback"); }); + const instance = await gateway(primary.port, fallback.port); + const result = await call(instance.port, { method: "POST", path: "/v1/responses", body: JSON.stringify({ model: "codex-test" }) }); + expect(result.body).toContain("data: first"); + expect(primaryHits).toBe(1); + expect(fallbackHits).toBe(0); +}); + +test("disconnecting a streaming client aborts upstream without fallback replay", async () => { + let primaryHits = 0; + const primary = await listen((_req, res) => { + primaryHits += 1; + res.writeHead(200, { "content-type": "text/event-stream" }); + res.write("data: first\n\n"); + setTimeout(() => res.write("data: later\n\n"), 100); + }); + let fallbackHits = 0; + const fallback = await listen((_req, res) => { fallbackHits += 1; res.end("fallback"); }); + const instance = await gateway(primary.port, fallback.port); + await new Promise((resolve, reject) => { + const req = httpRequest({ host: "127.0.0.1", port: instance.port, method: "POST", path: "/v1/responses", + headers: { host: `127.0.0.1:${instance.port}`, "content-type": "application/json" } }, response => { + response.once("data", () => { response.destroy(); resolve(); }); + }); + req.once("error", reject); + req.end(JSON.stringify({ model: "codex-test", stream: true })); + }); + await Bun.sleep(50); + expect(primaryHits).toBe(1); + expect(fallbackHits).toBe(0); +}); + +test("request ceiling and browser or Host requests are denied locally", async () => { + const primary = await listen((_req, res) => res.end("unexpected")); + const fallback = await listen((_req, res) => res.end("unexpected")); + const instance = await gateway(primary.port, fallback.port); + const origin = await call(instance.port, { headers: { origin: "https://example.test" } }); + expect(origin.status).toBe(403); + const badHost = await new Promise<{ status: number }>((resolve, reject) => { + const req = httpRequest({ host: "127.0.0.1", port: instance.port, path: "/healthz", headers: { host: "example.test" } }, res => resolve({ status: res.statusCode ?? 0 })); + req.once("error", reject); req.end(); + }); + expect(badHost.status).toBe(421); + const query = await call(instance.port, { path: "/healthz?x=1" }); + expect(query.status).toBe(404); + const healthPost = await call(instance.port, { method: "POST", path: "/healthz", body: "{}" }); + expect(healthPost.status).toBe(405); + const large = await call(instance.port, { method: "POST", path: "/v1/responses", body: "x".repeat(8 * 1024 * 1024 + 1) }); + expect(large.status).toBe(413); +}); + +test("stopped gateway remains identifiable at healthz but rejects ready and data routes", async () => { + const primary = await listen((_req, res) => res.end("unexpected")); + const fallback = await listen((_req, res) => res.end("unexpected")); + const instance = await gateway(primary.port, fallback.port, true, { + isStopped: () => true, + isPrimaryReady: () => { throw new Error("stopped probe must not call readiness"); }, + }); + const health = await call(instance.port, { path: "/healthz" }); + expect(health.status).toBe(200); + expect(JSON.parse(health.body)).toMatchObject({ service: "ocx-recovery-gateway" }); + expect((await call(instance.port, { path: "/readyz" })).status).toBe(503); + expect((await call(instance.port, { method: "POST", path: "/v1/responses", body: JSON.stringify({ model: "codex-test" }) })).status).toBe(503); +}); + +test("concurrent requests are capped before another 8 MiB body can be admitted", async () => { + let primaryHits = 0; + const held: import("node:http").ServerResponse[] = []; + const primary = await listen((_req, response) => { primaryHits += 1; held.push(response); }); + const fallback = await listen((_req, res) => res.end("unexpected")); + const instance = await gateway(primary.port, fallback.port, true, { maxConcurrentRequests: 64, headerTimeoutMs: 120_000 }); + for (let index = 0; index < 64; index += 1) { + const req = httpRequest({ host: "127.0.0.1", port: instance.port, method: "POST", path: "/v1/responses", + headers: { host: `127.0.0.1:${instance.port}`, "content-type": "application/json" } }); + req.on("error", () => {}); + req.end(JSON.stringify({ model: "codex-test", index })); + } + for (let attempt = 0; attempt < 100 && primaryHits < 64; attempt += 1) await Bun.sleep(10); + expect(primaryHits).toBe(64); + const rejected = await call(instance.port, { method: "POST", path: "/v1/responses", body: JSON.stringify({ model: "codex-test" }) }); + expect(rejected.status).toBe(503); + expect(rejected.body).toContain("gateway_busy"); + for (const response of held) response.end(); + await Bun.sleep(20); +}); + +test("self-proxy origins and upgrades fail closed", async () => { + const upstream = await listen((_req, res) => res.end("upstream")); + await expect(createGateway({ + port: upstream.port, primaryOrigin: `http://127.0.0.1:${upstream.port}`, fallbackOrigin: "http://127.0.0.1:9", + models: {}, readFallbackKey: async () => "x", isPrimaryReady: () => true, isStopped: () => false, onPrimaryFailure: () => {}, log: () => {}, + })).rejects.toThrow(); +}); + +test("only canonical numeric IPv4 loopback origins are accepted", async () => { + const invalidOrigins = [ + "http://localhost:1234", + "http://127.000.000.001:1234", + "http://2130706433:1234", + "http://[::1]:1234", + "http://127.0.0.1:01234", + ]; + for (const primaryOrigin of invalidOrigins) { + await expect(createGateway({ + port: 0, primaryOrigin, fallbackOrigin: "http://127.0.0.1:1235", + models: {}, readFallbackKey: async () => "x", isPrimaryReady: () => true, isStopped: () => false, onPrimaryFailure: () => {}, log: () => {}, + })).rejects.toThrow("canonical numeric loopback"); + } +}); + +test("a real Node child relays GET models through both primary and fallback", async () => { + await nodeGatewayGetProof(); +}); + +test("WebSocket upgrades receive 426 rather than a proxied connection", async () => { + const primary = await listen((_req, res) => res.end("primary")); + const fallback = await listen((_req, res) => res.end("fallback")); + const instance = await gateway(primary.port, fallback.port); + const response = await new Promise((resolve, reject) => { + const socket = connect(instance.port, "127.0.0.1"); + let received = ""; + socket.on("connect", () => socket.write(`GET /v1/realtime HTTP/1.1\r\nHost: 127.0.0.1:${instance.port}\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n`)); + socket.on("data", chunk => { received += chunk; }); + socket.on("end", () => resolve(received)); + socket.on("error", reject); + }); + expect(response).toContain("426 Upgrade Required"); +}); diff --git a/tests/windows/windows-recovery-intent.test.ts b/tests/windows/windows-recovery-intent.test.ts new file mode 100644 index 00000000000..75891d4457a --- /dev/null +++ b/tests/windows/windows-recovery-intent.test.ts @@ -0,0 +1,202 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { copyFileSync, existsSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { writeRecoveryIntentIfGuardianEnabled } from "../../src/lib/recovery-intent"; +import { repoPath } from "../helpers/repo-root"; + +const fixtures: string[] = []; +afterEach(() => { while (fixtures.length) rmSync(fixtures.pop()!, { recursive: true, force: true }); }); + +function homeFixture(): string { + const home = mkdtempSync(join(tmpdir(), "ocx-recovery-intent-")); + fixtures.push(home); + return home; +} + +function enableGuardian(home: string): void { + writeFileSync(join(home, "recovery-guardian.json"), JSON.stringify({ version: 1, enabled: true })); +} + +describe("persistent manual recovery intent", () => { + test("leaves no intent when the guardian marker is missing or explicitly disabled", async () => { + const missing = homeFixture(); + expect(await writeRecoveryIntentIfGuardianEnabled("stopped", { home: missing, at: 11 })).toBe(false); + expect(existsSync(join(missing, "recovery-intent.json"))).toBe(false); + + const disabled = homeFixture(); + writeFileSync(join(disabled, "recovery-guardian.json"), JSON.stringify({ version: 1, enabled: false })); + expect(await writeRecoveryIntentIfGuardianEnabled("running", { home: disabled, at: 12 })).toBe(false); + expect(existsSync(join(disabled, "recovery-intent.json"))).toBe(false); + }); + + test("writes the bounded v1 stopped and maintenance intents only for an enabled guardian", async () => { + const home = homeFixture(); + enableGuardian(home); + expect(await writeRecoveryIntentIfGuardianEnabled("stopped", { home, at: 21 })).toBe(true); + expect(JSON.parse(readFileSync(join(home, "recovery-intent.json"), "utf8"))).toEqual({ version: 1, mode: "stopped", at: 21 }); + + expect(await writeRecoveryIntentIfGuardianEnabled("maintenance", { home, at: 22, until: 202 })).toBe(true); + expect(JSON.parse(readFileSync(join(home, "recovery-intent.json"), "utf8"))).toEqual({ version: 1, mode: "maintenance", at: 22, until: 202 }); + }); + + test("fails closed without changing the intent for a malformed enabled-marker boundary", async () => { + const home = homeFixture(); + writeFileSync(join(home, "recovery-guardian.json"), "{ nope"); + await expect(writeRecoveryIntentIfGuardianEnabled("stopped", { home, at: 31 })).rejects.toThrow("Recovery guardian marker is malformed"); + expect(existsSync(join(home, "recovery-intent.json"))).toBe(false); + }); + + test("wires manual stop, receipt-backed API stop, and visible-launcher start/restart through the same fail-closed contract", () => { + const cli = readFileSync(repoPath("src/cli/index.ts"), "utf8"); + const api = readFileSync(repoPath("src/server/management-api.ts"), "utf8"); + const launcher = readFileSync(repoPath("scripts/windows-visible-proxy.ps1"), "utf8"); + const tray = readFileSync(repoPath("src/tray/windows-tray.ps1"), "utf8"); + + expect(cli).toContain('OPENCODEX_GUARDIAN_RECOVERY'); + expect(cli).toContain('writeRecoveryIntentIfGuardianEnabled("stopped")'); + expect(api).toContain("if (!holdsReceipt)"); + expect(api).toContain('writeRecoveryIntentIfGuardianEnabled("stopped")'); + expect(launcher).toContain('Set-RecoveryIntent -OpenCodexDirectory $OpenCodexHome -Mode "running"'); + expect(launcher).toContain('Set-RecoveryIntent -OpenCodexDirectory $OpenCodexHome -Mode "maintenance"'); + expect(launcher).toContain('$startInfo.EnvironmentVariables["OPENCODEX_GUARDIAN_RECOVERY"] = "1"'); + expect(launcher).toContain('Ensure-RecoveryGuardian -OpenCodexDirectory $OpenCodexHome -Root $ProjectRoot -EffectiveCodexHome $CodexHome -PrimaryPort $Port'); + expect(launcher).toContain("service -ceq 'ocx-recovery-gateway'"); + expect(launcher).toContain("Start-Process -FilePath $expectedNode"); + expect(launcher).toContain("ConvertTo-RecoveryGuardianArgument"); + expect(launcher).toContain("-ArgumentList $guardianArgs"); + expect(launcher).toContain('$marker.primaryPort -eq $PrimaryPort'); + expect(launcher).toContain('$null -ne $marker.fallback.models'); + expect(tray).toContain('Set-RecoveryIntent -OpenCodexHome $OpenCodexHome -Mode "stopped"'); + expect(tray).toContain('Set-RecoveryIntent -OpenCodexHome $OpenCodexHome -Mode "running"'); + expect(tray).toContain('Test-RecoveryGuardianIntentEnabled'); + expect(tray).toContain('if (-not (Test-RecoveryGuardianIntentEnabled $OpenCodexHome)) { return }'); + }); + + test("ships the tray recovery helper as an owned installed asset through status, rollback, and uninstall", () => { + const trayTs = readFileSync(repoPath("src/tray/windows.ts"), "utf8"); + expect(existsSync(repoPath("scripts/ocx-recovery-guardian/intent.ps1"))).toBe(true); + expect(trayTs).toContain('const INSTALLED_TRAY_RECOVERY_INTENT_FILE = "opencodex-recovery-intent.ps1"'); + expect(trayTs).toContain("sourceTrayRecoveryIntentPath()"); + expect(trayTs).toContain("installedTrayRecoveryIntentPath()"); + expect(trayTs).toContain("replaceWindowsTrayOwnedFile(installedRecoveryIntent"); + expect(trayTs).toContain("previousRecoveryIntentBytes"); + expect(trayTs).toContain("installedTrayRecoveryIntentPath()]"); + }); + + test("the PowerShell helper replaces existing stopped, maintenance, and running intents atomically", () => { + if (process.platform !== "win32") return; + const home = homeFixture(); + const helper = repoPath("scripts/ocx-recovery-guardian/intent.ps1"); + const ps = join(process.env.SystemRoot ?? "C:\\Windows", "System32", "WindowsPowerShell", "v1.0", "powershell.exe"); + const quote = (value: string) => `'${value.replaceAll("'", "''")}'`; + const intent = join(home, "recovery-intent.json"); + const code = `$null = & ${quote(helper)} -OpenCodexHome ${quote(home)} -Mode stopped; if (Test-Path -LiteralPath ${quote(intent)}) { exit 11 }; Set-Content -LiteralPath ${quote(join(home, "recovery-guardian.json"))} -Value '{"version":1,"enabled":true}' -NoNewline; Set-Content -LiteralPath ${quote(intent)} -Value '{"version":1,"mode":"stopped","at":1}' -NoNewline; $null = & ${quote(helper)} -OpenCodexHome ${quote(home)} -Mode maintenance -Until 77; $maintenance = Get-Content -LiteralPath ${quote(intent)} -Raw | ConvertFrom-Json; $null = & ${quote(helper)} -OpenCodexHome ${quote(home)} -Mode running; $running = Get-Content -LiteralPath ${quote(intent)} -Raw | ConvertFrom-Json; $null = & ${quote(helper)} -OpenCodexHome ${quote(home)} -Mode stopped; $stopped = Get-Content -LiteralPath ${quote(intent)} -Raw | ConvertFrom-Json; $temps = @(Get-ChildItem -LiteralPath ${quote(home)} -Filter '.recovery-intent.*.tmp'); @($maintenance, $running, $stopped, $temps.Count) | ConvertTo-Json -Compress`; + const run = Bun.spawnSync([ps, "-NoProfile", "-NonInteractive", "-Command", code], { stdout: "pipe", stderr: "pipe", timeout: 15_000 }); + expect(run.exitCode, Buffer.from(run.stderr).toString()).toBe(0); + const [maintenance, running, stopped, tempCount] = JSON.parse(Buffer.from(run.stdout).toString()) as [{ version: number; mode: string; until?: number }, { mode: string }, { mode: string }, number]; + expect(maintenance).toMatchObject({ version: 1, mode: "maintenance", until: 77 }); + expect(running).toMatchObject({ version: 1, mode: "running" }); + expect(stopped).toMatchObject({ version: 1, mode: "stopped" }); + expect(tempCount).toBe(0); + }, 20_000); + + test("visible launcher and tray invoke the enabled recovery helper with named parameters", () => { + if (process.platform !== "win32") return; + const root = homeFixture(); + const project = join(root, "project"); + const launcherHome = join(root, "launcher-home"); + const trayHome = join(root, "tray-home"); + const trayDir = join(root, "tray"); + const helper = repoPath("scripts/ocx-recovery-guardian/intent.ps1"); + mkdirSync(join(project, "scripts", "ocx-recovery-guardian"), { recursive: true }); + mkdirSync(launcherHome, { recursive: true }); + mkdirSync(trayHome, { recursive: true }); + mkdirSync(trayDir, { recursive: true }); + copyFileSync(helper, join(project, "scripts", "ocx-recovery-guardian", "intent.ps1")); + copyFileSync(helper, join(trayDir, "opencodex-recovery-intent.ps1")); + writeFileSync(join(launcherHome, "recovery-guardian.json"), '{"version":1,"enabled":true}'); + writeFileSync(join(trayHome, "recovery-guardian.json"), '{"version":1,"enabled":true}'); + + const launcher = readFileSync(repoPath("scripts/windows-visible-proxy.ps1"), "utf8"); + const launcherStart = launcher.indexOf("function Set-RecoveryIntent {"); + const launcherEnd = launcher.indexOf("function Get-VisibleProxyMutexName", launcherStart); + const tray = readFileSync(repoPath("src/tray/windows-tray.ps1"), "utf8"); + const trayStart = tray.indexOf("function Test-RecoveryGuardianIntentEnabled"); + const trayEnd = tray.indexOf("function Update-TrayState", trayStart); + expect(launcherStart).toBeGreaterThan(0); + expect(launcherEnd).toBeGreaterThan(launcherStart); + expect(trayStart).toBeGreaterThan(0); + expect(trayEnd).toBeGreaterThan(trayStart); + const quote = (value: string) => `'${value.replaceAll("'", "''")}'`; + const launcherScript = join(root, "invoke-launcher-intent.ps1"); + writeFileSync(launcherScript, `$ErrorActionPreference='Stop'\n$ProjectRoot=${quote(project)}\n${launcher.slice(launcherStart, launcherEnd)}\nSet-RecoveryIntent -OpenCodexDirectory ${quote(launcherHome)} -Mode stopped\n`); + const trayScript = join(trayDir, "invoke-tray-intent.ps1"); + writeFileSync(trayScript, `$ErrorActionPreference='Stop'\n${tray.slice(trayStart, trayEnd)}\nSet-RecoveryIntent -OpenCodexHome ${quote(trayHome)} -Mode maintenance -Until 123\n`); + const ps = join(process.env.SystemRoot ?? "C:\\Windows", "System32", "WindowsPowerShell", "v1.0", "powershell.exe"); + for (const script of [launcherScript, trayScript]) { + const run = Bun.spawnSync([ps, "-NoProfile", "-NonInteractive", "-File", script], { stdout: "pipe", stderr: "pipe", timeout: 15_000 }); + expect(run.exitCode, Buffer.from(run.stderr).toString()).toBe(0); + } + expect(JSON.parse(readFileSync(join(launcherHome, "recovery-intent.json"), "utf8"))).toMatchObject({ mode: "stopped" }); + expect(JSON.parse(readFileSync(join(trayHome, "recovery-intent.json"), "utf8"))).toMatchObject({ mode: "maintenance", until: 123 }); + }, 30_000); + + test("the visible launcher CheckOnly path parses but never initializes a guardian or writes intent", () => { + if (process.platform !== "win32") return; + const root = homeFixture(); + const project = join(root, "project"); + const home = join(root, "home"); + const codex = join(root, "codex"); + mkdirSync(join(project, "node_modules", "bun", "bin"), { recursive: true }); + mkdirSync(join(project, "src", "cli"), { recursive: true }); + mkdirSync(home, { recursive: true }); + mkdirSync(codex, { recursive: true }); + writeFileSync(join(project, "node_modules", "bun", "bin", "bun.exe"), "fixture"); + writeFileSync(join(project, "src", "cli", "index.ts"), "// fixture\n"); + const launcher = repoPath("scripts/windows-visible-proxy.ps1"); + const ps = join(process.env.SystemRoot ?? "C:\\Windows", "System32", "WindowsPowerShell", "v1.0", "powershell.exe"); + const run = Bun.spawnSync([ps, "-NoProfile", "-NonInteractive", "-File", launcher, + "-ProjectRoot", project, "-OpenCodexHome", home, "-CodexHome", codex, "-CheckOnly", "-NoPause"], + { stdout: "pipe", stderr: "pipe", timeout: 15_000 }); + expect(run.exitCode, Buffer.from(run.stderr).toString()).toBe(0); + expect(Buffer.from(run.stdout).toString()).toContain("check passed"); + expect(existsSync(join(home, "recovery-intent.json"))).toBe(false); + }, 20_000); + + test("the visible guardian launcher passes an exact config path containing spaces to Node", () => { + if (process.platform !== "win32") return; + const root = homeFixture(); + const project = join(root, "project with spaces"); + const home = join(root, "home with spaces"); + const codex = join(root, "codex with spaces"); + const guardianDir = join(project, "scripts", "ocx-recovery-guardian"); + mkdirSync(guardianDir, { recursive: true }); + mkdirSync(home, { recursive: true }); + mkdirSync(codex, { recursive: true }); + const marker = join(home, "recovery-guardian.json"); + const main = join(guardianDir, "main.cjs"); + const nodePath = join(process.env.ProgramFiles ?? "C:\\Program Files", "nodejs", "node.exe"); + if (!existsSync(nodePath)) return; + writeFileSync(main, "require('node:fs').writeFileSync(process.argv[3] + '.argv', JSON.stringify(process.argv.slice(1)));\n"); + writeFileSync(marker, JSON.stringify({ + version: 1, enabled: true, projectRoot: project, openCodexHome: home, codexHome: codex, + nodePath, listenPort: 31997, primaryPort: 10100, fallback: { models: { fixture: "fixture" } }, repair: {}, + })); + const source = readFileSync(repoPath("scripts/windows-visible-proxy.ps1"), "utf8"); + // The main launcher has the only column-zero `try`. Keep extraction below + // its function region, while permitting diagnostics before path validation. + const boundary = source.indexOf("\ntry {\n"); + expect(boundary).toBeGreaterThan(0); + expect(source.slice(boundary, boundary + 400)).toContain("$ProjectRoot = Resolve-AbsolutePath"); + const functions = source.slice(source.indexOf("$LogFileName ="), boundary); + const quote = (value: string) => `'${value.replaceAll("'", "''")}'`; + const psFile = join(root, "invoke-guardian.ps1"); + writeFileSync(psFile, `$ErrorActionPreference='Stop'\n$ProjectRoot=${quote(project)}\n$OpenCodexHome=${quote(home)}\n$CodexHome=${quote(codex)}\n$Port=10100\n${functions}\nforeach ($unsafe in @('quote"unsafe', ('control' + [char]10))) { try { ConvertTo-RecoveryGuardianArgument -Value $unsafe | Out-Null; throw 'unsafe argument was accepted' } catch { if ($_.Exception.Message -eq 'unsafe argument was accepted') { throw } } }\nEnsure-RecoveryGuardian -OpenCodexDirectory $OpenCodexHome -Root $ProjectRoot -EffectiveCodexHome $CodexHome -PrimaryPort $Port\nStart-Sleep -Milliseconds 600\n`); + const ps = join(process.env.SystemRoot ?? "C:\\Windows", "System32", "WindowsPowerShell", "v1.0", "powershell.exe"); + const run = Bun.spawnSync([ps, "-NoProfile", "-NonInteractive", "-File", psFile], { stdout: "pipe", stderr: "pipe", timeout: 15_000 }); + expect(run.exitCode, Buffer.from(run.stderr).toString()).toBe(0); + expect(JSON.parse(readFileSync(marker + ".argv", "utf8"))).toEqual([main, "--config", marker]); + }, 20_000); +}); diff --git a/tests/windows/windows-recovery-main.test.ts b/tests/windows/windows-recovery-main.test.ts new file mode 100644 index 00000000000..a3c35c34bf2 --- /dev/null +++ b/tests/windows/windows-recovery-main.test.ts @@ -0,0 +1,235 @@ +import { afterEach, expect, test } from "bun:test"; +import { mkdtemp, rm, readFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createServer } from "node:net"; +import { repoRoot } from "../helpers/repo-root"; +import { atomicJson, createGuardian, parseIntent, recoveryActionSucceeded } from "../../scripts/ocx-recovery-guardian/main.cjs"; +import { RecoveryPolicy } from "../../scripts/ocx-recovery-guardian/policy.cjs"; + +const cleanup: Array<() => Promise> = []; +afterEach(async () => { while (cleanup.length) await cleanup.pop()!(); }); +async function fixture() { + const home = await mkdtemp(join(tmpdir(), "ocx-guardian-main-")); + cleanup.push(() => rm(home, { recursive: true, force: true })); + const listener = createServer(); + await new Promise(resolve => listener.listen(0, "127.0.0.1", resolve)); + const port = (listener.address() as { port: number }).port; + await new Promise(resolve => listener.close(() => resolve())); + const config = join(home, "recovery-guardian.json"); + await atomicJson(config, { version: 1, enabled: true, projectRoot: repoRoot(), openCodexHome: home, codexHome: home, + listenPort: port, primaryPort: 10100, fallback: { origin: "http://127.0.0.1:20128", models: {}, key: { kind: "or-protected" } }, + repair: { origin: "http://127.0.0.1:11434/v1", key: { kind: "ollama-local" } } }); + return { home, config, intent: join(home, "recovery-intent.json") }; +} + +test("manual intent and action receipt are strict, not exit-code success", () => { + expect(parseIntent(null, 100).mode).toBe("stopped"); + expect(parseIntent({ version: 1, at: 0, mode: "running" }, 100).valid).toBe(true); + expect(parseIntent({ version: 1, at: 100, mode: "maintenance", until: 99999999 }, 100).valid).toBe(false); + expect(recoveryActionSucceeded({ ok: true, value: { action: "refused" } })).toBe(false); + expect(recoveryActionSucceeded({ ok: true, value: { action: "started" } })).toBe(true); +}); + +test("concurrent state writes remain complete JSON without temporary-file collisions", async () => { + const f = await fixture(); + const target = join(f.home, "state.json"); + await Promise.all(Array.from({ length: 16 }, (_, n) => atomicJson(target, { n }))); + expect(JSON.parse(await readFile(target, "utf8"))).toEqual({ n: 15 }); +}); + +test("real guardian entrypoint recovers once, records failed receipt, dispatches GLM, honors manual stop", async () => { + const f = await fixture(); + let now = 100000, healthy = true, actions = 0, repairs = 0; + await atomicJson(f.intent, { version: 1, mode: "running", at: now }); + const guardian = await createGuardian(f.config, { now: () => now, policyOptions: { startupGraceMs: 0 }, alive: () => true, + inspect: async () => ({ ok: true, value: { owned: true, pid: 42, start: "ticks", launcherPid: 43, launcherStart: "ticks" } }), + probe: async () => ({ ok: healthy, pid: 42 }), + action: async () => { actions++; return { ok: true, value: { action: "refused", reason: "stop-uncertain" } }; }, + repair: async () => { repairs++; return { outcome: "no_candidate", candidateCount: 0, requestCount: 1 }; } }); + cleanup.push(guardian.close); + await guardian.tick(); + expect(guardian.state().primaryReady).toBe(false); + now += 30000; await guardian.tick(); + expect(guardian.state().primaryReady).toBe(true); + healthy = false; + for (let n = 0; n < 12; n++) { now += 2000; await guardian.tick(); await guardian.drain(); } + await guardian.drain(); + expect(actions).toBe(1); + expect(repairs).toBe(1); + expect(guardian.state().recoveryBlocked).toBe(true); + expect(guardian.state().lastRecovery.ok).toBe(false); + expect(guardian.state().lastRepair.outcome).toBe("no_candidate"); + await atomicJson(f.intent, { version: 1, mode: "stopped", at: ++now }); + await guardian.tick(); + expect(guardian.state().state).toBe("stopped"); + const response = await fetch(`http://127.0.0.1:${guardian.server.port}/v1/models`); + expect(response.status).toBe(503); + expect((await response.json()).error.code).toBe("gateway_stopped"); +}); + +test("first start and new running intent require a fresh stable-ready interval", async () => { + const f = await fixture(); + let now = 100000; + await atomicJson(f.intent, { version: 1, mode: "running", at: now }); + const guardian = await createGuardian(f.config, { now: () => now, policyOptions: { recoveryStableMs: 4000 }, alive: () => true, + inspect: async () => ({ ok: true, value: { owned: true, pid: 42, start: "ticks", launcherPid: 43, launcherStart: "ticks" } }), + probe: async () => ({ ok: true, pid: 42 }) }); + cleanup.push(guardian.close); + await guardian.tick(); + expect(guardian.state().primaryReady).toBe(false); + now += 2000; await guardian.tick(); + expect(guardian.state().primaryReady).toBe(false); + now += 2000; await guardian.tick(); + expect(guardian.state().primaryReady).toBe(true); + await atomicJson(f.intent, { version: 1, mode: "running", at: ++now }); + await guardian.tick(); + expect(guardian.state().primaryReady).toBe(false); + now += 4000; await guardian.tick(); + expect(guardian.state().primaryReady).toBe(true); +}); + +test("accepted replacement that never becomes ready dispatches GLM before a second recovery", async () => { + const f = await fixture(); + let now = 100000, healthy = true, actions = 0, repairs = 0; + await atomicJson(f.intent, { version: 1, mode: "running", at: now }); + const guardian = await createGuardian(f.config, { now: () => now, + policyOptions: { startupGraceMs: 0, recoveryStableMs: 1000 }, alive: () => true, + inspect: async () => ({ ok: true, value: { owned: true, pid: 42, start: "ticks", launcherPid: 43, launcherStart: "ticks" } }), + probe: async () => ({ ok: healthy, pid: 42 }), + action: async () => { actions++; return { ok: true, value: { action: "started" } }; }, + repair: async () => { repairs++; return { outcome: "no_candidate", candidateCount: 0, requestCount: 1 }; } }); + cleanup.push(guardian.close); + await guardian.tick(); now += 1000; await guardian.tick(); + expect(guardian.state().primaryReady).toBe(true); + healthy = false; + for (let n = 0; n < 12 && actions === 0; n++) { + now += 2000; await guardian.tick(); await guardian.drain(); + } + expect(actions).toBe(1); + expect(repairs).toBe(0); + expect(guardian.state().lastRecovery.ok).toBe(true); + expect(guardian.state().recoveryBlocked).toBe(false); + expect(guardian.state().primaryReady).toBe(false); + now += 1000; await guardian.tick(); await guardian.drain(); + expect(repairs).toBe(1); + expect(actions).toBe(1); + expect(guardian.state().primaryReady).toBe(false); + now++; await guardian.tick(); await guardian.drain(); + expect(repairs).toBe(1); + expect(actions).toBe(1); +}); + +test("same PID is not ownership proof when the process identity changes", async () => { + const f = await fixture(); + let now = 100000, owned = true, start = "old-ticks"; + await atomicJson(f.intent, { version: 1, mode: "running", at: now }); + const guardian = await createGuardian(f.config, { now: () => now, policyOptions: { recoveryStableMs: 1000 }, alive: () => true, + inspect: async () => ({ ok: true, value: { owned, pid: 42, start, launcherPid: 43, launcherStart: "ticks" } }), + probe: async () => ({ ok: true, pid: 42 }) }); + cleanup.push(guardian.close); + await guardian.tick(); now += 1000; await guardian.tick(); + expect(guardian.state().primaryReady).toBe(true); + owned = false; start = "foreign-ticks"; now += 1000; + await guardian.tick(); + expect(guardian.state().primaryReady).toBe(false); + expect(guardian.state().state).toBe("foreign"); + owned = true; start = "new-owned-ticks"; now += 1000; + await guardian.tick(); + expect(guardian.state().primaryReady).toBe(false); + now += 1000; await guardian.tick(); + expect(guardian.state().primaryReady).toBe(true); +}); + +test("manual stop during asynchronous diagnosis preparation prevents external dispatch", async () => { + const f = await fixture(); + let now = 100000, repairs = 0; + await atomicJson(f.intent, { version: 1, mode: "running", at: now }); + const guardian = await createGuardian(f.config, { now: () => now, policyOptions: { startupGraceMs: 0 }, alive: () => true, + inspect: async () => ({ ok: true, value: { owned: true, pid: 42, start: "ticks", launcherPid: 43, launcherStart: "ticks" } }), + probe: async () => ({ ok: false, pid: 42 }), + action: async () => ({ ok: true, value: { action: "refused", reason: "stop-uncertain" } }), + beforeRepairDispatch: async () => atomicJson(f.intent, { version: 1, mode: "stopped", at: ++now }), + repair: async () => { repairs++; return { outcome: "no_candidate" }; } }); + cleanup.push(guardian.close); + for (let n = 0; n < 12; n++) { now += 2000; await guardian.tick(); await guardian.drain(); } + expect(repairs).toBe(0); + expect(guardian.state().primaryReady).toBe(false); +}); + +test("manual stop aborts in-flight GLM work instead of starting its fallback", async () => { + const f = await fixture(); + let now = 100000, dispatched = false, cancelled = false; + await atomicJson(f.intent, { version: 1, mode: "running", at: now }); + const guardian = await createGuardian(f.config, { now: () => now, policyOptions: { startupGraceMs: 0 }, alive: () => true, + inspect: async () => ({ ok: true, value: { owned: true, pid: 42, start: "ticks", launcherPid: 43, launcherStart: "ticks" } }), + probe: async () => ({ ok: false, pid: 42 }), + action: async () => ({ ok: true, value: { action: "refused", reason: "stop-uncertain" } }), + repair: async ({ signal }: { signal: AbortSignal }) => { + dispatched = true; + if (!signal) return { outcome: "failed", failureClass: "MISSING_SIGNAL" }; + return new Promise(resolve => signal.addEventListener("abort", () => { + cancelled = true; resolve({ outcome: "failed", failureClass: "CANCELLED" }); + }, { once: true })); + } }); + cleanup.push(guardian.close); + for (let n = 0; n < 13 && !dispatched; n++) { + now += 2000; await guardian.tick(); await new Promise(resolve => setTimeout(resolve, 20)); + } + expect(dispatched).toBe(true); + await atomicJson(f.intent, { version: 1, mode: "stopped", at: ++now }); + await guardian.tick(); await guardian.drain(); + expect(cancelled).toBe(true); + expect(guardian.state().lastRepair.failureClass).toBe("CANCELLED"); +}); + +test("observation errors withdraw primary admission", async () => { + const f = await fixture(); + let now = 100000, fail = false; + await atomicJson(f.intent, { version: 1, mode: "running", at: now }); + const guardian = await createGuardian(f.config, { now: () => now, policyOptions: { recoveryStableMs: 1000 }, alive: () => true, + inspect: async () => { if (fail) throw Error("local inspection failure"); return { ok: true, value: { owned: true, pid: 42, start: "ticks", launcherPid: 43, launcherStart: "ticks" } }; }, + probe: async () => ({ ok: true, pid: 42 }) }); + cleanup.push(guardian.close); + await guardian.tick(); now += 1000; await guardian.tick(); + expect(guardian.state().primaryReady).toBe(true); + fail = true; now += 1000; + await expect(guardian.tick()).rejects.toThrow("local inspection failure"); + expect(guardian.state().primaryReady).toBe(false); +}); + +test("companion restart cannot replay a previously in-flight recovery command", async () => { + const f = await fixture(); + const now = 100000; + await atomicJson(f.intent, { version: 1, mode: "running", at: now - 1000 }); + const prior = new RecoveryPolicy(); + prior.markRecoveryStarted(now - 500); + await atomicJson(join(f.home, "recovery-budget.json"), prior.exportSafeState(now)); + let actions = 0; + const guardian = await createGuardian(f.config, { now: () => now, alive: () => true, + inspect: async () => ({ ok: true, value: { owned: true, pid: 42, start: "ticks", launcherPid: 43, launcherStart: "ticks" } }), + probe: async () => ({ ok: false, pid: 42 }), + action: async () => { actions++; return { ok: true, value: { action: "started" } }; }, + repair: async () => ({ outcome: "no_candidate", candidateCount: 0, requestCount: 1 }) }); + cleanup.push(guardian.close); + await guardian.tick(); await guardian.drain(); + expect(guardian.state().recoveryBlocked).toBe(true); + expect(actions).toBe(0); +}); + +test("recovered readiness needs stable interval before main gateway uses primary again", async () => { + const f = await fixture(); + let now = 100000, healthy = true; + await atomicJson(f.intent, { version: 1, mode: "running", at: now }); + const guardian = await createGuardian(f.config, { now: () => now, policyOptions: { startupGraceMs: 1000, recoveryStableMs: 2000 }, alive: () => true, + inspect: async () => ({ ok: true, value: { owned: true, pid: 42, start: "ticks", launcherPid: 43, launcherStart: "ticks" } }), + probe: async () => ({ ok: healthy, pid: 42 }), + action: async () => ({ ok: true, value: { action: "started" } }) }); + cleanup.push(guardian.close); + await guardian.tick(); healthy = false; + for (let n = 0; n < 11; n++) { now += 2000; await guardian.tick(); await guardian.drain(); } + healthy = true; now += 1000; await guardian.tick(); + expect(guardian.state().primaryReady).toBe(false); + now += 2000; await guardian.tick(); + expect(guardian.state().primaryReady).toBe(true); +}); diff --git a/tests/windows/windows-recovery-ownership.test.ts b/tests/windows/windows-recovery-ownership.test.ts new file mode 100644 index 00000000000..45c6d3b0236 --- /dev/null +++ b/tests/windows/windows-recovery-ownership.test.ts @@ -0,0 +1,213 @@ +import { describe, expect, test } from "bun:test"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { repoPath } from "../helpers/repo-root"; + +const actionPath = repoPath("scripts/ocx-recovery-guardian/windows-action.ps1"); + +function actionSource() { + return Bun.file(actionPath).text(); +} + +function makeFixture() { + const root = mkdtempSync(join(tmpdir(), "ocx-recovery-windows-")); + const project = join(root, "project"); + const home = join(root, "home"); + const codex = join(root, "codex"); + mkdirSync(join(project, "node_modules", "bun", "bin"), { recursive: true }); + mkdirSync(join(project, "src", "cli"), { recursive: true }); + mkdirSync(join(project, "scripts"), { recursive: true }); + mkdirSync(home, { recursive: true }); + mkdirSync(codex, { recursive: true }); + writeFileSync(join(project, "node_modules", "bun", "bin", "bun.exe"), "fixture only"); + writeFileSync(join(project, "src", "cli", "index.ts"), "// fixture only\n"); + writeFileSync(join(project, "scripts", "windows-visible-proxy.ps1"), "# fixture only\n"); + return { root, project, home, codex }; +} + +describe("Windows recovery guardian action ownership", () => { + test("has a narrow structured interface and never serializes command lines", async () => { + const source = await actionSource(); + expect(source).toContain("[ValidateSet('Inspect', 'Recover')]"); + expect(source).toContain("ExpectedLauncherPid"); + expect(source).toContain("ExpectedLauncherStart"); + expect(source).toContain("ConvertTo-Json -Compress"); + expect(source).not.toMatch(/commandLine\s*=/i); + expect(source).not.toMatch(/Write-(Host|Verbose|Warning|Error).*CommandLine/i); + expect(source).toContain("if ($Mode -eq 'Recover')"); + expect(source).toContain("invalid-expected-identity"); + }); + + test("requires the exact Bun CLI and visible-launcher parent rather than Bun alone", async () => { + const source = await actionSource(); + expect(source).toContain("node_modules\\bun\\bin\\bun.exe"); + expect(source).toContain("src\\cli\\index.ts"); + expect(source).toContain("scripts\\windows-visible-proxy.ps1"); + expect(source).toContain("Get-CimInstance Win32_Process"); + expect(source).toContain("Get-CimInstance Win32_Process -Filter"); + expect(source).toContain("-OperationTimeoutSec 3"); + expect(source).toContain("Test-ExpectedIdentity"); + expect(source).toContain("Test-VisibleLauncherParent"); + expect(source).toContain("$Child.ParentProcessId -ne $ExpectedLauncherPid"); + expect(source).toContain("$CreationDate -is [DateTime]"); + expect(source).toContain("$CreationDate -is [DateTimeOffset]"); + expect(source).toContain("Get-StableCodexHome"); + expect(source).toContain("OcxGuardianCanonicalDirectory"); + expect(source).toContain("Test-OptionalPathArgument"); + expect(source).toContain("$launchArguments -join ' '"); + }); + + test("has no direct kill path and makes a foreign listener terminal before visible launch", async () => { + const source = await actionSource(); + expect(source).not.toMatch(/\b(taskkill|Stop-Process|Terminate\s*\()/i); + expect(source).toContain("foreign-listener"); + expect(source).toContain("port-closed"); + const collision = source.indexOf("foreign-listener"); + const start = source.lastIndexOf("Start-VisibleLauncher -ScriptPath"); + expect(collision).toBeGreaterThan(-1); + expect(start).toBeGreaterThan(collision); + }); + + test("refuses malformed manual intent in an isolated fake project without invoking a child", () => { + if (process.platform !== "win32") return; + const fixture = makeFixture(); + try { + writeFileSync(join(fixture.home, "recovery-intent.json"), "{ not-json"); + const run = Bun.spawnSync([ + join(process.env.SystemRoot ?? "C:\\Windows", "System32", "WindowsPowerShell", "v1.0", "powershell.exe"), + "-NoProfile", "-NonInteractive", "-File", actionPath, + "-Mode", "Recover", "-ProjectRoot", fixture.project, + "-OpenCodexHome", fixture.home, "-CodexHome", fixture.codex, + "-Port", "18991", "-ExpectedPid", "424242", "-ExpectedStart", "1", + "-ExpectedLauncherPid", "424243", "-ExpectedLauncherStart", "1", + ], { stdout: "pipe", stderr: "pipe", timeout: 15_000 }); + expect(run.exitCode, Buffer.from(run.stderr).toString()).toBe(0); + const result = JSON.parse(Buffer.from(run.stdout).toString()) as { action: string; reason: string }; + expect(result).toMatchObject({ action: "refused", reason: "invalid-intent" }); + } finally { + rmSync(fixture.root, { recursive: true, force: true }); + } + }, 20_000); + + test("uses the shared version/at intent envelope rather than a divergent schema", async () => { + const source = await actionSource(); + expect(source).toContain("$intent.version"); + expect(source).toContain("$intent.at"); + expect(source).not.toContain("$intent.schema"); + expect(source).toContain("$intent.until -le $intent.at"); + expect(source).toContain("$intent.PSObject.Properties['until']"); + }); + + test("accepts a running intent without until under StrictMode but rejects a maintenance intent without one", () => { + if (process.platform !== "win32") return; + const source = readFileSync(actionPath, "utf8"); + const start = source.indexOf("function Read-RecoveryIntent"); + const end = source.indexOf("function Test-CurrentRunningIntent", start); + expect(start).toBeGreaterThan(-1); + expect(end).toBeGreaterThan(start); + const directory = mkdtempSync(join(tmpdir(), "ocx-recovery-intent-schema-")); + const psFile = join(directory, "intent.ps1"); + const home = join(directory, "home"); + mkdirSync(home); + const quote = (value: string) => `'${value.replaceAll("'", "''")}'`; + writeFileSync(psFile, `$ErrorActionPreference='Stop'\nSet-StrictMode -Version Latest\n$IntentMaxBytes=16384\nfunction Read-BoundedJson { param([string]$Path,[int]$MaximumBytes) Get-Content -LiteralPath $Path -Raw | ConvertFrom-Json }\n${source.slice(start, end)}\n$intentHomeFixture=${quote(home)}\nSet-Content -LiteralPath (Join-Path $intentHomeFixture 'recovery-intent.json') -NoNewline -Value '{"version":1,"mode":"running","at":100}'\n$running = Read-RecoveryIntent -OpenCodexDirectory $intentHomeFixture\nSet-Content -LiteralPath (Join-Path $intentHomeFixture 'recovery-intent.json') -NoNewline -Value '{"version":1,"mode":"maintenance","at":100}'\n$maintenance = Read-RecoveryIntent -OpenCodexDirectory $intentHomeFixture\n@($running.valid,$running.reason,$maintenance.valid,$maintenance.reason) | ConvertTo-Json -Compress\n`); + try { + const ps = join(process.env.SystemRoot ?? "C:\\Windows", "System32", "WindowsPowerShell", "v1.0", "powershell.exe"); + const run = Bun.spawnSync([ps, "-NoProfile", "-NonInteractive", "-File", psFile], { stdout: "pipe", stderr: "pipe", timeout: 15_000 }); + expect(run.exitCode, Buffer.from(run.stderr).toString()).toBe(0); + expect(JSON.parse(Buffer.from(run.stdout).toString())).toEqual([true, "allowed", false, "maintenance-expired"]); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }, 20_000); + + test("revalidates the same running intent generation at both irreversible action edges", async () => { + const source = await actionSource(); + expect(source).toContain("function Test-RecoveryBoundary"); + expect(source).toContain("function Test-CurrentRunningIntent"); + expect(source).toContain("intent-changed"); + expect(source).toContain("intent-not-running"); + const beforeStop = source.indexOf("$beforeStopBoundary = Test-RecoveryBoundary"); + const stop = source.indexOf("$stopStatus = Invoke-GracefulProjectStop"); + const afterStop = source.indexOf("$afterStopBoundary = Test-RecoveryBoundary"); + const beforeStart = source.indexOf("$beforeStartBoundary = Test-RecoveryBoundary"); + const start = source.indexOf("Start-VisibleLauncher -ScriptPath"); + expect(beforeStop).toBeGreaterThan(-1); + expect(stop).toBeGreaterThan(beforeStop); + expect(afterStop).toBeGreaterThan(stop); + expect(beforeStart).toBeGreaterThan(afterStop); + expect(start).toBeGreaterThan(beforeStart); + expect(source.slice(beforeStop, stop)).toContain("-ExpectedIntentAt $intentAt"); + expect(source.slice(afterStop, beforeStart)).toContain("-ExpectedIntentAt $intentAt"); + expect(source.slice(beforeStart, start)).toContain("-ExpectedIntentAt $intentAt"); + }); + + test("refuses deterministic stop or maintenance races after the recovery operation has begun", () => { + if (process.platform !== "win32") return; + const source = readFileSync(actionPath, "utf8"); + const start = source.indexOf("function New-RecoveryBoundaryResult"); + const end = source.indexOf("function Initialize-DiscardDrainType", start); + expect(start).toBeGreaterThan(-1); + expect(end).toBeGreaterThan(start); + const boundary = source.slice(start, end); + const psDirectory = mkdtempSync(join(tmpdir(), "ocx-recovery-boundary-")); + const psFile = join(psDirectory, "boundary.ps1"); + const sample = "[pscustomobject]@{ owned=$false; alive=$false; listenerPid=0; pid=41; start='41'; launcherPid=42; launcherStart='42'; launcherAlive=$true; launcherOwned=$true }"; + writeFileSync(psFile, `$ErrorActionPreference='Stop'\n${boundary}\n$global:sample = ${sample}\nfunction Get-OwnershipSnapshot { $global:sample }\nfunction Same-Snapshot { param($Left,$Right) $true }\nfunction Test-PortClosedTwice { $true }\nfunction Test-CurrentRunningIntent { param([string]$OpenCodexDirectory,[long]$ExpectedAt) $global:intentCalls += 1; if ($global:intentCalls -eq 1) { return [pscustomobject]@{ valid=$true; reason='allowed'; mode='running'; at=$ExpectedAt } }; if ($global:race -eq 'stopped') { return [pscustomobject]@{ valid=$false; reason='manual-stop'; mode='stopped'; at=$ExpectedAt + 1 } }; return [pscustomobject]@{ valid=$false; reason='intent-not-running'; mode='maintenance'; at=$ExpectedAt + 1 } }\n$results = @()\nforeach ($race in @('stopped','maintenance')) { $global:race=$race; $global:intentCalls=0; $result = Test-RecoveryBoundary -Boundary before-start -ExpectedSnapshot $global:sample -BunPath 'b' -CliPath 'c' -ScriptPath 's' -Root 'r' -OpenCodexDirectory 'h' -CodexConfigured 'd' -CodexCanonical 'd' -ExpectedIntentAt 99; $results += $result.reason }\n$results | ConvertTo-Json -Compress\n`); + try { + const ps = join(process.env.SystemRoot ?? "C:\\Windows", "System32", "WindowsPowerShell", "v1.0", "powershell.exe"); + const run = Bun.spawnSync([ps, "-NoProfile", "-NonInteractive", "-File", psFile], { stdout: "pipe", stderr: "pipe", timeout: 15_000 }); + expect(run.exitCode, Buffer.from(run.stderr).toString()).toBe(0); + expect(JSON.parse(Buffer.from(run.stdout).toString())).toEqual(["manual-stop", "intent-not-running"]); + } finally { + rmSync(psDirectory, { recursive: true, force: true }); + } + }, 20_000); + + test("does not claim the current Bun test process as owned when it is not the exact launcher tree", () => { + if (process.platform !== "win32") return; + const fixture = makeFixture(); + try { + const run = Bun.spawnSync([ + join(process.env.SystemRoot ?? "C:\\Windows", "System32", "WindowsPowerShell", "v1.0", "powershell.exe"), + "-NoProfile", "-NonInteractive", "-File", actionPath, + "-Mode", "Inspect", "-ProjectRoot", repoPath(), + "-OpenCodexHome", fixture.home, "-CodexHome", fixture.codex, + "-Port", "18992", + ], { stdout: "pipe", stderr: "pipe", timeout: 15_000 }); + expect(run.exitCode, Buffer.from(run.stderr).toString()).toBe(0); + const result = JSON.parse(Buffer.from(run.stdout).toString()) as { owned: boolean }; + expect(result.owned).toBe(false); + expect(Buffer.from(run.stdout).toString()).not.toContain(process.execPath); + } finally { + rmSync(fixture.root, { recursive: true, force: true }); + } + }, 20_000); + + test("reports an isolated foreign listener but never treats it as a recovery target", async () => { + if (process.platform !== "win32") return; + const fixture = makeFixture(); + const server = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: () => new Response("fixture") }); + try { + await fetch(server.url); + const run = Bun.spawnSync([ + join(process.env.SystemRoot ?? "C:\\Windows", "System32", "WindowsPowerShell", "v1.0", "powershell.exe"), + "-NoProfile", "-NonInteractive", "-File", actionPath, + "-Mode", "Inspect", "-ProjectRoot", repoPath(), + "-OpenCodexHome", fixture.home, "-CodexHome", fixture.codex, + "-Port", String(server.port), + ], { stdout: "pipe", stderr: "pipe", timeout: 15_000 }); + expect(run.exitCode, Buffer.from(run.stderr).toString()).toBe(0); + const result = JSON.parse(Buffer.from(run.stdout).toString()) as { action: string; owned: boolean; listenerPid: number }; + expect(result).toMatchObject({ action: "inspect" }); + expect(result.owned).toBe(false); + expect(result.listenerPid).toBeGreaterThan(0); + } finally { + server.stop(true); + rmSync(fixture.root, { recursive: true, force: true }); + } + }, 20_000); + +}); diff --git a/tests/windows/windows-recovery-policy.test.ts b/tests/windows/windows-recovery-policy.test.ts new file mode 100644 index 00000000000..c7fe4df128e --- /dev/null +++ b/tests/windows/windows-recovery-policy.test.ts @@ -0,0 +1,163 @@ +import { expect, test } from "bun:test"; + +const { RecoveryPolicy } = require("../../scripts/ocx-recovery-guardian/policy.cjs") as { + RecoveryPolicy: new (options?: Record) => { + observe(sample: Record, now: number): Record; + markRecoveryStarted(now: number): Record; + markRecoveryFinished(result: { ok: boolean }, now: number): void; + exportSafeState(): Record; + importSafeState(value: unknown, now: number): boolean; + reset(options?: { resetBudget?: boolean; now?: number }): void; + }; +}; + +const healthy = { ready: true, health: true, alive: true, owned: true }; +const unavailable = { ready: false, health: false, alive: true, owned: true }; + +test("only the configured consecutive failure threshold enters fallback", () => { + const policy = new RecoveryPolicy({ startupGraceMs: 0 }); + + expect(policy.observe(healthy, 0)).toEqual({ state: "healthy", useFallback: false, action: "none", reason: "ready" }); + expect(policy.observe(unavailable, 2_000)).toMatchObject({ state: "suspect", useFallback: false, action: "none" }); + expect(policy.observe(unavailable, 4_000)).toMatchObject({ state: "suspect", useFallback: false, action: "none" }); + expect(policy.observe(unavailable, 6_000)).toMatchObject({ state: "fallback", useFallback: true, action: "none" }); +}); + +test("an owned dead child requests one immediate recovery, while a live freeze waits for the bounded failure duration", () => { + const dead = { ready: false, health: false, alive: false, owned: true }; + const policy = new RecoveryPolicy({ startupGraceMs: 0, recoverAfterMs: 20_000 }); + expect(policy.observe(healthy, 0)).toMatchObject({ state: "healthy" }); + expect(policy.observe(dead, 2_000)).toMatchObject({ state: "fallback", useFallback: true, action: "recover", reason: "owned-child-dead" }); + + const frozen = new RecoveryPolicy({ startupGraceMs: 0, recoverAfterMs: 20_000 }); + frozen.observe(healthy, 0); + expect(frozen.observe(unavailable, 2_000)).toMatchObject({ action: "none" }); + expect(frozen.observe(unavailable, 20_000)).toMatchObject({ action: "none" }); + expect(frozen.observe(unavailable, 22_000)).toMatchObject({ state: "fallback", action: "recover", reason: "failure-duration" }); +}); + +test("startup grace and short 503 blips do not request recovery", () => { + const policy = new RecoveryPolicy({ startupGraceMs: 45_000 }); + expect(policy.observe(unavailable, 0)).toMatchObject({ state: "suspect", useFallback: false, action: "none", reason: "startup-grace" }); + expect(policy.observe({ ready: true, health: false, alive: true, owned: true }, 2_000)).toMatchObject({ state: "suspect", action: "none" }); + expect(policy.observe(unavailable, 44_000)).toMatchObject({ state: "suspect", useFallback: false, action: "none" }); + expect(policy.observe(healthy, 45_000)).toMatchObject({ state: "fallback", useFallback: true, action: "none" }); +}); + +test("manual stop, dead launcher, foreign identity, and unknown ownership never self-respawn", () => { + const stopped = new RecoveryPolicy({ startupGraceMs: 0 }); + expect(stopped.observe({ ...unavailable, manualStop: true }, 0)).toMatchObject({ state: "stopped", useFallback: false, action: "none" }); + expect(stopped.observe(unavailable, 60_000)).toMatchObject({ state: "stopped", action: "none" }); + + const launcherGone = new RecoveryPolicy({ startupGraceMs: 0 }); + expect(launcherGone.observe({ ...unavailable, launcherAlive: false }, 0)).toMatchObject({ state: "stopped", action: "none" }); + + const foreign = new RecoveryPolicy({ startupGraceMs: 0 }); + expect(foreign.observe({ ...unavailable, owned: false, identityChanged: true }, 0)).toMatchObject({ state: "foreign", useFallback: true, action: "none" }); + expect(foreign.observe({ ...unavailable, owned: false }, 60_000)).toMatchObject({ state: "foreign", action: "none" }); + + const unknown = new RecoveryPolicy({ startupGraceMs: 0 }); + expect(unknown.observe({ ready: false, health: false, alive: false, owned: false }, 0)).toMatchObject({ state: "foreign", action: "none" }); +}); + +test("recovery success requires thirty seconds of readiness before fallback returns to healthy", () => { + const policy = new RecoveryPolicy({ startupGraceMs: 0, recoveryStableMs: 30_000 }); + expect(policy.observe({ ready: false, health: false, alive: false, owned: true }, 0)).toMatchObject({ action: "recover" }); + policy.markRecoveryStarted(0); + policy.markRecoveryFinished({ ok: true }, 1_000); + expect(policy.observe(healthy, 1_000)).toMatchObject({ state: "recovering", useFallback: true, action: "none" }); + expect(policy.observe(healthy, 30_999)).toMatchObject({ state: "recovering", useFallback: true }); + expect(policy.observe(healthy, 31_000)).toMatchObject({ state: "healthy", useFallback: false, action: "none" }); +}); + +test("a recovered child may use startup grace before its full stable-ready interval under a bounded deadline", () => { + const policy = new RecoveryPolicy({ startupGraceMs: 45_000, recoveryStableMs: 30_000 }); + const dead = { ready: false, health: false, alive: false, owned: true }; + expect(policy.observe(dead, 0)).toMatchObject({ action: "recover" }); + policy.markRecoveryStarted(0); + policy.markRecoveryFinished({ ok: true }, 1); + expect(policy.observe(healthy, 45_001)).toMatchObject({ state: "recovering", useFallback: true, action: "none" }); + expect(policy.observe(healthy, 75_001)).toMatchObject({ state: "healthy", useFallback: false, action: "none" }); +}); + +test("an accepted recovery receipt that never becomes ready diagnoses immediately without consuming another recovery attempt", () => { + const policy = new RecoveryPolicy({ startupGraceMs: 45_000, recoveryStableMs: 30_000, maxAttempts: 2 }); + expect(policy.markRecoveryStarted(0)).toMatchObject({ state: "recovering", reason: "recovery-started" }); + expect(policy.markRecoveryFinished({ ok: true }, 0)).toMatchObject({ state: "recovering", reason: "awaiting-stable-ready" }); + + expect(policy.observe(unavailable, 74_999)).toMatchObject({ state: "recovering", action: "none" }); + expect(policy.observe(unavailable, 75_000)).toEqual({ + state: "failed", useFallback: true, action: "diagnose", reason: "recovery-not-stable", + }); + expect(policy.exportSafeState()).toMatchObject({ + attempts: [0], awaitingReady: false, recoveryStartedAt: null, recoveryDeadline: null, lastFailedAttemptAt: 75_000, + }); + expect(policy.observe(unavailable, 75_001)).toEqual({ + state: "suspect", useFallback: false, action: "none", reason: "transient-failure", + }); +}); + +test("manual stop or unknown ownership wins over an awaiting-ready expiry", () => { + for (const sample of [ + { ...unavailable, manualStop: true }, + { ...unavailable, owned: false }, + ]) { + const policy = new RecoveryPolicy({ startupGraceMs: 45_000, recoveryStableMs: 30_000 }); + policy.markRecoveryStarted(0); + policy.markRecoveryFinished({ ok: true }, 0); + expect(policy.observe(sample, 75_000)).toMatchObject({ + state: sample.manualStop ? "stopped" : "foreign", action: "none", + }); + } +}); + +test("a timed-out recovery clears its in-progress latch so the bounded retry can be decided", () => { + const policy = new RecoveryPolicy({ startupGraceMs: 0, recoverAfterMs: 1_000 }); + const dead = { ready: false, health: false, alive: false, owned: true }; + expect(policy.observe(dead, 0)).toMatchObject({ action: "recover" }); + policy.markRecoveryStarted(0); + expect(policy.observe(dead, 1_000)).toMatchObject({ state: "failed", action: "none", reason: "recovery-timeout" }); + expect(policy.observe(dead, 6_000)).toMatchObject({ state: "fallback", action: "recover", reason: "owned-child-dead" }); +}); + +test("failed recoveries use bounded backoff, retain the rolling budget, and diagnose once when exhausted", () => { + const policy = new RecoveryPolicy({ startupGraceMs: 0, maxAttempts: 2, recoverAfterMs: 20_000 }); + const dead = { ready: false, health: false, alive: false, owned: true }; + expect(policy.observe(dead, 0)).toMatchObject({ action: "recover" }); + policy.markRecoveryStarted(0); + policy.markRecoveryFinished({ ok: false }, 1); + expect(policy.observe(dead, 2_000)).toMatchObject({ action: "none", reason: "recovery-backoff" }); + expect(policy.observe(dead, 5_001)).toMatchObject({ action: "recover" }); + policy.markRecoveryStarted(5_001); + expect(policy.markRecoveryFinished({ ok: false }, 5_002)).toMatchObject({ state: "failed", action: "diagnose", reason: "attempt-budget-exhausted" }); + expect(policy.observe(dead, 20_002)).toMatchObject({ state: "failed", action: "none", reason: "attempt-budget-exhausted" }); + expect(policy.observe(dead, 22_002)).toMatchObject({ state: "failed", action: "none", reason: "attempt-budget-exhausted" }); + policy.reset({ now: 30_000 }); + expect(policy.observe(dead, 30_000)).toMatchObject({ state: "failed", action: "none" }); + policy.reset({ resetBudget: true, now: 30_000 }); + expect(policy.observe(dead, 30_000)).toMatchObject({ action: "recover" }); +}); + +test("safe restart state retains a bounded attempt budget and invalid state fails closed", () => { + const original = new RecoveryPolicy({ startupGraceMs: 0 }); + const dead = { ready: false, health: false, alive: false, owned: true }; + original.observe(dead, 0); + original.markRecoveryStarted(0); + original.markRecoveryFinished({ ok: false }, 1); + const restored = new RecoveryPolicy({ startupGraceMs: 0 }); + expect(restored.importSafeState(original.exportSafeState(), 2_000)).toBe(true); + expect(restored.observe(dead, 2_000)).toMatchObject({ action: "none", reason: "recovery-backoff" }); + const invalid = new RecoveryPolicy({ startupGraceMs: 0 }); + expect(invalid.importSafeState({ attempts: ["bad"] }, 0)).toBe(false); + expect(invalid.observe(dead, 60_000)).toMatchObject({ state: "foreign", action: "none" }); +}); + +test("attempt budget is released only when its rolling window expires", () => { + const policy = new RecoveryPolicy({ startupGraceMs: 0, maxAttempts: 1, attemptWindowMs: 10_000 }); + const dead = { ready: false, health: false, alive: false, owned: true }; + expect(policy.observe(dead, 0)).toMatchObject({ action: "recover" }); + policy.markRecoveryStarted(0); + expect(policy.markRecoveryFinished({ ok: false }, 1)).toMatchObject({ action: "diagnose" }); + expect(policy.observe(dead, 10_000)).toMatchObject({ action: "none", reason: "attempt-budget-exhausted" }); + expect(policy.observe(dead, 10_001)).toMatchObject({ action: "recover" }); +}); diff --git a/tests/windows/windows-recovery-repair.test.ts b/tests/windows/windows-recovery-repair.test.ts new file mode 100644 index 00000000000..b3e65738832 --- /dev/null +++ b/tests/windows/windows-recovery-repair.test.ts @@ -0,0 +1,253 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { tmpdir } from "node:os"; +import { repoPath } from "../helpers/repo-root"; + +const { runRepair } = require(repoPath("scripts", "ocx-recovery-guardian", "repair.cjs")) as { + runRepair: (input: Record) => Promise>; +}; + +const ALLOWLIST = [ + "src/lib/runtime-diagnostics.ts", + "src/lib/runtime-diagnostics-child.ts", + "src/responses/state.ts", + "src/codex/user-identity.ts", + "scripts/windows-visible-proxy.ps1", + "src/tray/windows-tray.ps1", +]; +const homes: string[] = []; + +function fixture(source = "export const value = 1;\n") { + const home = join(tmpdir(), `ocx-repair-${crypto.randomUUID()}`); + const projectRoot = join(home, "project"); + const incidentDir = join(home, "incident"); + mkdirSync(projectRoot, { recursive: true }); + mkdirSync(incidentDir, { recursive: true }); + for (const relativePath of ALLOWLIST) { + const target = join(projectRoot, ...relativePath.split("/")); + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, relativePath.endsWith(".ps1") ? "$value = 1\n" : source); + } + homes.push(home); + return { projectRoot, incidentDir }; +} + +function response(output: unknown) { + return async () => new Response(JSON.stringify({ choices: [{ message: { content: JSON.stringify(output) } }] }), { status: 200 }); +} + +function request(input: { projectRoot: string; incidentDir: string; fetchFn: typeof fetch }) { + return runRepair({ + incident: { reason: "event_loop_delay", healthReady: false, attempts: 2, pid: 86488, timing: { delayMs: 7764 }, rawLog: "must-not-leave" }, + projectRoot: input.projectRoot, + incidentDir: input.incidentDir, + endpoint: "http://127.0.0.1:20128/v1", + readKey: async () => "test-key", + fetchFn: input.fetchFn, + }); +} + +afterEach(() => { + while (homes.length) rmSync(homes.pop()!, { recursive: true, force: true }); +}); + +describe("ocx recovery guardian isolated GLM repair", () => { + test("sends only sanitized bounded material and writes a review candidate, never production", async () => { + const paths = fixture(); + let captured: { url: string; init: RequestInit } | null = null; + const fetchFn: typeof fetch = async (url, init) => { + captured = { url: String(url), init: init! }; + return response({ diagnosis: "bounded candidate", patches: [{ path: "src/lib/runtime-diagnostics.ts", find: "value = 1", replace: "value = 2" }] })(url, init); + }; + const result = await request({ ...paths, fetchFn }); + + expect(result).toMatchObject({ outcome: "candidate_ready", model: "glm-5.3-flash", requestCount: 1, candidateCount: 1 }); + expect(captured?.url).toBe("http://127.0.0.1:20128/v1/chat/completions"); + const body = JSON.parse(String(captured?.init.body)); + expect(body).toMatchObject({ model: "ollama-local/glm-5.3-flash:cloud", stream: false, max_tokens: 4096, temperature: 0, response_format: { type: "json_object" } }); + expect(captured?.init.redirect).toBe("error"); + expect(JSON.stringify(body)).not.toContain("must-not-leave"); + expect(JSON.stringify(body)).not.toContain(paths.incidentDir); + expect(JSON.stringify(body)).not.toContain(paths.projectRoot); + expect(readFileSync(join(paths.projectRoot, "src", "lib", "runtime-diagnostics.ts"), "utf8")).toBe("export const value = 1;\n"); + expect(readFileSync(join(paths.incidentDir, "candidate", "src", "lib", "runtime-diagnostics.ts"), "utf8")).toContain("value = 2"); + const metadata = JSON.parse(readFileSync(join(paths.incidentDir, "candidate", "metadata.json"), "utf8")); + expect(JSON.stringify(metadata)).not.toContain("value = 2"); + expect(metadata.patches[0]).toMatchObject({ path: "src/lib/runtime-diagnostics.ts", patchStatus: expect.any(String) }); + }); + + test("rejects a path outside the six-file allowlist", async () => { + const paths = fixture(); + const result = await request({ ...paths, fetchFn: response({ diagnosis: "bad path", patches: [{ path: "src/server/index.ts", find: "x", replace: "y" }] }) }); + expect(result).toMatchObject({ outcome: "failed", failureClass: "MODEL_OUTPUT_INVALID" }); + expect(existsSync(join(paths.incidentDir, "candidate"))).toBeFalse(); + }); + + test("rejects an ambiguous exact find and keeps production intact", async () => { + const paths = fixture("export const value = 1;\nexport const another = 1;\n"); + const result = await request({ ...paths, fetchFn: response({ diagnosis: "ambiguous", patches: [{ path: "src/lib/runtime-diagnostics.ts", find: " = 1", replace: " = 2" }] }) }); + expect(result).toMatchObject({ outcome: "failed", failureClass: "MODEL_OUTPUT_INVALID" }); + expect(readFileSync(join(paths.projectRoot, "src", "lib", "runtime-diagnostics.ts"), "utf8")).toContain("another = 1"); + }); + + test("rejects credential-like model echoes without creating candidates", async () => { + const paths = fixture(); + const result = await request({ ...paths, fetchFn: response({ diagnosis: "api_key=do-not-store", patches: [] }) }); + expect(result).toMatchObject({ outcome: "failed", failureClass: "MODEL_OUTPUT_INVALID" }); + expect(existsSync(join(paths.incidentDir, "candidate"))).toBeFalse(); + }); + + test("classifies network failure without a disk artifact", async () => { + const paths = fixture(); + const fetchFn: typeof fetch = async () => { throw new TypeError("offline"); }; + const result = await request({ ...paths, fetchFn }); + expect(result).toMatchObject({ outcome: "failed", failureClass: "NETWORK", requestCount: 1 }); + expect(existsSync(join(paths.incidentDir, "candidate"))).toBeFalse(); + }); + + test("uses no more than one root-supplied fallback endpoint with an independent key", async () => { + const paths = fixture(); + const urls: string[] = []; + const authorizations: string[] = []; + const fetchFn: typeof fetch = async (url, init) => { + urls.push(String(url)); + authorizations.push(String((init?.headers as Record).authorization)); + if (urls.length === 1) throw new TypeError("offline"); + return response({ diagnosis: "fallback candidate", patches: [{ path: "src/lib/runtime-diagnostics.ts", find: "value = 1", replace: "value = 2" }] })(url, init); + }; + const result = await runRepair({ + incident: { reason: "event_loop_delay", healthReady: false, attempts: 1, timing: { delayMs: 1 } }, + projectRoot: paths.projectRoot, + incidentDir: paths.incidentDir, + endpoint: "http://127.0.0.1:20128/v1", + fallbackEndpoint: "https://api.mnnai.ru/v1", + readKey: async () => "primary-key", + readFallbackKey: async () => "fallback-key", + fetchFn, + }); + expect(result).toMatchObject({ outcome: "candidate_ready", requestCount: 2 }); + expect(urls).toEqual(["http://127.0.0.1:20128/v1/chat/completions", "https://api.mnnai.ru/v1/chat/completions"]); + expect(authorizations).toEqual(["Bearer primary-key", "Bearer fallback-key"]); + }); + + test("rejects a fallback endpoint when no independent fallback key reader is supplied", async () => { + const paths = fixture(); + const result = await runRepair({ + incident: { reason: "event_loop_delay", healthReady: false, attempts: 1, timing: { delayMs: 1 } }, + projectRoot: paths.projectRoot, + incidentDir: paths.incidentDir, + endpoint: "http://127.0.0.1:20128/v1", + fallbackEndpoint: "https://api.mnnai.ru/v1", + readKey: async () => "primary-key", + fetchFn: response({ diagnosis: "not called", patches: [] }), + }); + expect(result).toMatchObject({ outcome: "failed", failureClass: "INPUT_INVALID", requestCount: 0 }); + }); + + test("forwards caller cancellation to an in-flight request and never starts fallback", async () => { + const paths = fixture(); + const aborter = new AbortController(); + let fetchCalls = 0; + let fallbackKeyReads = 0; + let started!: () => void; + const requestStarted = new Promise(resolve => { started = resolve; }); + const fetchFn: typeof fetch = async (_url, init) => new Promise((_resolve, reject) => { + fetchCalls += 1; + started(); + init?.signal?.addEventListener("abort", () => reject(Object.assign(new Error("stopped"), { name: "AbortError" })), { once: true }); + }); + const pending = runRepair({ + incident: { reason: "event_loop_delay", healthReady: false, attempts: 1, timing: { delayMs: 1 } }, + projectRoot: paths.projectRoot, + incidentDir: paths.incidentDir, + endpoint: "http://127.0.0.1:11434/v1", + fallbackEndpoint: "http://127.0.0.1:20128/v1", + readKey: async () => "ollama-placeholder", + readFallbackKey: async () => { fallbackKeyReads += 1; return "fallback-key"; }, + fetchFn, + signal: aborter.signal, + }); + await requestStarted; + aborter.abort(); + await expect(pending).resolves.toMatchObject({ outcome: "cancelled", failureClass: "CANCELLED", requestCount: 1 }); + expect(fetchCalls).toBe(1); + expect(fallbackKeyReads).toBe(0); + }); + + test("does not dispatch when cancellation races during readKey", async () => { + const paths = fixture(); + const aborter = new AbortController(); + let fetchCalls = 0; + const result = await runRepair({ + incident: { reason: "event_loop_delay", healthReady: false, attempts: 1, timing: { delayMs: 1 } }, + projectRoot: paths.projectRoot, + incidentDir: paths.incidentDir, + endpoint: "http://127.0.0.1:11434/v1", + readKey: async () => { aborter.abort(); return "never-dispatch"; }, + fetchFn: async () => { fetchCalls += 1; return response({ diagnosis: "not called", patches: [] })(); }, + signal: aborter.signal, + }); + expect(result).toMatchObject({ outcome: "cancelled", failureClass: "CANCELLED", requestCount: 0 }); + expect(fetchCalls).toBe(0); + }); + + test("returns the model diagnosis when no patch candidate is proposed", async () => { + const paths = fixture(); + const result = await request({ ...paths, fetchFn: response({ diagnosis: "insufficient bounded evidence", patches: [] }) }); + expect(result).toMatchObject({ outcome: "no_candidate", diagnosis: "insufficient bounded evidence", candidateCount: 0 }); + expect(existsSync(join(paths.incidentDir, "candidate"))).toBeFalse(); + }); + + test("uses the Ollama cloud wire model without changing the fixed logical model", async () => { + const paths = fixture(); + let wireModel: string | null = null; + const fetchFn: typeof fetch = async (_url, init) => { + wireModel = JSON.parse(String(init?.body)).model; + return response({ diagnosis: "Ollama candidate", patches: [{ path: "src/lib/runtime-diagnostics.ts", find: "value = 1", replace: "value = 2" }] })(); + }; + const result = await runRepair({ + incident: { reason: "event_loop_delay", healthReady: false, attempts: 1, timing: { delayMs: 1 } }, + projectRoot: paths.projectRoot, + incidentDir: paths.incidentDir, + endpoint: "http://127.0.0.1:11434/v1", + readKey: async () => "placeholder-only", + fetchFn, + }); + expect(result).toMatchObject({ outcome: "candidate_ready", model: "glm-5.3-flash", requestCount: 1 }); + expect(wireModel).toBe("glm-5.3-flash:cloud"); + }); + + test("uses the fixed loopback OpenRouter Ollama-local wire model without changing logical model", async () => { + const paths = fixture(); + let wireModel: string | null = null; + let requestUrl: string | null = null; + const fetchFn: typeof fetch = async (url, init) => { + requestUrl = String(url); + wireModel = JSON.parse(String(init?.body)).model; + return response({ diagnosis: "OpenRouter candidate", patches: [{ path: "src/lib/runtime-diagnostics.ts", find: "value = 1", replace: "value = 2" }] })(); + }; + const result = await runRepair({ + incident: { reason: "event_loop_delay", healthReady: false, attempts: 1, timing: { delayMs: 1 } }, + projectRoot: paths.projectRoot, + incidentDir: paths.incidentDir, + endpoint: "http://127.0.0.1:20128/v1", + readKey: async () => "connection-bound-key", + fetchFn, + }); + expect(result).toMatchObject({ outcome: "candidate_ready", model: "glm-5.3-flash", requestCount: 1 }); + expect(requestUrl).toBe("http://127.0.0.1:20128/v1/chat/completions"); + expect(wireModel).toBe("ollama-local/glm-5.3-flash:cloud"); + }); + + test("never overwrites a pre-existing candidate file", async () => { + const paths = fixture(); + const candidate = join(paths.incidentDir, "candidate", "src", "lib", "runtime-diagnostics.ts"); + mkdirSync(dirname(candidate), { recursive: true }); + writeFileSync(candidate, "keep-existing-candidate\n"); + const result = await request({ ...paths, fetchFn: response({ diagnosis: "candidate", patches: [{ path: "src/lib/runtime-diagnostics.ts", find: "value = 1", replace: "value = 2" }] }) }); + expect(result).toMatchObject({ outcome: "failed", failureClass: "CANDIDATE_WRITE" }); + expect(readFileSync(candidate, "utf8")).toBe("keep-existing-candidate\n"); + expect(readFileSync(join(paths.projectRoot, "src", "lib", "runtime-diagnostics.ts"), "utf8")).toBe("export const value = 1;\n"); + }); +}); diff --git a/tests/windows/windows-tray.test.ts b/tests/windows/windows-tray.test.ts index 5b15a2ae4d4..58ee6b67f57 100644 --- a/tests/windows/windows-tray.test.ts +++ b/tests/windows/windows-tray.test.ts @@ -368,7 +368,7 @@ describe("Windows tray packaging and command safety", () => { expect(cli).toContain("isProxyReplacement(previous, live)"); expect(cli).toContain("process.exitCode = result.ok ? 0 : 1"); expect(cli).toContain("waitForProxy(40_000)"); - expect(cli).toContain("await handleProxyRestart(() => handleTrayProxyStart(false))"); + expect(cli).toContain("await handleProxyRestart(() => handleTrayProxyStart(false, false))"); expect(cli).toContain("function detachedStartEnvironment()"); expect(cli).toContain("delete env.OCX_SERVICE"); expect(cli).not.toContain("OCX_KEEP_ROUTING"); From f9ad580276a8cf14138e1f98c5db667b0d631519 Mon Sep 17 00:00:00 2001 From: Codex Developer Date: Fri, 25 Sep 2026 02:07:00 +0800 Subject: [PATCH 4/7] fix(windows): make the manual-stop intent chain durable A guardian-driven recovery child must not be read as an operator's durable manual stop, and a stop that cannot write its intent must refuse loudly instead of leaving the proxy serving while the file claims `stopped`. - recovery-intent: fence `maintenance` with the shared `at < until <= at+180000` contract on every reader and writer (CLI, tray, launcher, action script), restore the durable intent verbatim when a later refusal rolls the stop back, and drop a re-affirmation of an intent that already reads `running`. - guardian: adopt the opening inspection instead of paying for a second ~5 s PowerShell spawn, decay launcher corroboration so a generation cannot freeze a stale pid+start pair in place, and re-inspect after a recover so the recovery path stops double-paying Inspect. - incidents: count only directories this process generated against the retention budget, and never walk a reparse point into `fs.rm(recursive)`. - gateway: hoist the per-request allowed-host set and local port out of the request path. - intent.ps1/launcher: parenthesise the reparse test (Windows PowerShell binds `-or` and `-and` left-to-right at equal precedence, so an unparenthesised guard lets a small symlink fail open) and refuse a malformed marker before dispatching any lifecycle action. Verification: focused Windows files re-run quiet and exclusive (recovery-main 18/0, recovery-intent green on a quiet box, policy, ownership, repair, tray), each new assertion mutation-proven to fail alone. `bun x tsc --noEmit` and `bun run privacy:scan` green. --- .../content/docs/reference/cli/lifecycle.md | 22 +++ scripts/ocx-recovery-guardian/gateway.cjs | 58 +++--- scripts/ocx-recovery-guardian/intent.ps1 | 18 +- scripts/ocx-recovery-guardian/main.cjs | 95 ++++++++-- scripts/ocx-recovery-guardian/policy.cjs | 2 +- scripts/ocx-recovery-guardian/repair.cjs | 23 ++- .../ocx-recovery-guardian/windows-action.ps1 | 32 +++- src/cli/index.ts | 78 ++++++-- src/lib/recovery-intent.ts | 79 ++++++-- src/server/background-lifecycle.ts | 46 +++-- src/server/management-api.ts | 34 ++-- src/tray/windows-tray.ps1 | 68 +++++-- src/tray/windows.ts | 36 +++- .../windows/windows-recovery-gateway.test.ts | 109 ++++++++--- tests/windows/windows-recovery-intent.test.ts | 171 ++++++++++++++++- tests/windows/windows-recovery-main.test.ts | 176 ++++++++++++++++++ .../windows-recovery-ownership.test.ts | 74 ++++++-- tests/windows/windows-recovery-policy.test.ts | 2 +- tests/windows/windows-recovery-repair.test.ts | 22 ++- .../windows-tray-restart-hardening.test.ts | 81 +++++++- tests/windows/windows-tray.test.ts | 124 +++++++++++- 21 files changed, 1154 insertions(+), 196 deletions(-) diff --git a/docs-site/src/content/docs/reference/cli/lifecycle.md b/docs-site/src/content/docs/reference/cli/lifecycle.md index 3473d3b19cc..c2ebaa2ea38 100644 --- a/docs-site/src/content/docs/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/reference/cli/lifecycle.md @@ -65,6 +65,28 @@ It does not enter the forced-stop fallback for a process already observed to hav receipt-backed deferral still leaves final restoration and receipt cleanup with the parent; failure to restore shared client configuration keeps the stop failed and its receipt outstanding. +On Windows, `ocx stop` can also record the durable manual-stop instruction for the recovery +guardian, but only when that guardian is opted in. The switch is the file +`$OPENCODEX_HOME/recovery-guardian.json`, and it is not a two-key flag: that file *is* the +guardian's whole configuration — `projectRoot`, `openCodexHome`, `codexHome`, `nodePath`, +`listenPort`, `primaryPort`, `fallback.models` and `repair` — validated field by field by both the +visible launcher and the guardian itself, which refuse to start the proxy unless it describes the +approved local companion. The lifecycle commands look only at `version` and `enabled`, so an +operator who writes a minimal `{"version": 1, "enabled": true}` marker gets intent bookkeeping +without a running guardian. No `ocx` command creates that file for you today; treat the guardian as +operator-managed. + +Once the marker is present, `ocx stop` writes `recovery-intent.json` as `stopped` so the guardian +does not bring the proxy back, while `ocx start` and `ocx ensure` affirm `running`, and a tray +restart signs a bounded `maintenance` window (`until` must be after `at` and no more than three +minutes out). Without the marker neither file is created, and a guardian-spawned recovery child +writes no `stopped` intent of its own. A missing or malformed intent is decoded as `stopped`, and +while a home reads as stopped the guardian's gateway answers `/readyz` and every proxied route with +`503` — only `/healthz` keeps answering, reporting `primaryReady: false` — fail-closed on purpose. +When the stop cannot record that intent, nothing is dispatched: `ocx stop` prints +`❌ Stop refused: ` and exits 1, and the dashboard's stop route answers +`503 recovery_intent_unavailable` — rather than reporting a stop the guardian will immediately undo. + `ocx stop --json` runs exactly the same stop path and prints one versioned summary document (`schema: "ocx-stop/1"`) on stdout, while the human progress lines move to stderr. The summary carries the outcome class (`stopped`, `not-running`, `history-incomplete`, `history-deferred`, diff --git a/scripts/ocx-recovery-guardian/gateway.cjs b/scripts/ocx-recovery-guardian/gateway.cjs index 11662a63a73..b10aba333d8 100644 --- a/scripts/ocx-recovery-guardian/gateway.cjs +++ b/scripts/ocx-recovery-guardian/gateway.cjs @@ -115,29 +115,6 @@ function readBody(request) { }); } -function hasPreviousResponseId(body) { - try { - const parsed = JSON.parse(body.toString("utf8")); - return typeof parsed.previous_response_id === "string" && parsed.previous_response_id.length > 0; - } catch { - return false; - } -} - -function exactFallbackModel(body, models) { - let parsed; - try { parsed = JSON.parse(body.toString("utf8")); } catch { return null; } - if (!parsed || typeof parsed !== "object" || typeof parsed.model !== "string") return null; - const mapped = Object.prototype.hasOwnProperty.call(models, parsed.model) ? models[parsed.model] : null; - return typeof mapped === "string" && mapped.length > 0 ? mapped : null; -} - -function rewriteFallbackModel(body, mappedModel) { - const parsed = JSON.parse(body.toString("utf8")); - parsed.model = mappedModel; - return Buffer.from(JSON.stringify(parsed)); -} - function requestUpstream({ origin, method, path, body, headers, headerTimeoutMs, clientRequest, clientResponse, onFailure, log }) { return new Promise(resolve => { const upstreamHeaders = { ...headers }; @@ -152,8 +129,13 @@ function requestUpstream({ origin, method, path, body, headers, headerTimeoutMs, }); let settled = false; let failureNotified = false; + // A client that hangs up makes the upstream teardown look like an upstream + // failure (ECONNRESET/aborted). That is the reader leaving, not the primary + // dying, and charging it would push every in-flight request onto the + // fallback model for the whole stability window. + let clientCancelled = false; const notifyFailure = () => { - if (failureNotified) return; + if (failureNotified || clientCancelled) return; failureNotified = true; try { onFailure(); } catch { /* recovery notification cannot affect the request */ } }; @@ -184,9 +166,9 @@ function requestUpstream({ origin, method, path, body, headers, headerTimeoutMs, if (!clientResponse.headersSent) writeJson(clientResponse, 502, { error: { code: "upstream_unavailable" } }); settle({ ok: false }); }); - clientRequest.once("aborted", () => upstream.destroy()); + clientRequest.once("aborted", () => { clientCancelled = true; upstream.destroy(); }); clientResponse.once("close", () => { - if (!clientResponse.writableEnded) upstream.destroy(); + if (!clientResponse.writableEnded) { clientCancelled = true; upstream.destroy(); } }); if (body && body.length) upstream.end(body); else upstream.end(); }); @@ -238,8 +220,6 @@ async function createGateway(options) { response.once("finish", release); response.once("close", release); try { - const localPort = server.address().port; - const allowedHosts = new Set([`127.0.0.1:${localPort}`, `localhost:${localPort}`]); if (!allowedHosts.has(String(request.headers.host || "").toLowerCase())) { writeJson(response, 421, { error: { code: "invalid_host" } }); return; @@ -298,18 +278,23 @@ async function createGateway(options) { }); return; } - if (request.method === "POST" && hasPreviousResponseId(body)) { - writeJson(response, 503, { error: { code: "fallback_requires_fresh_full_context" } }); - return; - } let fallbackBody = body; if (request.method === "POST") { - const mappedModel = exactFallbackModel(body, modelMap); - if (!mappedModel) { + // One parse per request: this body can be 8 MiB and every decision below + // reads the same document. + let parsed = null; + try { parsed = JSON.parse(body.toString("utf8")); } catch { /* not JSON, so neither a continuation nor a mappable model */ } + if (typeof parsed?.previous_response_id === "string" && parsed.previous_response_id.length > 0) { + writeJson(response, 503, { error: { code: "fallback_requires_fresh_full_context" } }); + return; + } + const mapped = typeof parsed?.model === "string" && Object.prototype.hasOwnProperty.call(modelMap, parsed.model) + ? modelMap[parsed.model] : null; + if (typeof mapped !== "string" || !mapped) { writeJson(response, 503, { error: { code: "fallback_model_unavailable" } }); return; } - fallbackBody = rewriteFallbackModel(body, mappedModel); + fallbackBody = Buffer.from(JSON.stringify({ ...parsed, model: mapped })); } let fallbackKey; try { fallbackKey = await readFallbackKey(); } catch { @@ -347,6 +332,9 @@ async function createGateway(options) { }); }); const localPort = server.address().port; + // Fixed once the socket is bound. Rebuilding this per request cost a `server.address()` + // object, two template strings and a Set on every proxied call. + const allowedHosts = new Set([`127.0.0.1:${localPort}`, `localhost:${localPort}`]); if (primaryOrigin.port === String(localPort) || fallbackOrigin.port === String(localPort)) { await new Promise(resolve => server.close(resolve)); throw new Error("gateway cannot proxy to itself"); diff --git a/scripts/ocx-recovery-guardian/intent.ps1 b/scripts/ocx-recovery-guardian/intent.ps1 index 0a688bdf769..12b1232e10b 100644 --- a/scripts/ocx-recovery-guardian/intent.ps1 +++ b/scripts/ocx-recovery-guardian/intent.ps1 @@ -9,7 +9,9 @@ $maxBytes = 16KB function Assert-RecoveryIntentPath([string]$Path) { $item = Get-Item -LiteralPath $Path -Force -ErrorAction Stop - if ((([int]$item.Attributes -band [int][System.IO.FileAttributes]::ReparsePoint) -ne 0) -or -not $item.PSIsContainer -and $item.Length -gt $maxBytes) { + # Windows PowerShell binds -or and -and left-to-right at equal precedence, so + # the reparse test must be parenthesised or a small symlink fails open. + if ((([int]$item.Attributes -band [int][System.IO.FileAttributes]::ReparsePoint) -ne 0) -or ((-not $item.PSIsContainer) -and ($item.Length -gt $maxBytes))) { throw 'Recovery guardian marker is malformed; manual lifecycle action was not dispatched.' } return $item @@ -28,12 +30,20 @@ function Test-RecoveryGuardianEnabled([string]$OcHome) { } if (Test-RecoveryGuardianEnabled $OpenCodexHome) { - if ($Mode -ne 'maintenance' -and $Until -ne 0) { throw 'Recovery intent maintenance deadline is invalid.' } - if ($Mode -eq 'maintenance' -and $Until -le 0) { throw 'Recovery intent maintenance deadline is invalid.' } + $at = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + if ($Mode -ne 'maintenance') { + if ($Until -ne 0) { throw 'Recovery intent maintenance deadline is invalid.' } + } elseif ($Until -le $at -or $Until -gt ($at + 180000)) { + # Same contract as parseIntent in main.cjs: at < until <= at + 180000. A + # reader decodes any other maintenance intent as 'stopped', which fences all + # gateway traffic and refuses recovery, so an out-of-window deadline must + # never be persisted. + throw 'Recovery intent maintenance deadline is invalid.' + } $intentPath = Join-Path $OpenCodexHome 'recovery-intent.json' if (Test-Path -LiteralPath $intentPath) { $null = Assert-RecoveryIntentPath $intentPath } - $intent = [ordered]@{ version = 1; mode = $Mode; at = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() } + $intent = [ordered]@{ version = 1; mode = $Mode; at = $at } if ($Mode -eq 'maintenance') { $intent.until = $Until } $tmpPath = Join-Path $OpenCodexHome ('.recovery-intent.' + $PID + '.' + [Guid]::NewGuid().ToString('N') + '.tmp') try { diff --git a/scripts/ocx-recovery-guardian/main.cjs b/scripts/ocx-recovery-guardian/main.cjs index b794855a29f..e16802285f4 100644 --- a/scripts/ocx-recovery-guardian/main.cjs +++ b/scripts/ocx-recovery-guardian/main.cjs @@ -8,7 +8,17 @@ const { spawn } = require('node:child_process'); const { randomUUID } = require('node:crypto'); const { createGateway } = require('./gateway.cjs'); const { RecoveryPolicy } = require('./policy.cjs'); -const { runRepair } = require('./repair.cjs'); +const { runRepair, REPAIR_ORIGINS } = require('./repair.cjs'); + +// The local OpenRouter-style listener the fallback and repair routes share. +const LOCAL_ORIGIN = 'http://127.0.0.1:20128'; +// Incident directories are read by a human once and by nothing in the repository, so the +// newest few are all a long-lived guardian should keep. +const REPAIR_INCIDENTS_KEPT = 20; +// How far the boot instant implied by /healthz's `uptime` may sit from the one a snapshot +// was taken at before it counts as another process generation. A wrap that slips inside +// this window is caught by a later observation. +const BOOT_TOLERANCE_MS = 5000; const pause = ms => new Promise(resolve => setTimeout(resolve, ms)); const alive = pid => { try { process.kill(pid, 0); return true; } catch { return false; } }; @@ -20,12 +30,17 @@ async function jsonFile(file, max = 65536) { return JSON.parse(await fs.readFile(file, 'utf8')); } async function atomicJson(file, value) { + const encoded = JSON.stringify(value) + '\n'; const next = (writes.get(file) || Promise.resolve()).catch(() => {}).then(async () => { const stat = await fs.lstat(file).catch(error => { if (error.code !== 'ENOENT') throw error; }); if (stat && (!stat.isFile() || stat.isSymbolicLink())) throw Error('unsafe_state_file'); + // The observation loop rewrites status and budget every couple of seconds. + // Compare inside the queue so a concurrent writer cannot slip between this + // read and the replacement below. + if (stat && await fs.readFile(file, 'utf8') === encoded) return; const temp = `${file}.${process.pid}.${randomUUID()}.tmp`; try { - await fs.writeFile(temp, JSON.stringify(value) + '\n', { mode: 0o600, flag: 'wx' }); + await fs.writeFile(temp, encoded, { mode: 0o600, flag: 'wx' }); await fs.rename(temp, file); } finally { await fs.unlink(temp).catch(error => { if (error.code !== 'ENOENT') throw error; }); } }); @@ -48,10 +63,10 @@ async function loadSettings(file) { if (!Number.isInteger(cfg[key]) || cfg[key] < 1 || cfg[key] > 65535) throw Error('invalid_port'); } if (cfg.listenPort === cfg.primaryPort) throw Error('gateway_loop'); - if (cfg.fallback?.origin !== 'http://127.0.0.1:20128' || !cfg.fallback.models || Array.isArray(cfg.fallback.models)) throw Error('missing_fallback'); + if (cfg.fallback?.origin !== LOCAL_ORIGIN || !cfg.fallback.models || Array.isArray(cfg.fallback.models)) throw Error('missing_fallback'); if (Object.entries(cfg.fallback.models).some(([k, v]) => !k || typeof v !== 'string' || !v)) throw Error('invalid_models'); - if (!['http://127.0.0.1:11434/v1', 'http://127.0.0.1:20128/v1', 'https://api.mnnai.ru/v1'].includes(cfg.repair?.origin)) throw Error('invalid_repair'); - if (cfg.repair.fallbackOrigin !== undefined && cfg.repair.fallbackOrigin !== 'http://127.0.0.1:20128/v1') throw Error('invalid_repair_fallback'); + if (!REPAIR_ORIGINS.includes(cfg.repair?.origin)) throw Error('invalid_repair'); + if (cfg.repair.fallbackOrigin !== undefined && cfg.repair.fallbackOrigin !== `${LOCAL_ORIGIN}/v1`) throw Error('invalid_repair_fallback'); return cfg; } @@ -112,8 +127,10 @@ async function createGuardian(configFile, dependencies = {}) { const statusFile = path.join(cfg.openCodexHome, 'recovery-status.json'); const budgetFile = path.join(cfg.openCodexHome, 'recovery-budget.json'); const intentFile = path.join(cfg.openCodexHome, 'recovery-intent.json'); + const blockedFile = path.join(cfg.openCodexHome, 'recovery-blocked.json'); const policy = new RecoveryPolicy(dependencies.policyOptions); let primaryReady = false, stopped = true, closing = false, snapshot = null; + let snapshotBoot = null, lastStatusKey = '', openingInspection = false, snapshotAt = 0; let recovering = false, repairRunning = false, currentState = 'starting', lastIntentAt = -1; let lastIntentSignature = '', recoveryBlocked = null, failureSince = null, lastRecovery = null, lastRepair = null; let readyIdentity = '', readySince = null, repairController = null; @@ -134,20 +151,20 @@ async function createGuardian(configFile, dependencies = {}) { '-ProjectRoot', cfg.projectRoot, '-OpenCodexHome', cfg.openCodexHome, '-CodexHome', cfg.codexHome, '-Port', String(cfg.primaryPort)]; const inspect = dependencies.inspect || (() => boundedCommand(powershell, actionArgs('Inspect'))); const initial = await inspect(); - if (initial.ok && initial.value?.owned === true) snapshot = initial.value; + if (initial.ok && initial.value?.owned === true) { snapshot = initial.value; openingInspection = true; snapshotAt = now(); } try { const saved = await jsonFile(budgetFile); if (!policy.importSafeState(saved, now())) throw Error('invalid_budget'); } catch (error) { if (error.code !== 'ENOENT') throw Error('guardian_budget_invalid'); } - try { recoveryBlocked = await jsonFile(path.join(cfg.openCodexHome, 'recovery-blocked.json')); } + try { recoveryBlocked = await jsonFile(blockedFile); } catch (error) { if (error.code !== 'ENOENT') throw Error('guardian_block_invalid'); } // A previous companion may have exited while its external stop command was // still running. A new observer must not replay that uncertain transaction. if (policy.exportSafeState().recoveryStartedAt !== null && !recoveryBlocked) { recoveryBlocked = { version: 1, at: now(), reason: 'previous_recovery_uncertain' }; - await atomicJson(path.join(cfg.openCodexHome, 'recovery-blocked.json'), recoveryBlocked); + await atomicJson(blockedFile, recoveryBlocked); } const server = await createGateway({ port: cfg.listenPort, primaryOrigin, fallbackOrigin: cfg.fallback.origin, @@ -170,7 +187,7 @@ async function createGuardian(configFile, dependencies = {}) { chunks.push(next.value); } const body = JSON.parse(Buffer.concat(chunks).toString('utf8')); - return { ok: response.ok && body.service === 'opencodex' && (route !== '/readyz' || body.status === 'ready'), pid: body.pid }; + return { ok: response.ok && body.service === 'opencodex' && (route !== '/readyz' || body.status === 'ready'), pid: body.pid, uptime: body.uptime }; } catch { return { ok: false }; } } @@ -201,6 +218,19 @@ async function createGuardian(configFile, dependencies = {}) { healthReady: primaryReady, pid: snapshot?.pid, attempts: policy.exportSafeState().attempts.length, timing: { durationMs: Math.max(0, claimAt - (failureSince ?? claimAt)), observedAtMs: claimAt } }; await atomicJson(path.join(incidentDir, 'incident.json'), incident); + // Retention is best-effort: losing a race here must not lose the incident that was + // just recorded. Only directories whose name matches the shape written above count + // against the budget, so a stray file cannot evict a real incident, and a reparse + // point is never walked by `fs.rm(recursive)` the way the rest of this subsystem + // refuses to follow one. + try { + const incidentsRoot = path.dirname(incidentDir); + const entries = await fs.readdir(incidentsRoot, { withFileTypes: true }); + const generated = entries.filter(entry => entry.isDirectory() && !entry.isSymbolicLink() + && /^\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z-\d+$/.test(entry.name)).map(entry => entry.name); + const stale = generated.sort().reverse().slice(REPAIR_INCIDENTS_KEPT); + for (const name of stale) await fs.rm(path.join(incidentsRoot, name), { recursive: true, force: true }); + } catch { /* the next incident retries */ } if (dependencies.beforeRepairDispatch) await dependencies.beforeRepairDispatch(); const freshIntent = parseIntent(await jsonFile(intentFile, 8192), now()); if (closing || stopped || controller.signal.aborted || !freshIntent.valid @@ -241,14 +271,17 @@ async function createGuardian(configFile, dependencies = {}) { log('recovery_attempt_failed', { reason: result.uncertain ? 'stop_result_uncertain' : 'recovery_action_failed' }); if (result.uncertain || ['stop-uncertain', 'stop-not-confirmed'].includes(result.value?.reason)) { recoveryBlocked = { version: 1, at: now(), intentAt: lastIntentAt, reason: 'stop_result_uncertain' }; - await atomicJson(path.join(cfg.openCodexHome, 'recovery-blocked.json'), recoveryBlocked); + await atomicJson(blockedFile, recoveryBlocked); } dispatch(() => diagnose('recovery_action_failed')); } else { policy.markRecoveryFinished({ ok: true }, now()); log('recovery_start_received', { confirmedReady: false }); const updated = await inspect(); - if (updated.ok && updated.value?.owned === true) snapshot = updated.value; + // Recovery replaced the process, so this is a fresh opening inspection: the next + // tick that agrees on the pid lends it a boot instant instead of paying for a + // second, identical spawn. + if (updated.ok && updated.value?.owned === true) { snapshot = updated.value; snapshotBoot = null; openingInspection = true; snapshotAt = now(); } } } catch { policy.markRecoveryFinished({ ok: false }, now()); log('recovery_aborted', { reason: 'intent_or_identity_changed' }); } finally { recovering = false; await atomicJson(budgetFile, policy.exportSafeState()).catch(() => {}); } @@ -270,18 +303,44 @@ async function createGuardian(configFile, dependencies = {}) { stopped = intent.mode === 'stopped'; maintenanceUntil = intent.mode === 'maintenance' ? intent.until : 0; if (hadIntent && !stopped && (wasStopped || currentState === 'foreign')) policy.reset({ now: now() }); + if (hadIntent && intent.mode === 'running' && recoveryBlocked) { + // A fresh durable running intent is a new instruction from the operator. + // The uncertain-stop latch must not disable recovery for this home for + // the rest of the process' life, or a restart would re-read it from disk. + recoveryBlocked = null; + await fs.unlink(blockedFile).catch(() => {}); + } log('intent_observed', { mode: intent.mode, valid: intent.valid }); } stopped = intent.mode === 'stopped'; if (stopped || intent.mode === 'maintenance') repairController?.abort(); const [health, ready] = await Promise.all([probe('/healthz'), probe('/readyz')]); const healthPid = Number.isInteger(health.pid) && health.pid > 0 ? health.pid : null; - if (healthPid) { + // Inspect resolves the owning process tree through WMI and is far more expensive than + // a health probe, so a healthy steady state must not re-run it every couple of seconds. + // Skipping is only safe for a generation the snapshot can corroborate: Windows reuses + // pids, so a pid alone cannot tell a wrap apart from the process it replaced. The boot + // instant derived from /healthz's `uptime` can, and a probe that reports no uptime is + // not corroborated at all, so it is always inspected. + const boot = Number.isFinite(health.uptime) ? now() - health.uptime * 1000 : null; + // The opening inspection resolved this very generation, so the first probe that agrees on + // its pid lends it a boot instant instead of paying for a second, identical spawn. + if (openingInspection && boot !== null && healthPid === snapshot?.pid) { snapshotBoot = boot; snapshotAt = now(); } + openingInspection = false; + // Corroboration decays. `snapshot` carries the launcher identity that `recover()` later + // hands to the action script, and pid+boot agreeing forever would otherwise freeze it + // even if the launching process went away behind us. A minute between re-resolutions + // costs one spawn per ~30 ticks instead of one per tick. + const corroborated = boot !== null && snapshotBoot !== null && Math.abs(boot - snapshotBoot) <= BOOT_TOLERANCE_MS + && now() - snapshotAt <= 60_000; + if (healthPid && (healthPid !== snapshot?.pid || !corroborated)) { const updated = await inspect(); if (updated.ok && updated.value?.owned === true && updated.value.pid === healthPid) { snapshot = updated.value; + snapshotBoot = boot; + snapshotAt = now(); if (currentState === 'foreign') policy.reset({ now: now() }); - } else snapshot = null; + } else { snapshot = null; snapshotBoot = null; } } const observedReady = !stopped && health.ok && ready.ok && healthPid === ready.pid && snapshot?.owned === true && snapshot.pid === healthPid; if (!observedReady && !stopped && failureSince === null) failureSince = now(); @@ -305,7 +364,15 @@ async function createGuardian(configFile, dependencies = {}) { if (primaryReady) failureSince = null; if (decision.state !== currentState) { currentState = decision.state; log('state_changed', { state: currentState, ready: primaryReady }); } if (decision.state === 'stopped') stopped = true; - await atomicJson(statusFile, state()); + // `at` is a wall-clock stamp and this loop runs every couple of seconds, so the status + // payload always differs while nothing it records has changed: dedup on the state, not + // on the timestamp, or the inspect throttle above buys nothing. + const status = state(); + const statusKey = JSON.stringify({ ...status, at: 0 }); + if (statusKey !== lastStatusKey) { + lastStatusKey = statusKey; + await atomicJson(statusFile, status); + } await atomicJson(budgetFile, policy.exportSafeState()); if (decision.action === 'recover' && !recoveryBlocked) dispatch(recover); if (decision.action === 'diagnose') dispatch(() => diagnose(decision.reason || 'recovery_budget_exhausted')); diff --git a/scripts/ocx-recovery-guardian/policy.cjs b/scripts/ocx-recovery-guardian/policy.cjs index ffef6bf4df1..5578c3ea39a 100644 --- a/scripts/ocx-recovery-guardian/policy.cjs +++ b/scripts/ocx-recovery-guardian/policy.cjs @@ -50,7 +50,7 @@ class RecoveryPolicy { if (!sample || typeof sample !== "object") return this.#foreign(); if (sample.manualStop === true) return this.#stop("manual-stop"); if (sample.owned === true && sample.launcherAlive === false) return this.#stop("launcher-stopped"); - if (sample.owned !== true || sample.identityChanged === true) return this.#foreign(); + if (sample.owned !== true) return this.#foreign(); const ready = sample.ready === true && sample.health === true && sample.alive === true; if (this.recoveryStartedAt !== null || this.awaitingReady) return this.#observeRecovery(ready, current); if (ready) return this.#observeReady(current); diff --git a/scripts/ocx-recovery-guardian/repair.cjs b/scripts/ocx-recovery-guardian/repair.cjs index 17356699c70..c390a4e55ca 100644 --- a/scripts/ocx-recovery-guardian/repair.cjs +++ b/scripts/ocx-recovery-guardian/repair.cjs @@ -8,6 +8,15 @@ const path = require("node:path"); const { spawnSync } = require("node:child_process"); const MODEL = "glm-5.3-flash"; +// The only endpoints this companion will talk to, each with the wire model that +// vendor expects. main.cjs admits configuration against the same list, so an +// origin cannot be accepted by one and unknown to the other. +const WIRE_MODELS = Object.freeze({ + "http://127.0.0.1:11434/v1": "glm-5.3-flash:cloud", + "http://127.0.0.1:20128/v1": "ollama-local/glm-5.3-flash:cloud", + "https://api.mnnai.ru/v1": MODEL, +}); +const REPAIR_ORIGINS = Object.freeze(Object.keys(WIRE_MODELS)); const DEADLINE_MS = 90_000; const MAX_RESPONSE_BYTES = 128 * 1024; const MAX_SOURCE_BYTES = 16 * 1024; @@ -53,19 +62,13 @@ function sanitizeIncident(incident) { function normalizeEndpoint(value) { let parsed; try { parsed = new URL(value); } catch { return null; } - const normalizedPath = parsed.pathname.replace(/\/+$/, ""); if (parsed.username || parsed.password || parsed.search || parsed.hash) return null; - const loopback = parsed.protocol === "http:" && parsed.hostname === "127.0.0.1" && parsed.port === "20128"; - const ollama = parsed.protocol === "http:" && parsed.hostname === "127.0.0.1" && parsed.port === "11434"; - const mnn = parsed.protocol === "https:" && parsed.hostname === "api.mnnai.ru" && (parsed.port === "" || parsed.port === "443"); - if ((loopback || ollama || mnn) && normalizedPath === "/v1") return `${parsed.protocol}//${parsed.host}/v1`; - return null; + const endpoint = `${parsed.protocol}//${parsed.host}${parsed.pathname.replace(/\/+$/, "")}`; + return REPAIR_ORIGINS.includes(endpoint) ? endpoint : null; } function wireModelFor(endpoint) { - if (endpoint === "http://127.0.0.1:11434/v1") return "glm-5.3-flash:cloud"; - if (endpoint === "http://127.0.0.1:20128/v1") return "ollama-local/glm-5.3-flash:cloud"; - return MODEL; + return WIRE_MODELS[endpoint]; } function isWithin(base, target) { @@ -351,4 +354,4 @@ async function runRepair({ incident, projectRoot, incidentDir, endpoint, fallbac return receipt(sanitized, { outcome: "failed", failureClass: lastFailure, requestCount }); } -module.exports = { runRepair }; +module.exports = { runRepair, REPAIR_ORIGINS }; diff --git a/scripts/ocx-recovery-guardian/windows-action.ps1 b/scripts/ocx-recovery-guardian/windows-action.ps1 index 1c7625d0335..dfbcf1b49f8 100644 --- a/scripts/ocx-recovery-guardian/windows-action.ps1 +++ b/scripts/ocx-recovery-guardian/windows-action.ps1 @@ -188,12 +188,7 @@ function Test-OptionalPathArgument { function Test-VisibleLauncherParent { param([AllowNull()]$Parent, [int]$ProcessId, [string]$Start, [string]$ScriptPath, [string]$Root, [string]$OpenHome, [string]$CodexConfigured, [string]$CodexCanonical) if (-not (Test-ExpectedIdentity -Process $Parent -ProcessId $ProcessId -Start $Start)) { return $false } - $powershell = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe' - if (-not (Test-ExactPath -Left $Parent.ExecutablePath -Right $powershell)) { return $false } - if (-not (Test-ArgumentToken -Text $Parent.CommandLine -Flag '-File' -Value $ScriptPath)) { return $false } - if (-not (Test-ArgumentToken -Text $Parent.CommandLine -Flag '-ProjectRoot' -Value $Root)) { return $false } - if (-not (Test-OptionalPathArgument -Text $Parent.CommandLine -Flag '-OpenCodexHome' -Configured $OpenHome -Canonical $OpenHome)) { return $false } - return Test-OptionalPathArgument -Text $Parent.CommandLine -Flag '-CodexHome' -Configured $CodexConfigured -Canonical $CodexCanonical + return Test-VisibleLauncherCandidate -Parent $Parent -ScriptPath $ScriptPath -Root $Root -OpenHome $OpenHome -CodexConfigured $CodexConfigured -CodexCanonical $CodexCanonical } function Test-VisibleLauncherCandidate { @@ -324,14 +319,19 @@ function Read-RecoveryIntent { $path = Join-Path $OpenCodexDirectory 'recovery-intent.json' if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { return [pscustomobject]@{ valid = $false; reason = 'missing-intent'; mode = ''; at = 0L } } $intent = Read-BoundedJson -Path $path -MaximumBytes $IntentMaxBytes - if ($null -eq $intent -or -not ($intent.version -is [int] -or $intent.version -is [long]) -or $intent.version -ne 1 -or $intent.mode -isnot [string] -or -not ($intent.at -is [int] -or $intent.at -is [long]) -or $intent.at -lt 0) { + # Set-StrictMode makes a missing property a terminating error, and the early + # Recover read runs outside any try: test presence before reading the value, + # or a hand-edited intent file kills the script before it can emit a receipt. + if ($null -eq $intent -or $null -eq $intent.PSObject.Properties['version'] -or $null -eq $intent.PSObject.Properties['mode'] -or $null -eq $intent.PSObject.Properties['at'] ` + -or -not ($intent.version -is [int] -or $intent.version -is [long]) -or $intent.version -ne 1 -or $intent.mode -isnot [string] -or -not ($intent.at -is [int] -or $intent.at -is [long]) -or $intent.at -lt 0) { return [pscustomobject]@{ valid = $false; reason = 'invalid-intent'; mode = ''; at = 0L } } if ($intent.mode -notin @('stopped', 'running', 'maintenance')) { return [pscustomobject]@{ valid = $false; reason = 'invalid-intent'; mode = ''; at = 0L } } if ($intent.mode -eq 'stopped') { return [pscustomobject]@{ valid = $false; reason = 'manual-stop'; mode = 'stopped'; at = [long]$intent.at } } if ($intent.mode -eq 'maintenance') { try { - if (-not ($intent.until -is [int] -or $intent.until -is [long]) -or $intent.until -le $intent.at) { throw 'invalid' } + # One window for every writer and reader: at < until <= at + 180000 (main.cjs parseIntent). + if (-not ($intent.until -is [int] -or $intent.until -is [long]) -or $intent.until -le $intent.at -or $intent.until -gt ($intent.at + 180000)) { throw 'invalid' } $nowMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() if ($intent.until -le $nowMs) { throw 'expired' } } catch { return [pscustomobject]@{ valid = $false; reason = 'maintenance-expired'; mode = 'maintenance'; at = [long]$intent.at } } @@ -418,12 +418,25 @@ public static class OcxGuardianDiscardDrain { '@ | Out-Null } +function ConvertTo-NativeArgument { + param([Parameter(Mandatory)][string]$Value) + # Windows flattens an argument array into one command line, where a quoted + # path ending in backslashes swallows its own closing quote and merges the + # next parameter into it. Rejecting quotes and control characters keeps the + # value a single argument whatever follows. Same convention as + # ConvertTo-RecoveryGuardianArgument in windows-visible-proxy.ps1. + if ([string]::IsNullOrWhiteSpace($Value) -or $Value.IndexOf('"') -ge 0 -or $Value -match '[\x00-\x1F]') { + throw 'unsafe-native-argument' + } + return '"' + ($Value -replace '(\\+)$', '$1$1') + '"' +} + function Invoke-GracefulProjectStop { param([string]$BunPath, [string]$CliPath, [string]$OpenCodexDirectory, [string]$CodexDirectory) Initialize-DiscardDrainType $info = New-Object System.Diagnostics.ProcessStartInfo $info.FileName = $BunPath - $info.Arguments = ('"{0}" stop' -f $CliPath.Replace('"', '""')) + $info.Arguments = ('{0} stop' -f (ConvertTo-NativeArgument $CliPath)) $info.WorkingDirectory = $ProjectRoot $info.UseShellExecute = $false $info.CreateNoWindow = $true @@ -455,7 +468,6 @@ function Start-VisibleLauncher { param([string]$ScriptPath, [string]$CodexHomeArgument) $powershell = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe' if (-not (Test-Path -LiteralPath $powershell -PathType Leaf)) { return $false } - function ConvertTo-NativeArgument([string]$Value) { return '"' + $Value.Replace('"', '\"') + '"' } $launchArguments = @('-NoProfile', '-File', (ConvertTo-NativeArgument $ScriptPath), '-ProjectRoot', (ConvertTo-NativeArgument $ProjectRoot), '-OpenCodexHome', (ConvertTo-NativeArgument $OpenCodexHome), '-CodexHome', (ConvertTo-NativeArgument $CodexHomeArgument), '-Port', ([string]$Port), '-ConsoleLevel', 'Warn') $launcher = Start-Process -FilePath $powershell -ArgumentList ($launchArguments -join ' ') -WorkingDirectory $ProjectRoot -WindowStyle Normal -PassThru return $null -ne $launcher diff --git a/src/cli/index.ts b/src/cli/index.ts index d76c7208dd0..a2f97247d85 100755 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -140,7 +140,7 @@ import { selfLaunchArgv } from "../lib/self-launch-argv"; import { initializeNodeLauncherContext } from "./launcher-context"; import { createLocalAttestationSecret } from "../lib/local-management-attestation"; import { MEMORY_DRAIN_RESTART_MS, REPLACEMENT_READY_TIMEOUT_MS } from "../lib/system-restart-contract"; -import { writeRecoveryIntentIfGuardianEnabled } from "../lib/recovery-intent"; +import { writeRecoveryIntentIfGuardianEnabled, backupRecoveryIntent, restoreRecoveryIntent, type RecoveryIntentBackup } from "../lib/recovery-intent"; /** * A failed shell-hook reconcile is not cosmetic: a stale hook keeps sourcing @@ -468,6 +468,11 @@ async function handleStart(options: { block?: boolean } = {}) { // live daemon holding resources while it overwrites its own binary. await maybeShowUpdatePrompt(); + // Every path that intends to bring this home's proxy up has to say so: the guardian + // reads a stale `stopped` as "the user stopped this on purpose" and stops recovering + // crashes for the whole life of the new process. + await writeRecoveryIntentIfGuardianEnabled("running"); + type StartServerModule = typeof import("../server"); type BoundStart = { server: ReturnType; @@ -745,6 +750,9 @@ async function handleEnsure(options: { existingIsSuccess?: boolean } = {}): Prom return false; } const live = owner.live; + // The proxy is up, or this command is about to bring it up: same durable `running` + // intent `ocx start` records, before either branch can return. + await writeRecoveryIntentIfGuardianEnabled("running"); if (live) { if (options.existingIsSuccess === false) { console.error("Proxy appeared while restart was confirming absence; no start was attempted."); @@ -816,9 +824,16 @@ async function handleEnsure(options: { existingIsSuccess?: boolean } = {}): Prom return true; } -/** Fixed tray action: start the proxy without depending on codexAutoStart. */ -async function handleTrayProxyStart(existingIsSuccess = true, writeRunningIntent = true): Promise { - if (writeRunningIntent) await writeRecoveryIntentIfGuardianEnabled("running"); +/** + * Fixed tray action: start the proxy without depending on codexAutoStart. + * + * Intent authority, decided deliberately: this command writes no intent, so it stays + * tray-only — the tray pre-signs `running` before it spawns this, which is what covers the + * already-live path that starts no child. A child this does spawn re-affirms `running` + * itself (`handleStart`, `handleEnsure`, the visible launcher), and the writer drops a + * re-affirmation of an intent that already reads `running`, so one start signs once. + */ +async function handleTrayProxyStart(existingIsSuccess = true): Promise { const ok = await runTrayProxyStart({ findLive: findLiveProxy, existingIsSuccess, @@ -914,8 +929,14 @@ async function handleProxyRestart( async function handleTrayProxyRestart(): Promise { await writeRecoveryIntentIfGuardianEnabled("maintenance", { until: Date.now() + 180_000 }); - const restarted = await handleProxyRestart(() => handleTrayProxyStart(false, false)); - if (restarted) await writeRecoveryIntentIfGuardianEnabled("running"); + try { + await handleProxyRestart(() => handleTrayProxyStart(false)); + } finally { + // Unconditional, and a fence that only has to cover the restart window is correct + // either way: the guardian reader never compares `until` to now, so a fence left + // behind by a failed restart would keep this home from ever recovering on its own. + await writeRecoveryIntentIfGuardianEnabled("running"); + } } async function handleRestartStartWhenStopped(): Promise { @@ -994,15 +1015,44 @@ async function restoreSharedClientStateAfterStop(): Promise<{ historyOnly: boole return { historyOnly, historyDeferred, other }; } -async function handleStop() { - // A guardian-driven recovery child is carrying out a bounded automatic repair, - // not an operator's durable manual-stop instruction. - if (process.env.OPENCODEX_GUARDIAN_RECOVERY !== "1") { - await writeRecoveryIntentIfGuardianEnabled("stopped"); - } +async function handleStop(): Promise { + // The lease first: an intent written before the mutation lease could be refused would + // leave the file claiming `stopped` while the proxy keeps serving. const lease = acquireOwnershipMutationLease(serviceStatePaths()); - try { return await handleStopUnlocked(); } - finally { lease.release(); } + try { + let stopIntent: RecoveryIntentBackup | null = null; + // A guardian-driven recovery child is carrying out a bounded automatic repair, + // not an operator's durable manual-stop instruction. + if (process.env.OPENCODEX_GUARDIAN_RECOVERY !== "1") { + try { + stopIntent = backupRecoveryIntent(); + await writeRecoveryIntentIfGuardianEnabled("stopped"); + } catch (error) { + // Fail closed with an operator-facing line rather than a stack out of the CLI + // top level, which would leave the proxy running and the reason unreadable. + const message = error instanceof Error ? error.message : String(error); + console.error(`❌ Stop refused: ${message}`); + console.error(" Nothing was stopped. Repair or remove the recovery guardian marker in this home, then rerun 'ocx stop'."); + process.exitCode = 1; + return { + ok: false, + summary: summarizeStopRun( + { service: "absent", proxy: "unknown", sharedTeardown: "skipped", inheritedTeardownBlocks: false, receiptClearFailed: false }, + { failed: true, historyOnly: false, historyDeferred: false, exitCode: 1 }, + ), + }; + } + } + const outcome = await handleStopUnlocked(); + // A refused stop leaves this proxy serving, and the durable `stopped` this run wrote + // would have the guardian gateway fence a home that never stopped. + if (stopIntent && !outcome.summary.runtimeDown && !await restoreRecoveryIntent(stopIntent)) { + console.error("❌ The stop was refused and recovery-intent.json could not be restored; check it against the running proxy."); + } + return outcome; + } finally { + lease.release(); + } } async function handleStopUnlocked() { diff --git a/src/lib/recovery-intent.ts b/src/lib/recovery-intent.ts index 1053004532d..8c527cc07c7 100644 --- a/src/lib/recovery-intent.ts +++ b/src/lib/recovery-intent.ts @@ -1,4 +1,4 @@ -import { lstatSync, readFileSync } from "node:fs"; +import { lstatSync, readFileSync, rmSync } from "node:fs"; import { join, resolve } from "node:path"; import { atomicWriteFileAsync } from "../config/atomic-write"; @@ -19,8 +19,11 @@ export interface WriteRecoveryIntentOptions { until?: number; } -const MARKER_MAX_BYTES = 16 * 1024; -const INTENT_MAX_BYTES = 16 * 1024; +const STATE_MAX_BYTES = 16 * 1024; +// The guardian reader rejects a maintenance intent outside this window, so a writer +// must not create one it would decode as `stopped` +// (scripts/ocx-recovery-guardian/main.cjs, parseIntent). +const MAINTENANCE_WINDOW_MAX_MS = 180_000; function unsafeMarker(): never { throw new Error("Recovery guardian marker is malformed; manual lifecycle action was not dispatched."); @@ -30,15 +33,23 @@ function isMissing(error: unknown): boolean { return typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT"; } -function assertSafeExistingFile(path: string, maxBytes: number): ReturnType { +function assertSafeExistingFile(path: string): void { const stat = lstatSync(path); - if (!stat.isFile() || stat.isSymbolicLink() || stat.size > maxBytes) unsafeMarker(); - return stat; + if (!stat.isFile() || stat.size > STATE_MAX_BYTES) unsafeMarker(); } function assertSafeExistingDirectory(path: string): void { - const stat = lstatSync(path); - if (!stat.isDirectory() || stat.isSymbolicLink()) unsafeMarker(); + if (!lstatSync(path).isDirectory()) unsafeMarker(); +} + +/** True when the durable intent already reads `running`. */ +function runningIntentWritten(home: string): boolean { + try { + const stored = JSON.parse(readFileSync(join(home, "recovery-intent.json"), "utf8")) as { mode?: unknown }; + return stored.mode === "running"; + } catch { + return false; + } } /** @@ -52,7 +63,7 @@ export function recoveryGuardianEnabled(home: string = getConfigDir()): boolean const marker = join(root, "recovery-guardian.json"); try { assertSafeExistingDirectory(root); - assertSafeExistingFile(marker, MARKER_MAX_BYTES); + assertSafeExistingFile(marker); } catch (error) { if (isMissing(error)) return false; throw error; @@ -70,6 +81,42 @@ export function recoveryGuardianEnabled(home: string = getConfigDir()): boolean return true; } +/** + * The durable intent exactly as one run found it, so a refusal can put it back. An + * unreadable file is a refusal in itself: rolling back to `absent` would delete an + * operator's instruction that this run never wrote. + */ +export interface RecoveryIntentBackup { + path: string; + bytes: Buffer | null; +} + +/** Read the intent before a lifecycle write, or null when this home has no guardian. */ +export function backupRecoveryIntent(home = getConfigDir()): RecoveryIntentBackup | null { + const root = resolve(home); + if (!recoveryGuardianEnabled(root)) return null; + const intentPath = join(root, "recovery-intent.json"); + try { + assertSafeExistingFile(intentPath); + return { path: intentPath, bytes: readFileSync(intentPath) }; + } catch (error) { + if (!isMissing(error)) throw error; + return { path: intentPath, bytes: null }; + } +} + +/** Restore the bytes a refused action found, removing the file when it found none. */ +export async function restoreRecoveryIntent(backup: RecoveryIntentBackup | null): Promise { + if (!backup) return true; + try { + if (backup.bytes === null) rmSync(backup.path, { force: true }); + else await atomicWriteFileAsync(backup.path, backup.bytes.toString("utf8")); + return true; + } catch { + return false; + } +} + /** Write a non-secret, bounded manual-recovery intent when the guardian opted in. */ export async function writeRecoveryIntentIfGuardianEnabled( mode: RecoveryIntentMode, @@ -82,16 +129,26 @@ export async function writeRecoveryIntentIfGuardianEnabled( if (mode !== "running" && mode !== "stopped" && mode !== "maintenance") { throw new Error("Recovery intent mode is invalid."); } - if (options.until !== undefined && (!Number.isSafeInteger(options.until) || options.until <= at)) { + if (options.until !== undefined + && (!Number.isSafeInteger(options.until) || options.until <= at || options.until > at + MAINTENANCE_WINDOW_MAX_MS)) { throw new Error("Recovery intent maintenance deadline is invalid."); } if (mode !== "maintenance" && options.until !== undefined) { throw new Error("Only a maintenance intent may carry a deadline."); } + // A maintenance intent without a deadline decodes as `stopped` in every reader, which + // fences the whole home, so the window is mandatory here exactly as in main.cjs. + if (mode === "maintenance" && options.until === undefined) { + throw new Error("Recovery intent maintenance deadline is invalid."); + } + // The tray, the visible launcher and the proxy itself all affirm `running` for one + // start. Each fresh signature restarts the guardian's stabilisation window, so a + // re-affirmation must not rewrite the file. + if (mode === "running" && runningIntentWritten(home)) return true; const intentPath = join(home, "recovery-intent.json"); try { - assertSafeExistingFile(intentPath, INTENT_MAX_BYTES); + assertSafeExistingFile(intentPath); } catch (error) { if (!isMissing(error)) throw error; } diff --git a/src/server/background-lifecycle.ts b/src/server/background-lifecycle.ts index 5125e3d1885..35fc881f4a1 100644 --- a/src/server/background-lifecycle.ts +++ b/src/server/background-lifecycle.ts @@ -1,7 +1,6 @@ import type { StorageCleanupPolicy } from "../types"; import { join } from "node:path"; import { getConfigDir } from "../config/paths"; -import { startRuntimeDiagnostics } from "../lib/runtime-diagnostics"; import { getActiveTurnCount } from "./lifecycle"; import { responseStateMetrics } from "../responses/state"; import { startStateStoreSweeper } from "../lib/state-store-sweeper"; @@ -36,10 +35,12 @@ import { type PolicyApply = (policy: StorageCleanupPolicy) => void; +type RecorderHandle = { stop(): void }; + type ProcessLoops = { - diagnostics: ReturnType | null; - memoryWatchdog: MemoryWatchdog; - stateStoreSweeper: ReturnType; + diagnostics: RecorderHandle | null; + memoryWatchdog: MemoryWatchdog | null; + stateStoreSweeper: ReturnType | null; }; type LeaseOwner = { @@ -64,23 +65,30 @@ function setLivePolicyOwner(applyPolicy: PolicyApply | null): void { } function startProcessLoops(applyPolicy: PolicyApply): ProcessLoops { - let diagnostics: ReturnType | null = null; - let memoryWatchdog: MemoryWatchdog | null = null; - let stateStoreSweeper: ReturnType | null = null; + const loops: ProcessLoops = { diagnostics: null, memoryWatchdog: null, stateStoreSweeper: null }; try { - memoryWatchdog = startMemoryWatchdog(); + loops.memoryWatchdog = startMemoryWatchdog(); if (process.env.OPENCODEX_RUNTIME_DIAGNOSTICS === "1") { - try { - diagnostics = startRuntimeDiagnostics(join(getConfigDir(), "runtime-diagnostics.jsonl"), () => { + // Opt-in, and loaded only when it is opted in. The one thing that opts in today is + // scripts/windows-visible-proxy.ps1 (the desktop launcher, where an event-loop stall is + // a frozen window), so no other install pays for the recorder or its child process. + void import("../lib/runtime-diagnostics").then(module => module.startRuntimeDiagnostics( + join(getConfigDir(), "runtime-diagnostics.jsonl"), + () => { const turns = getActiveTurnCount(); const state = turns > 0 ? responseStateMetrics() : null; return { activeTurns: turns, responseBytes: state?.totalBytes ?? null, spillWrites: state?.spillWrites ?? null, spillFailures: state?.spillWriteFailures ?? null, spillTimeoutRefusals: state?.spillAclTimeoutMemoRefusals ?? null }; - }); - } catch { console.warn("[runtime-diagnostics] recorder could not start"); } + }, + )).then(recorder => { + // The identity check is the teardown race: if these loops were replaced or rolled + // back while the module loaded, the recorder has no owner and stops itself. + if (processLoops === loops) loops.diagnostics = recorder; + else recorder.stop(); + }).catch(() => console.warn("[runtime-diagnostics] recorder could not start")); } - stateStoreSweeper = startStateStoreSweeper(); + loops.stateStoreSweeper = startStateStoreSweeper(); setLivePolicyOwner(applyPolicy); startStorageCleanupScheduler(); // Opt-in: the tick itself is a no-op unless config.quotaResetNotify is enabled with a @@ -113,11 +121,11 @@ function startProcessLoops(applyPolicy: PolicyApply): ProcessLoops { .catch(() => { // The next poll tick retries. }); - return { memoryWatchdog, stateStoreSweeper, diagnostics }; + return loops; } catch (error) { - diagnostics?.stop(); - memoryWatchdog?.stop(); - stateStoreSweeper?.stop(); + loops.diagnostics?.stop(); + loops.memoryWatchdog?.stop(); + loops.stateStoreSweeper?.stop(); stopStorageCleanupScheduler(); stopQuotaResetPoller(); stopCatalogAutoRefresh(); @@ -130,8 +138,8 @@ function stopProcessLoops(): void { const loops = processLoops; processLoops = null; loops?.diagnostics?.stop(); - loops?.memoryWatchdog.stop(); - loops?.stateStoreSweeper.stop(); + loops?.memoryWatchdog?.stop(); + loops?.stateStoreSweeper?.stop(); stopStorageCleanupScheduler(); stopQuotaResetPoller(); stopCatalogAutoRefresh(); diff --git a/src/server/management-api.ts b/src/server/management-api.ts index 1e5018d6842..fa002ed5df5 100644 --- a/src/server/management-api.ts +++ b/src/server/management-api.ts @@ -85,7 +85,7 @@ import type { CatalogDisposition, ConvergeCodex } from "../codex/convergence-typ import { normalizeCatalogDisposition } from "../codex/catalog-refresh-status"; import { managementBodyTooLargeResponse } from "./management/body"; import { handleSessionRoutes } from "./management/session-routes"; -import { writeRecoveryIntentIfGuardianEnabled } from "../lib/recovery-intent"; +import { writeRecoveryIntentIfGuardianEnabled, backupRecoveryIntent, restoreRecoveryIntent, type RecoveryIntentBackup } from "../lib/recovery-intent"; import { packageVersion } from "../lib/package-version"; // installed npm version instead of a stale hardcode. @@ -329,8 +329,10 @@ export async function handleManagementAPI( const { deferralMatchesReceipt } = await import("../config/pending-teardown"); const { deferralHonored, performStopTeardown } = await import("./stop-teardown"); const holdsReceipt = deferralHonored(url, deferralMatchesReceipt); + let stopIntent: RecoveryIntentBackup | null = null; if (!holdsReceipt) { try { + stopIntent = backupRecoveryIntent(); await writeRecoveryIntentIfGuardianEnabled("stopped"); } catch { return jsonResponse({ @@ -340,13 +342,21 @@ export async function handleManagementAPI( }, 503, req, config); } } + // Every refusal below answers "Nothing was changed", and the durable `stopped` written + // above would leave the guardian gateway fencing a home whose proxy is still serving. + const refuseStop = async (body: unknown, status: number): Promise => { + if (!await restoreRecoveryIntent(stopIntent)) { + console.warn("[opencodex] stop refused and recovery-intent.json could not be restored"); + } + return jsonResponse(body, status, req, config); + }; const respawnRisk = holdsReceipt ? "none" : installedServiceRespawnRisk(); if (respawnRisk === "respawnable") { - return jsonResponse({ + return await refuseStop({ success: false, code: "respawnable_service", message: "This proxy is managed by a Task Scheduler wrapper that can respawn it, so the stop must be run by `ocx stop`, which verifies the respawn window. Nothing was changed.", - }, 409, req, config); + }, 409); } if (respawnRisk === "self-unload") { // This proxy IS the launchd/systemd job, so stopping the manager below would @@ -356,21 +366,21 @@ export async function handleManagementAPI( // proxy (#4023). Refuse before touching anything, like the Windows branch above. // `ocx stop` is safe because it runs outside this process and owns the teardown // through its receipt, which is why the receipt-backed caller never reaches here. - return jsonResponse({ + return await refuseStop({ success: false, code: "self_unload_service", message: "This proxy is running as the installed service, so stopping the manager from inside it would end this process before native Codex is restored. Run `ocx stop`, which stops the service from outside and completes the restore. Nothing was changed.", - }, 409, req, config); + }, 409); } if (respawnRisk === "unknown") { // Do NOT send them to `ocx stop`: it maps the same unanswerable probe to a stop // failure, so that advice would be a loop. The scheduler query itself is what needs // fixing (#3008). - return jsonResponse({ + return await refuseStop({ success: false, code: "service_state_unknown", message: "The Windows Task Scheduler state could not be read, so this proxy cannot tell whether a wrapper would respawn it. Nothing was changed. Run `ocx service status` to see the query error, repair Task Scheduler access, then retry.", - }, 409, req, config); + }, 409); } let serviceStop: import("../service").ServiceStopOutcome; try { @@ -380,7 +390,7 @@ export async function handleManagementAPI( // The installed service belongs to another CODEX_HOME/OPENCODEX_HOME: it would respawn // this proxy immediately, and its shared config is not ours to tear down. Refuse the // stop instead of half-performing it. 409, not 500 — the request is well-formed. - return jsonResponse({ success: false, message: err.message }, 409, req, config); + return await refuseStop({ success: false, message: err.message }, 409); } throw err; } @@ -388,18 +398,18 @@ export async function handleManagementAPI( // so this route used to tear down shared config and exit while a manager that refused // to stop was still there to respawn the proxy (#3008). if (serviceStop === "failed") { - return jsonResponse({ + return await refuseStop({ success: false, message: "The installed service manager did not stop; it may respawn the proxy. Shared client config was left alone. Run `ocx stop` from the home that owns the service.", - }, 409, req, config); + }, 409); } if (serviceStop === "state-unknown") { // Same case, same remedy as the pre-check: the query is what needs fixing. - return jsonResponse({ + return await refuseStop({ success: false, code: "service_state_unknown", message: "The Windows Task Scheduler state could not be read, so this proxy cannot tell whether a wrapper would respawn it. Shared client config was left alone. Run `ocx service status` to see the query error, repair Task Scheduler access, then retry.", - }, 409, req, config); + }, 409); } // The pre-check above already refused the respawnable case without a receipt, so // reaching here with one means the parent owns the verification. diff --git a/src/tray/windows-tray.ps1 b/src/tray/windows-tray.ps1 index bdaaa661987..25e65dffacc 100644 --- a/src/tray/windows-tray.ps1 +++ b/src/tray/windows-tray.ps1 @@ -221,17 +221,51 @@ function Read-ListenTarget { } function Read-JsonUrl([string]$Url) { - $request = [System.Net.HttpWebRequest]::Create($Url) - $request.Method = "GET" - $request.Timeout = 700 - $request.ReadWriteTimeout = 700 - $response = $request.GetResponse() - try { - $reader = New-Object System.IO.StreamReader($response.GetResponseStream()) - try { return ($reader.ReadToEnd() | ConvertFrom-Json) } finally { $reader.Dispose() } - } finally { - $response.Dispose() + # GetResponse() stalled the WinForms UI thread for up to its 700 ms timeout on every + # 3 s tick. The request now runs on its own task: a tick waits 100 ms for the loopback + # round-trip and otherwise repeats what the last completed read proved. The 3 s ceiling + # replaces the timeout the task-based call ignores, so a wedged proxy still goes stale. + $task = $script:jsonTask + if ($null -eq $task) { + try { + $request = [System.Net.HttpWebRequest]::Create($Url) + $request.Method = "GET" + $request.Timeout = 700 + $request.ReadWriteTimeout = 700 + $task = $request.GetResponseAsync() + } catch { + $script:jsonPayload = $null + return $null + } + $script:jsonRequest = $request + $script:jsonTask = $task + $script:jsonTaskStarted = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + } + # A faulted task makes Wait() throw, and a task left installed would throw on every later + # tick and latch the tray Offline: observe the fault here and re-issue on the next tick. + $pending = $true + try { $pending = -not $task.Wait(100) } catch { $pending = $false } + $now = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + if ($pending -and ($now - $script:jsonTaskStarted) -lt 3000) { + return $script:jsonPayload + } + $script:jsonTask = $null + $script:jsonPayload = $null + if ($pending) { + # Abandoned at the ceiling and no later tick reads its response, so Abort() is what + # releases the socket: without it the tray spends one connection every 3 s. + try { $script:jsonRequest.Abort() } catch { } + } else { + try { + $response = $task.Result + try { + $reader = New-Object System.IO.StreamReader($response.GetResponseStream()) + try { $script:jsonPayload = ($reader.ReadToEnd() | ConvertFrom-Json) } finally { $reader.Dispose() } + } finally { $response.Dispose() } + } catch { } } + $script:jsonRequest = $null + return $script:jsonPayload } $notify = New-Object System.Windows.Forms.NotifyIcon @@ -268,6 +302,10 @@ $script:pendingStarted = 0L $script:pendingDeadline = 0L $script:pendingOldProxyPid = $null $script:pendingProcess = $null +$script:jsonTask = $null +$script:jsonRequest = $null +$script:jsonTaskStarted = 0L +$script:jsonPayload = $null function Set-PendingAction([string]$Action, [int]$TimeoutSeconds) { if ($null -ne $script:pendingAction) { @@ -463,9 +501,12 @@ function Update-TrayState { } $openItem.add_Click({ Start-OcxCommand @("gui") }) +# The intent write comes FIRST: a refused write throws, and a latched pending action +# would lock these three menu items for its whole budget and then report a failure the +# tray never actually dispatched. $startItem.add_Click({ - if (-not (Set-PendingAction "Start Proxy" 75)) { return } Set-RecoveryIntent -OpenCodexHome $OpenCodexHome -Mode "running" + if (-not (Set-PendingAction "Start Proxy" 75)) { return } $statusItem.Text = "Proxy: Starting..." # service start can spend 20s and the CLI then observes health for another 40s. $startProcess = Start-OcxCommand @("__tray-start") -TrackExit @@ -476,8 +517,8 @@ $startItem.add_Click({ } }) $stopItem.add_Click({ - if (-not (Set-PendingAction "Stop Proxy" 15)) { return } Set-RecoveryIntent -OpenCodexHome $OpenCodexHome -Mode "stopped" + if (-not (Set-PendingAction "Stop Proxy" 15)) { return } $statusItem.Text = "Proxy: Stopping..." $stopProcess = Start-OcxCommand @("stop") -TrackExit if ($stopProcess -is [System.Diagnostics.Process]) { @@ -487,8 +528,9 @@ $stopItem.add_Click({ } }) $restartItem.add_Click({ + # No intent write here: `ocx __tray-restart` signs the maintenance fence and clears it in + # the same function, and a second signature per restart resets the guardian's window. if (-not (Set-PendingAction "Restart Proxy" 160)) { return } - Set-RecoveryIntent -OpenCodexHome $OpenCodexHome -Mode "maintenance" -Until ([DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + 180000) $statusItem.Text = "Proxy: Restarting..." # /api/system/restart may drain active work for 60s and then spend up to 70s # handing off to an identity-verified replacement. The tray observes health/PID diff --git a/src/tray/windows.ts b/src/tray/windows.ts index 565bbf95680..3b9e1e96cc8 100644 --- a/src/tray/windows.ts +++ b/src/tray/windows.ts @@ -6,6 +6,7 @@ import { join, resolve, win32 as win32Path } from "node:path"; import { expandUserPath, getConfigDir } from "../config"; import { durableBunRuntime } from "../lib/bun-runtime"; import type { BunRuntimeSource } from "../lib/bun-runtime"; +import { recoveryGuardianEnabled } from "../lib/recovery-intent"; import { forgetEphemeralSecretPath, hardenSecretDir, hardenSecretPath } from "../lib/windows-secret-acl"; import { recordOwnedConfigPath } from "../lib/config-ownership"; import { renameAtomicFile } from "../lib/windows-atomic-replace"; @@ -75,6 +76,32 @@ function installedTrayRecoveryIntentPath(): string { return join(getConfigDir(), INSTALLED_TRAY_RECOVERY_INTENT_FILE); } +/** A published install ships no `scripts/` directory, so there is no helper to copy. */ +function trayRecoveryIntentHelperShipped(): boolean { + return existsSync(sourceTrayRecoveryIntentPath()); +} + +/** + * Whether THIS home requires the installed intent helper. The installed tray script + * refuses every lifecycle click when a guardian marker sits without its helper, so + * ownership must ask the same question of the home — asking the observer's source tree + * certified an npm-installed proxy as healthy while each click threw, and a reinstall + * could not fix what status had just approved. Install gates the copy on the same + * shipped test, so neither side can leave a hand-placed marker permanently stale. + */ +export function trayHomeRequiresRecoveryIntentHelper( + home = getConfigDir(), + helperShipped = trayRecoveryIntentHelperShipped(), +): boolean { + if (!helperShipped) return false; + try { + return recoveryGuardianEnabled(home); + } catch { + // A malformed marker is fail-closed in the tray script too. + return true; + } +} + export function windowsTrayStatePathsOwned( state: Pick & { launcherPath?: string }, configDir = getConfigDir(), @@ -480,7 +507,7 @@ function trayStatusFrom(registered: string | null): WindowsTrayStatus { const running = heartbeatProcessAlive(heartbeat); const registrationOwned = state !== null && registered === state.runCommand - && [state.bun, state.cli, state.script, ...(state.launcherPath ? [state.launcherPath] : []), ...installedTrayIconPaths(), ...(existsSync(sourceTrayRecoveryIntentPath()) ? [installedTrayRecoveryIntentPath()] : [])] + && [state.bun, state.cli, state.script, ...(state.launcherPath ? [state.launcherPath] : []), ...installedTrayIconPaths(), ...(trayHomeRequiresRecoveryIntentHelper() ? [installedTrayRecoveryIntentPath()] : [])] .every(path => existsSync(path)); const stale = windowsTrayRegistrationIsStale({ registered: registered !== null, @@ -714,9 +741,10 @@ export function installWindowsTray(startNow = true): WindowsTrayStatus { if (!hardenedDir.ok) throw new Error("Windows tray directory ACL hardening did not complete; refusing to install persistence."); replaceWindowsTrayOwnedFile(entry.script, readFileSync(sourceScript)); // `scripts/` is outside the npm package allowlist, so a published install has no - // guardian to carry and gets no helper. The tray script keeps its own fail-closed - // check for a home whose marker exists without the helper beside it. - if (existsSync(sourceRecoveryIntent)) { + // guardian to carry and gets no helper — the same test status applies to the home, and + // the tray script keeps its own fail-closed check for a marker without a helper beside + // it. + if (trayRecoveryIntentHelperShipped()) { replaceWindowsTrayOwnedFile(installedRecoveryIntent, readFileSync(sourceRecoveryIntent)); } for (const pair of iconPairs) replaceWindowsTrayOwnedFile(pair.installed, readFileSync(pair.source)); diff --git a/tests/windows/windows-recovery-gateway.test.ts b/tests/windows/windows-recovery-gateway.test.ts index d4fde5b9576..ba7c34d6b9c 100644 --- a/tests/windows/windows-recovery-gateway.test.ts +++ b/tests/windows/windows-recovery-gateway.test.ts @@ -167,11 +167,13 @@ test("a post-dispatch primary failure is never replayed to fallback", async () = const primary = await listen((_req, res) => { primaryFailures += 1; res.writeHead(503); res.end("down"); }); let fallbackHits = 0; const fallback = await listen((_req, res) => { fallbackHits += 1; res.end("fallback"); }); - const instance = await gateway(primary.port, fallback.port); + let charged = 0; + const instance = await gateway(primary.port, fallback.port, true, { onPrimaryFailure: () => { charged += 1; } }); const result = await call(instance.port, { method: "POST", path: "/v1/responses", body: JSON.stringify({ model: "codex-test" }) }); expect(result.status).toBe(503); expect(primaryFailures).toBe(1); expect(fallbackHits).toBe(0); + expect(charged).toBe(1); }); test("primary 4xx is relayed without declaring the primary globally failed", async () => { @@ -216,30 +218,6 @@ test("streaming primary output is passed through once and client close does not expect(fallbackHits).toBe(0); }); -test("disconnecting a streaming client aborts upstream without fallback replay", async () => { - let primaryHits = 0; - const primary = await listen((_req, res) => { - primaryHits += 1; - res.writeHead(200, { "content-type": "text/event-stream" }); - res.write("data: first\n\n"); - setTimeout(() => res.write("data: later\n\n"), 100); - }); - let fallbackHits = 0; - const fallback = await listen((_req, res) => { fallbackHits += 1; res.end("fallback"); }); - const instance = await gateway(primary.port, fallback.port); - await new Promise((resolve, reject) => { - const req = httpRequest({ host: "127.0.0.1", port: instance.port, method: "POST", path: "/v1/responses", - headers: { host: `127.0.0.1:${instance.port}`, "content-type": "application/json" } }, response => { - response.once("data", () => { response.destroy(); resolve(); }); - }); - req.once("error", reject); - req.end(JSON.stringify({ model: "codex-test", stream: true })); - }); - await Bun.sleep(50); - expect(primaryHits).toBe(1); - expect(fallbackHits).toBe(0); -}); - test("request ceiling and browser or Host requests are denied locally", async () => { const primary = await listen((_req, res) => res.end("unexpected")); const fallback = await listen((_req, res) => res.end("unexpected")); @@ -322,6 +300,87 @@ test("a real Node child relays GET models through both primary and fallback", as await nodeGatewayGetProof(); }); +// The guardian runs under Node, where a torn-down upstream surfaces as an aborted +// or reset response. Bun does not emit those events, so only a child process can +// tell a client hang-up apart from a primary that actually failed. +async function nodeGatewayCancelProof() { + const gatewayPath = JSON.stringify(repoPath("scripts", "ocx-recovery-guardian", "gateway.cjs")); + const script = ` + const http = require("node:http"); + const { createGateway } = require(${gatewayPath}); + const listen = handler => new Promise(resolve => { + const server = http.createServer(handler); + server.listen(0, "127.0.0.1", () => resolve(server)); + }); + const close = server => new Promise(resolve => { server.close(resolve); server.closeAllConnections?.(); }); + (async () => { + let charged = 0; + let mode = "hold"; + let primary = null; + let gateway = null; + try { + primary = await listen((_req, res) => { + res.writeHead(200, { "content-type": "text/event-stream" }); + res.write("data: first\\n\\n"); + if (mode === "abort") res.socket.destroy(); + }); + gateway = await createGateway({ + port: 0, + primaryOrigin: "http://127.0.0.1:" + primary.address().port, + fallbackOrigin: "http://127.0.0.1:9", + models: {}, readFallbackKey: async () => "fixed-test-key", isPrimaryReady: () => true, + isStopped: () => false, onPrimaryFailure: () => { charged += 1; }, log: () => {}, + }); + await new Promise((resolve, reject) => { + const request = http.request({ host: "127.0.0.1", port: gateway.port, method: "POST", path: "/v1/responses", + headers: { host: "127.0.0.1:" + gateway.port, "content-type": "application/json" } }, response => { + response.once("data", () => { response.destroy(); resolve(); }); + }); + request.on("error", reject); + request.end(JSON.stringify({ model: "codex-test", stream: true })); + }); + await new Promise(resolve => setTimeout(resolve, 400)); + if (charged !== 0) throw new Error("a client cancel charged " + charged + " primary failure(s)"); + // Positive control: the same counter must dare to move, or the line above + // only proves that nothing ever charges it. + mode = "abort"; + await new Promise(resolve => { + const again = http.request({ host: "127.0.0.1", port: gateway.port, method: "POST", path: "/v1/responses", + headers: { host: "127.0.0.1:" + gateway.port, "content-type": "application/json" } }, response => { + response.resume(); + response.once("end", resolve); + }); + again.on("error", resolve); + again.end(JSON.stringify({ model: "codex-test", stream: true })); + }); + await new Promise(resolve => setTimeout(resolve, 400)); + if (charged === 0) throw new Error("an upstream abort charged nothing, so the cancel assertion above is vacuous"); + } finally { + // Always release the sockets: a child that hangs on a leaked stream would + // report this check as a timeout instead of as the failure it is. + if (gateway) await gateway.close(); + if (primary) await close(primary); + } + })().catch(error => { console.error(error && error.message || error); process.exitCode = 1; }); + `; + await new Promise((resolve, reject) => { + const child = spawn("node.exe", ["-e", script], { stdio: ["ignore", "ignore", "pipe"], windowsHide: true }); + let stderr = ""; + const timer = setTimeout(() => child.kill(), 20_000); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", chunk => { stderr += chunk; }); + child.once("error", error => { clearTimeout(timer); reject(error); }); + child.once("close", code => { + clearTimeout(timer); + if (code === 0) resolve(); else reject(new Error(`Node gateway cancel proof failed (${code}): ${stderr}`)); + }); + }); +} + +test("a real Node child keeps the primary ready across a client cancel", async () => { + await nodeGatewayCancelProof(); +}); + test("WebSocket upgrades receive 426 rather than a proxied connection", async () => { const primary = await listen((_req, res) => res.end("primary")); const fallback = await listen((_req, res) => res.end("fallback")); diff --git a/tests/windows/windows-recovery-intent.test.ts b/tests/windows/windows-recovery-intent.test.ts index 75891d4457a..c62501615af 100644 --- a/tests/windows/windows-recovery-intent.test.ts +++ b/tests/windows/windows-recovery-intent.test.ts @@ -1,9 +1,10 @@ import { afterEach, describe, expect, test } from "bun:test"; -import { copyFileSync, existsSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { writeRecoveryIntentIfGuardianEnabled } from "../../src/lib/recovery-intent"; +import { writeRecoveryIntentIfGuardianEnabled, backupRecoveryIntent, restoreRecoveryIntent } from "../../src/lib/recovery-intent"; +import { parseIntent } from "../../scripts/ocx-recovery-guardian/main.cjs"; import { repoPath } from "../helpers/repo-root"; const fixtures: string[] = []; @@ -92,16 +93,155 @@ describe("persistent manual recovery intent", () => { const ps = join(process.env.SystemRoot ?? "C:\\Windows", "System32", "WindowsPowerShell", "v1.0", "powershell.exe"); const quote = (value: string) => `'${value.replaceAll("'", "''")}'`; const intent = join(home, "recovery-intent.json"); - const code = `$null = & ${quote(helper)} -OpenCodexHome ${quote(home)} -Mode stopped; if (Test-Path -LiteralPath ${quote(intent)}) { exit 11 }; Set-Content -LiteralPath ${quote(join(home, "recovery-guardian.json"))} -Value '{"version":1,"enabled":true}' -NoNewline; Set-Content -LiteralPath ${quote(intent)} -Value '{"version":1,"mode":"stopped","at":1}' -NoNewline; $null = & ${quote(helper)} -OpenCodexHome ${quote(home)} -Mode maintenance -Until 77; $maintenance = Get-Content -LiteralPath ${quote(intent)} -Raw | ConvertFrom-Json; $null = & ${quote(helper)} -OpenCodexHome ${quote(home)} -Mode running; $running = Get-Content -LiteralPath ${quote(intent)} -Raw | ConvertFrom-Json; $null = & ${quote(helper)} -OpenCodexHome ${quote(home)} -Mode stopped; $stopped = Get-Content -LiteralPath ${quote(intent)} -Raw | ConvertFrom-Json; $temps = @(Get-ChildItem -LiteralPath ${quote(home)} -Filter '.recovery-intent.*.tmp'); @($maintenance, $running, $stopped, $temps.Count) | ConvertTo-Json -Compress`; + const until = Date.now() + 60_000; + const code = `$null = & ${quote(helper)} -OpenCodexHome ${quote(home)} -Mode stopped; if (Test-Path -LiteralPath ${quote(intent)}) { exit 11 }; Set-Content -LiteralPath ${quote(join(home, "recovery-guardian.json"))} -Value '{"version":1,"enabled":true}' -NoNewline; Set-Content -LiteralPath ${quote(intent)} -Value '{"version":1,"mode":"stopped","at":1}' -NoNewline; $null = & ${quote(helper)} -OpenCodexHome ${quote(home)} -Mode maintenance -Until ${until}; $maintenance = Get-Content -LiteralPath ${quote(intent)} -Raw | ConvertFrom-Json; $null = & ${quote(helper)} -OpenCodexHome ${quote(home)} -Mode running; $running = Get-Content -LiteralPath ${quote(intent)} -Raw | ConvertFrom-Json; $null = & ${quote(helper)} -OpenCodexHome ${quote(home)} -Mode stopped; $stopped = Get-Content -LiteralPath ${quote(intent)} -Raw | ConvertFrom-Json; $temps = @(Get-ChildItem -LiteralPath ${quote(home)} -Filter '.recovery-intent.*.tmp'); @($maintenance, $running, $stopped, $temps.Count) | ConvertTo-Json -Compress`; const run = Bun.spawnSync([ps, "-NoProfile", "-NonInteractive", "-Command", code], { stdout: "pipe", stderr: "pipe", timeout: 15_000 }); expect(run.exitCode, Buffer.from(run.stderr).toString()).toBe(0); const [maintenance, running, stopped, tempCount] = JSON.parse(Buffer.from(run.stdout).toString()) as [{ version: number; mode: string; until?: number }, { mode: string }, { mode: string }, number]; - expect(maintenance).toMatchObject({ version: 1, mode: "maintenance", until: 77 }); + expect(maintenance).toMatchObject({ version: 1, mode: "maintenance", until }); expect(running).toMatchObject({ version: 1, mode: "running" }); expect(stopped).toMatchObject({ version: 1, mode: "stopped" }); expect(tempCount).toBe(0); }, 20_000); + test("the PowerShell helper fails closed on a reparse point and on an out-of-contract maintenance window", () => { + if (process.platform !== "win32") return; + const root = homeFixture(); + const helper = repoPath("scripts/ocx-recovery-guardian/intent.ps1"); + const ps = join(process.env.SystemRoot ?? "C:\\Windows", "System32", "WindowsPowerShell", "v1.0", "powershell.exe"); + const quote = (value: string) => `'${value.replaceAll("'", "''")}'`; + const invoke = (home: string, command: string) => Bun.spawnSync( + [ps, "-NoProfile", "-NonInteractive", "-Command", `$ErrorActionPreference='Stop'; & ${quote(helper)} -OpenCodexHome ${quote(home)} ${command}`], + { stdout: "pipe", stderr: "pipe", timeout: 15_000 }); + const intentOf = (home: string) => join(home, "recovery-intent.json"); + + // A reparse-point marker must refuse the write. Windows file symlinks need + // developer-mode rights, so fall back to a directory junction: it carries + // the same ReparsePoint attribute bit the guard tests. + const home = join(root, "home"); + mkdirSync(home); + const outside = join(root, "outside.json"); + writeFileSync(outside, '{"version":1,"enabled":true}'); + let reparseHome = home; + try { + symlinkSync(outside, join(home, "recovery-guardian.json"), "file"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EPERM") throw error; + symlinkSync(home, join(root, "linked-home"), "junction"); + writeFileSync(join(home, "recovery-guardian.json"), '{"version":1,"enabled":true}'); + reparseHome = join(root, "linked-home"); + } + const refused = invoke(reparseHome, "-Mode stopped"); + expect(refused.exitCode, Buffer.from(refused.stdout).toString()).not.toBe(0); + expect(existsSync(intentOf(home))).toBe(false); + + const enabled = join(root, "enabled"); + mkdirSync(enabled); + writeFileSync(join(enabled, "recovery-guardian.json"), '{"version":1,"enabled":true}'); + // parseIntent in main.cjs decodes any intent outside at < until <= at+180000 + // as 'stopped', which fences every request, so the writer must reject it. + for (const until of [77, Date.now() - 1, Date.now() + 400_000]) { + const rejected = invoke(enabled, `-Mode maintenance -Until ${until}`); + expect(rejected.exitCode, `until=${until}`).not.toBe(0); + expect(existsSync(intentOf(enabled)), `until=${until}`).toBe(false); + } + expect(invoke(enabled, `-Mode maintenance -Until ${Date.now() + 60_000}`).exitCode).toBe(0); + expect(JSON.parse(readFileSync(intentOf(enabled), "utf8"))).toMatchObject({ mode: "maintenance" }); + expect(invoke(enabled, "-Mode running").exitCode).toBe(0); + expect(invoke(enabled, `-Mode running -Until ${Date.now() + 60_000}`).exitCode).not.toBe(0); + }, 30_000); + + test("a refused stop restores the durable intent exactly as it found it", async () => { + const home = homeFixture(); + enableGuardian(home); + const intentPath = join(home, "recovery-intent.json"); + const found = { version: 1, mode: "running", at: 5 }; + writeFileSync(intentPath, JSON.stringify(found) + "\n"); + const backup = backupRecoveryIntent(home); + expect(await writeRecoveryIntentIfGuardianEnabled("stopped", { home, at: 6 })).toBe(true); + expect(JSON.parse(readFileSync(intentPath, "utf8")).mode).toBe("stopped"); + expect(await restoreRecoveryIntent(backup)).toBe(true); + expect(JSON.parse(readFileSync(intentPath, "utf8"))).toEqual(found); + // A home that had no intent file still has none afterwards: the rollback must not + // invent an instruction this run never found, and must not delete one it did. + const fresh = homeFixture(); + enableGuardian(fresh); + const absent = backupRecoveryIntent(fresh); + await writeRecoveryIntentIfGuardianEnabled("stopped", { home: fresh, at: 7 }); + expect(existsSync(join(fresh, "recovery-intent.json"))).toBe(true); + expect(await restoreRecoveryIntent(absent)).toBe(true); + expect(existsSync(join(fresh, "recovery-intent.json"))).toBe(false); + // No guardian means nothing was written, so there is nothing to put back. + const plain = homeFixture(); + expect(backupRecoveryIntent(plain)).toBeNull(); + expect(await restoreRecoveryIntent(null)).toBe(true); + // Every write here hardens the file's ACL through a real icacls spawn, so the wait is + // intrinsic to the assertion; this file already budgets its spawn-bound cases this way. + }, 20_000); + + test("every refusal site that answers 'nothing was changed' rolls the intent back", () => { + const cli = readFileSync(repoPath("src/cli/index.ts"), "utf8"); + const api = readFileSync(repoPath("src/server/management-api.ts"), "utf8"); + expect(cli).toMatch(/stopIntent = backupRecoveryIntent\(\);\s*\n\s*await writeRecoveryIntentIfGuardianEnabled\("stopped"\)/); + // `runtimeDown` is the stop's own evidence that a proxy of this home is still serving. + expect(cli).toMatch(/if \(stopIntent && !outcome\.summary\.runtimeDown && !await restoreRecoveryIntent\(stopIntent\)\)/); + const stop = api.slice(api.indexOf('if (url.pathname === "/api/stop"'), api.indexOf('if (url.pathname.startsWith("/api/native-main-profiles")')); + expect(stop).toContain("stopIntent = backupRecoveryIntent();"); + expect(stop).toContain("return await refuseStop("); + // A bare 409 from this route means the durable `stopped` outlived a proxy that never + // stopped, which fences the whole home through the guardian gateway. + expect(stop).not.toContain(", 409, req, config)"); + }); + + test("writer, reader and action script agree on every maintenance-window cell", async () => { + // One contract, three parties: at < until <= at + 180000. Anything else decodes as + // `stopped` in the guardian reader, which fences all gateway traffic. + const at = Date.now() - 1000; + const cells: Array<[string, number | undefined, boolean]> = [ + ["until missing", undefined, false], + ["until <= at", at, false], + ["until > at+180000", at + 180_001, false], + ["until == at+180000", at + 180_000, true], + ]; + const bodies: string[] = []; + for (const [label, until, accepted] of cells) { + const home = homeFixture(); + enableGuardian(home); + const intent: { version: number; mode: string; at: number; until?: number } = { version: 1, mode: "maintenance", at }; + if (until !== undefined) intent.until = until; + bodies.push(JSON.stringify(intent)); + const wrote = await writeRecoveryIntentIfGuardianEnabled("maintenance", { home, at, until }).then(() => true, () => false); + expect(wrote, `writer rejected ${label}`).toBe(accepted); + expect(parseIntent(intent, at + 1000).valid, `reader on ${label}`).toBe(accepted); + } + if (process.platform !== "win32") return; + const source = readFileSync(repoPath("scripts/ocx-recovery-guardian/windows-action.ps1"), "utf8"); + const start = source.indexOf("function Read-RecoveryIntent"); + const end = source.indexOf("function Test-CurrentRunningIntent", start); + expect(start).toBeGreaterThan(-1); + expect(end).toBeGreaterThan(start); + const directory = homeFixture(); + const script = join(directory, "cells.ps1"); + const quote = (value: string) => `'${value.replaceAll("'", "''")}'`; + writeFileSync(script, [ + "$ErrorActionPreference='Stop'", + "Set-StrictMode -Version Latest", + "$IntentMaxBytes=16384", + "function Read-BoundedJson { param([string]$Path,[int]$MaximumBytes) Get-Content -LiteralPath $Path -Raw | ConvertFrom-Json }", + source.slice(start, end), + `$cellHome = ${quote(directory)}`, + `$bodies = @(${bodies.map(body => quote(body)).join(", ")})`, + "foreach ($b in $bodies) {", + " Set-Content -LiteralPath (Join-Path $cellHome 'recovery-intent.json') -NoNewline -Value $b", + " [Console]::Out.WriteLine((Read-RecoveryIntent -OpenCodexDirectory $cellHome).valid)", + "}", + ].join("\r\n")); + const ps = join(process.env.SystemRoot ?? "C:\\Windows", "System32", "WindowsPowerShell", "v1.0", "powershell.exe"); + const run = Bun.spawnSync([ps, "-NoProfile", "-NonInteractive", "-File", script], { stdout: "pipe", stderr: "pipe", timeout: 30_000 }); + expect(run.exitCode, Buffer.from(run.stderr).toString()).toBe(0); + expect(Buffer.from(run.stdout).toString().trim().toLowerCase().split(/\r?\n/)).toEqual(cells.map(([, , accepted]) => String(accepted))); + }, 40_000); + test("visible launcher and tray invoke the enabled recovery helper with named parameters", () => { if (process.platform !== "win32") return; const root = homeFixture(); @@ -133,14 +273,15 @@ describe("persistent manual recovery intent", () => { const launcherScript = join(root, "invoke-launcher-intent.ps1"); writeFileSync(launcherScript, `$ErrorActionPreference='Stop'\n$ProjectRoot=${quote(project)}\n${launcher.slice(launcherStart, launcherEnd)}\nSet-RecoveryIntent -OpenCodexDirectory ${quote(launcherHome)} -Mode stopped\n`); const trayScript = join(trayDir, "invoke-tray-intent.ps1"); - writeFileSync(trayScript, `$ErrorActionPreference='Stop'\n${tray.slice(trayStart, trayEnd)}\nSet-RecoveryIntent -OpenCodexHome ${quote(trayHome)} -Mode maintenance -Until 123\n`); + const trayUntil = Date.now() + 60_000; + writeFileSync(trayScript, `$ErrorActionPreference='Stop'\n${tray.slice(trayStart, trayEnd)}\nSet-RecoveryIntent -OpenCodexHome ${quote(trayHome)} -Mode maintenance -Until ${trayUntil}\n`); const ps = join(process.env.SystemRoot ?? "C:\\Windows", "System32", "WindowsPowerShell", "v1.0", "powershell.exe"); for (const script of [launcherScript, trayScript]) { const run = Bun.spawnSync([ps, "-NoProfile", "-NonInteractive", "-File", script], { stdout: "pipe", stderr: "pipe", timeout: 15_000 }); expect(run.exitCode, Buffer.from(run.stderr).toString()).toBe(0); } expect(JSON.parse(readFileSync(join(launcherHome, "recovery-intent.json"), "utf8"))).toMatchObject({ mode: "stopped" }); - expect(JSON.parse(readFileSync(join(trayHome, "recovery-intent.json"), "utf8"))).toMatchObject({ mode: "maintenance", until: 123 }); + expect(JSON.parse(readFileSync(join(trayHome, "recovery-intent.json"), "utf8"))).toMatchObject({ mode: "maintenance", until: trayUntil }); }, 30_000); test("the visible launcher CheckOnly path parses but never initializes a guardian or writes intent", () => { @@ -149,19 +290,35 @@ describe("persistent manual recovery intent", () => { const project = join(root, "project"); const home = join(root, "home"); const codex = join(root, "codex"); + const guardianDir = join(project, "scripts", "ocx-recovery-guardian"); + const nodePath = join(process.env.ProgramFiles ?? "C:\\Program Files", "nodejs", "node.exe"); + if (!existsSync(nodePath)) return; + mkdirSync(guardianDir, { recursive: true }); mkdirSync(join(project, "node_modules", "bun", "bin"), { recursive: true }); mkdirSync(join(project, "src", "cli"), { recursive: true }); mkdirSync(home, { recursive: true }); mkdirSync(codex, { recursive: true }); writeFileSync(join(project, "node_modules", "bun", "bin", "bun.exe"), "fixture"); writeFileSync(join(project, "src", "cli", "index.ts"), "// fixture\n"); + writeFileSync(join(guardianDir, "main.cjs"), "require('node:fs').writeFileSync(process.argv[3] + '.argv', JSON.stringify(process.argv.slice(1)));\n"); + copyFileSync(repoPath("scripts/ocx-recovery-guardian/intent.ps1"), join(guardianDir, "intent.ps1")); + // Without an approved marker there is no guardian to initialize and no intent to fence, + // so both absence assertions below would pass for the wrong reason. The ports are ones + // nothing listens on: a developer's real proxy on the default 10100 would let a + // mutation that skips the early exit still leave no intent, by exiting on health instead. + const marker = join(home, "recovery-guardian.json"); + writeFileSync(marker, JSON.stringify({ + version: 1, enabled: true, projectRoot: project, openCodexHome: home, codexHome: codex, + nodePath, listenPort: 31997, primaryPort: 31998, fallback: { models: { fixture: "fixture" } }, repair: {}, + })); const launcher = repoPath("scripts/windows-visible-proxy.ps1"); const ps = join(process.env.SystemRoot ?? "C:\\Windows", "System32", "WindowsPowerShell", "v1.0", "powershell.exe"); const run = Bun.spawnSync([ps, "-NoProfile", "-NonInteractive", "-File", launcher, - "-ProjectRoot", project, "-OpenCodexHome", home, "-CodexHome", codex, "-CheckOnly", "-NoPause"], + "-ProjectRoot", project, "-OpenCodexHome", home, "-CodexHome", codex, "-Port", "31998", "-CheckOnly", "-NoPause"], { stdout: "pipe", stderr: "pipe", timeout: 15_000 }); expect(run.exitCode, Buffer.from(run.stderr).toString()).toBe(0); expect(Buffer.from(run.stdout).toString()).toContain("check passed"); + expect(existsSync(marker + ".argv")).toBe(false); expect(existsSync(join(home, "recovery-intent.json"))).toBe(false); }, 20_000); diff --git a/tests/windows/windows-recovery-main.test.ts b/tests/windows/windows-recovery-main.test.ts index a3c35c34bf2..13bfb27e38e 100644 --- a/tests/windows/windows-recovery-main.test.ts +++ b/tests/windows/windows-recovery-main.test.ts @@ -1,4 +1,5 @@ import { afterEach, expect, test } from "bun:test"; +import { existsSync, mkdirSync, readdirSync, statSync, utimesSync, writeFileSync } from "node:fs"; import { mkdtemp, rm, readFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -38,6 +39,31 @@ test("concurrent state writes remain complete JSON without temporary-file collis expect(JSON.parse(await readFile(target, "utf8"))).toEqual({ n: 15 }); }); +test("the steady tick rewrites the budget file only when its content would change", async () => { + const f = await fixture(); + let now = 100000; + await atomicJson(f.intent, { version: 1, mode: "running", at: now }); + const guardian = await createGuardian(f.config, { now: () => now, policyOptions: { recoveryStableMs: 1000 }, alive: () => true, + inspect: async () => ({ ok: true, value: { owned: true, pid: 42, start: "ticks", launcherPid: 43, launcherStart: "ticks" } }), + probe: async () => ({ ok: true, pid: 42 }) }); + cleanup.push(guardian.close); + const budget = join(f.home, "recovery-budget.json"); + const epoch = new Date(1000); + await guardian.tick(); + const tickOnce = async () => { + // The written bytes are identical either way, so only the file identity can show + // whether the queue replaced it. An epoch mtime is far outside any granularity. + utimesSync(budget, epoch, epoch); + now += 1000; + await guardian.tick(); + return statSync(budget).mtimeMs; + }; + expect(await tickOnce()).toBe(1000); + expect(await tickOnce()).toBe(1000); + await atomicJson(f.intent, { version: 1, mode: "stopped", at: ++now }); + expect(await tickOnce()).not.toBe(1000); +}); + test("real guardian entrypoint recovers once, records failed receipt, dispatches GLM, honors manual stop", async () => { const f = await fixture(); let now = 100000, healthy = true, actions = 0, repairs = 0; @@ -68,6 +94,90 @@ test("real guardian entrypoint recovers once, records failed receipt, dispatches expect((await response.json()).error.code).toBe("gateway_stopped"); }); +test("an incident record stops accumulating once the write-only backlog outgrows the newest few", async () => { + const f = await fixture(); + let now = 100000, healthy = true, repairs = 0; + await atomicJson(f.intent, { version: 1, mode: "running", at: now }); + // Back-dated names sort before this fixture's clock (~00:02:04 after the drive below), + // so the fresh incident is the newest entry by construction. + const incidentsRoot = join(f.home, "recovery-incidents"); + const backlog = Array.from({ length: 25 }, (_, i) => `1970-01-01T00-00-${String(i + 1).padStart(2, "0")}-000Z-1`); + for (const name of backlog) mkdirSync(join(incidentsRoot, name), { recursive: true }); + // Anything the guardian did not name must neither spend the budget nor be deleted: one + // trailing-sorting entry would otherwise evict a real incident, and a `notes.txt` would + // silently shrink how much history survives. + mkdirSync(join(incidentsRoot, "zz-notes"), { recursive: true }); + writeFileSync(join(incidentsRoot, "notes.txt"), "keep me"); + const guardian = await createGuardian(f.config, { now: () => now, policyOptions: { startupGraceMs: 0 }, alive: () => true, + inspect: async () => ({ ok: true, value: { owned: true, pid: 42, start: "ticks", launcherPid: 43, launcherStart: "ticks" } }), + probe: async () => ({ ok: healthy, pid: 42 }), + action: async () => ({ ok: true, value: { action: "refused", reason: "stop-uncertain" } }), + repair: async () => { repairs++; return { outcome: "no_candidate", candidateCount: 0, requestCount: 1 }; } }); + cleanup.push(guardian.close); + await guardian.tick(); + now += 30000; await guardian.tick(); + healthy = false; + for (let n = 0; n < 12; n++) { now += 2000; await guardian.tick(); await guardian.drain(); } + await guardian.drain(); + expect(repairs).toBe(1); + + const kept = readdirSync(incidentsRoot); + const known = new Set(backlog); + const generated = kept.filter(name => known.has(name) || /^\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z-\d+$/.test(name)); + expect(generated).toHaveLength(20); + expect(kept).toContain("zz-notes"); + expect(kept).toContain("notes.txt"); + expect(generated.filter(name => !known.has(name))).toHaveLength(1); + expect(generated).toContain(backlog[24]); + expect(generated).not.toContain(backlog[0]); + expect(generated).not.toContain(backlog[4]); +}); + +test("a corroborated generation is re-resolved before its launcher identity can go stale", async () => { + const f = await fixture(); + let now = 100000, inspections = 0; + await atomicJson(f.intent, { version: 1, mode: "running", at: now }); + // `uptime` grows with the fake clock so the boot instant derived from it stays constant. + // Without that, advancing `now` alone would break corroboration and the test would pass + // whether or not the decay bound exists — it proved vacuous both ways on the first run. + const guardian = await createGuardian(f.config, { now: () => now, policyOptions: { startupGraceMs: 0 }, alive: () => true, + inspect: async () => { inspections += 1; return { ok: true, value: { owned: true, pid: 42, start: "ticks", launcherPid: 43, launcherStart: "ticks" } }; }, + probe: async () => ({ ok: true, pid: 42, uptime: (now - 70000) / 1000 }) }); + cleanup.push(guardian.close); + await guardian.tick(); + expect(inspections).toBe(1); + // Same pid, same boot instant, and the budget spent: without a decay bound this is the + // state that freezes `snapshot` forever, so a launcher that exited behind us would keep + // handing `recover()` an expected pid that no longer exists. + now += 61_000; await guardian.tick(); + expect(inspections).toBe(2); + // The refresh re-arms adoption, so the next tick is settled again rather than inspecting + // once per tick the way the pre-fix code did. + now += 2000; await guardian.tick(); + expect(inspections).toBe(2); +}); + +test("the opening inspection is adopted, not paid for twice, and a new pid is still inspected", async () => { + const f = await fixture(); + let now = 100000, inspections = 0, healthPid = 42; + await atomicJson(f.intent, { version: 1, mode: "running", at: now }); + const guardian = await createGuardian(f.config, { now: () => now, policyOptions: { startupGraceMs: 0 }, alive: () => true, + inspect: async () => { inspections += 1; return { ok: true, value: { owned: true, pid: 42, start: "ticks", launcherPid: 43, launcherStart: "ticks" } }; }, + probe: async () => ({ ok: true, pid: healthPid, uptime: 30 }) }); + cleanup.push(guardian.close); + expect(inspections).toBe(1); + await guardian.tick(); + // The boot instant derived from this probe corroborates the snapshot the opening + // inspection already took, so a second ~5 s spawn here would be pure waste. + expect(inspections).toBe(1); + now += 2000; await guardian.tick(); + expect(inspections).toBe(1); + // A pid change is a different process generation and must still be resolved. + healthPid = 44; + now += 2000; await guardian.tick(); + expect(inspections).toBe(2); +}); + test("first start and new running intent require a fresh stable-ready interval", async () => { const f = await fixture(); let now = 100000; @@ -141,6 +251,52 @@ test("same PID is not ownership proof when the process identity changes", async expect(guardian.state().primaryReady).toBe(true); }); +test("a corroborated healthy generation is not re-inspected and a pid reuse wrap is", async () => { + const f = await fixture(); + const BOOT = 50000; + let now = 100000, owned = true, boot = BOOT, inspections = 0; + await atomicJson(f.intent, { version: 1, mode: "running", at: now }); + const guardian = await createGuardian(f.config, { now: () => now, policyOptions: { recoveryStableMs: 1000 }, alive: () => true, + inspect: async () => { inspections += 1; return { ok: true, value: { owned, pid: 42, start: `${boot}-ticks`, launcherPid: 43, launcherStart: "ticks" } }; }, + probe: async () => ({ ok: true, pid: 42, uptime: (now - boot) / 1000 }) }); + cleanup.push(guardian.close); + await guardian.tick(); now += 1000; await guardian.tick(); + expect(guardian.state().primaryReady).toBe(true); + const steady = inspections; + now += 1000; await guardian.tick(); now += 1000; await guardian.tick(); + // The WMI inspection costs a PowerShell spawn, and a healthy steady state on the same + // process generation must not pay one every couple of seconds. + expect(inspections).toBe(steady); + expect(guardian.state().primaryReady).toBe(true); + // Windows hands the same pid to a new process: the answer on the port is identical and + // only the boot instant says otherwise, so an unowned generation cannot stay admitted. + owned = false; boot = now - 500; now += 1000; + await guardian.tick(); + expect(inspections).toBe(steady + 1); + expect(guardian.state().primaryReady).toBe(false); + expect(guardian.state().state).toBe("foreign"); +}); + +test("a fresh running intent clears a stale uncertain-stop block", async () => { + const f = await fixture(); + let now = 100000; + const blockedFile = join(f.home, "recovery-blocked.json"); + await atomicJson(blockedFile, { version: 1, at: 1, reason: "stop_result_uncertain" }); + await atomicJson(f.intent, { version: 1, mode: "running", at: now }); + const guardian = await createGuardian(f.config, { now: () => now, alive: () => true, + inspect: async () => ({ ok: true, value: { owned: true, pid: 42, start: "ticks", launcherPid: 43, launcherStart: "ticks" } }), + probe: async () => ({ ok: true, pid: 42 }) }); + cleanup.push(guardian.close); + await guardian.tick(); + // The intent this companion started on is not new authorization: it may be the + // very transaction the previous process died inside of. + expect(guardian.state().recoveryBlocked).toBe(true); + await atomicJson(f.intent, { version: 1, mode: "running", at: ++now }); + await guardian.tick(); + expect(guardian.state().recoveryBlocked).toBe(false); + expect(existsSync(blockedFile)).toBe(false); +}); + test("manual stop during asynchronous diagnosis preparation prevents external dispatch", async () => { const f = await fixture(); let now = 100000, repairs = 0; @@ -198,6 +354,26 @@ test("observation errors withdraw primary admission", async () => { expect(guardian.state().primaryReady).toBe(false); }); +test("an unchanged observation does not rewrite the status file", async () => { + const f = await fixture(); + let now = 100000; + await atomicJson(f.intent, { version: 1, mode: "running", at: now }); + const guardian = await createGuardian(f.config, { now: () => now, policyOptions: { recoveryStableMs: 1000 }, alive: () => true, + inspect: async () => ({ ok: true, value: { owned: true, pid: 42, start: "ticks", launcherPid: 43, launcherStart: "ticks" } }), + probe: async () => ({ ok: true, pid: 42 }) }); + cleanup.push(guardian.close); + const statusFile = join(f.home, "recovery-status.json"); + await guardian.tick(); now += 1000; await guardian.tick(); + expect(guardian.state().primaryReady).toBe(true); + const written = await readFile(statusFile, "utf8"); + now += 1000; await guardian.tick(); now += 1000; await guardian.tick(); + expect(await readFile(statusFile, "utf8")).toBe(written); + // What the file records still reaches disk the moment it changes. + await atomicJson(f.intent, { version: 1, mode: "stopped", at: ++now }); + await guardian.tick(); + expect(JSON.parse(await readFile(statusFile, "utf8")).state).toBe("stopped"); +}); + test("companion restart cannot replay a previously in-flight recovery command", async () => { const f = await fixture(); const now = 100000; diff --git a/tests/windows/windows-recovery-ownership.test.ts b/tests/windows/windows-recovery-ownership.test.ts index 45c6d3b0236..5f928ee9c28 100644 --- a/tests/windows/windows-recovery-ownership.test.ts +++ b/tests/windows/windows-recovery-ownership.test.ts @@ -70,26 +70,74 @@ describe("Windows recovery guardian action ownership", () => { expect(start).toBeGreaterThan(collision); }); - test("refuses malformed manual intent in an isolated fake project without invoking a child", () => { + test("refuses a malformed or field-less manual intent in an isolated fake project without invoking a child", () => { if (process.platform !== "win32") return; + // StrictMode turns a missing `version`/`mode`/`at` into a terminating + // PropertyNotFound, and the early Recover read runs outside any try: an + // unreadable intent must still produce one structured receipt line. + for (const body of ["{ not-json", '{"mode":"running","at":100}', '{"version":1,"at":100}', '{"version":1,"mode":"running"}', '{"version":1,"mode":"running","at":"soon"}']) { + const fixture = makeFixture(); + try { + writeFileSync(join(fixture.home, "recovery-intent.json"), body); + const run = Bun.spawnSync([ + join(process.env.SystemRoot ?? "C:\\Windows", "System32", "WindowsPowerShell", "v1.0", "powershell.exe"), + "-NoProfile", "-NonInteractive", "-File", actionPath, + "-Mode", "Recover", "-ProjectRoot", fixture.project, + "-OpenCodexHome", fixture.home, "-CodexHome", fixture.codex, + "-Port", "18991", "-ExpectedPid", "424242", "-ExpectedStart", "1", + "-ExpectedLauncherPid", "424243", "-ExpectedLauncherStart", "1", + ], { stdout: "pipe", stderr: "pipe", timeout: 15_000 }); + expect(run.exitCode, `${body}: ${Buffer.from(run.stderr).toString()}`).toBe(0); + const result = JSON.parse(Buffer.from(run.stdout).toString()) as { action: string; reason: string }; + expect(result).toMatchObject({ action: "refused", reason: "invalid-intent" }); + } finally { + rmSync(fixture.root, { recursive: true, force: true }); + } + } + }, 60_000); + + test("quotes a native launcher argument so a trailing backslash cannot merge parameters", () => { + if (process.platform !== "win32") return; + const source = readFileSync(actionPath, "utf8"); + const start = source.indexOf("function ConvertTo-NativeArgument"); + const end = source.indexOf("function Invoke-GracefulProjectStop", start); + expect(start).toBeGreaterThan(-1); + expect(end).toBeGreaterThan(start); + // The graceful-stop command line is the second consumer of the same quoting. + expect(source).toContain("$info.Arguments = ('{0} stop' -f (ConvertTo-NativeArgument $CliPath))"); const fixture = makeFixture(); + const ps = join(process.env.SystemRoot ?? "C:\\Windows", "System32", "WindowsPowerShell", "v1.0", "powershell.exe"); + const quote = (value: string) => `'${value.replaceAll("'", "''")}'`; + const child = join(fixture.root, "child.ps1"); + writeFileSync(child, "param([string]$ProjectRoot = 'MISSING', [int]$Port = -1)\n[Console]::Out.WriteLine((@{ ProjectRoot = $ProjectRoot; Port = $Port } | ConvertTo-Json -Compress))\n"); + const runner = join(fixture.root, "run.ps1"); + const out = join(fixture.root, "out.txt"); + // Get-FullPath keeps the separator for a rooted path, which is the shape the + // old hand-rolled quoting turned into one merged argument. + const trailing = join(fixture.project, "scripts") + "\\"; + writeFileSync(runner, [ + `$ErrorActionPreference='Stop'`, + `Set-StrictMode -Version Latest`, + source.slice(start, end), + `$path = ${quote(trailing)}`, + `$line = '-NoProfile -NonInteractive -File ' + (ConvertTo-NativeArgument ${quote(child)}) + ' -ProjectRoot ' + (ConvertTo-NativeArgument $path) + ' -Port 10100'`, + `Start-Process -FilePath ${quote(ps)} -ArgumentList $line -RedirectStandardOutput ${quote(out)} -NoNewWindow -Wait`, + `$rejected = $true`, + `try { ConvertTo-NativeArgument 'a"b' | Out-Null; $rejected = $false } catch { }`, + `@($path, ('' + (Get-Content -LiteralPath ${quote(out)} -Raw)), $rejected) | ConvertTo-Json -Compress`, + ``, + ].join("\n")); try { - writeFileSync(join(fixture.home, "recovery-intent.json"), "{ not-json"); - const run = Bun.spawnSync([ - join(process.env.SystemRoot ?? "C:\\Windows", "System32", "WindowsPowerShell", "v1.0", "powershell.exe"), - "-NoProfile", "-NonInteractive", "-File", actionPath, - "-Mode", "Recover", "-ProjectRoot", fixture.project, - "-OpenCodexHome", fixture.home, "-CodexHome", fixture.codex, - "-Port", "18991", "-ExpectedPid", "424242", "-ExpectedStart", "1", - "-ExpectedLauncherPid", "424243", "-ExpectedLauncherStart", "1", - ], { stdout: "pipe", stderr: "pipe", timeout: 15_000 }); + const run = Bun.spawnSync([ps, "-NoProfile", "-NonInteractive", "-File", runner], { stdout: "pipe", stderr: "pipe", timeout: 30_000 }); expect(run.exitCode, Buffer.from(run.stderr).toString()).toBe(0); - const result = JSON.parse(Buffer.from(run.stdout).toString()) as { action: string; reason: string }; - expect(result).toMatchObject({ action: "refused", reason: "invalid-intent" }); + const [sent, received, rejected] = JSON.parse(Buffer.from(run.stdout).toString()) as [string, string, boolean]; + const bound = JSON.parse(received.trim()) as { ProjectRoot: string; Port: number }; + expect(bound).toEqual({ ProjectRoot: sent, Port: 10100 }); + expect(rejected).toBe(true); } finally { rmSync(fixture.root, { recursive: true, force: true }); } - }, 20_000); + }, 45_000); test("uses the shared version/at intent envelope rather than a divergent schema", async () => { const source = await actionSource(); diff --git a/tests/windows/windows-recovery-policy.test.ts b/tests/windows/windows-recovery-policy.test.ts index c7fe4df128e..5a04bd00143 100644 --- a/tests/windows/windows-recovery-policy.test.ts +++ b/tests/windows/windows-recovery-policy.test.ts @@ -53,7 +53,7 @@ test("manual stop, dead launcher, foreign identity, and unknown ownership never expect(launcherGone.observe({ ...unavailable, launcherAlive: false }, 0)).toMatchObject({ state: "stopped", action: "none" }); const foreign = new RecoveryPolicy({ startupGraceMs: 0 }); - expect(foreign.observe({ ...unavailable, owned: false, identityChanged: true }, 0)).toMatchObject({ state: "foreign", useFallback: true, action: "none" }); + expect(foreign.observe({ ...unavailable, owned: false }, 0)).toMatchObject({ state: "foreign", useFallback: true, action: "none" }); expect(foreign.observe({ ...unavailable, owned: false }, 60_000)).toMatchObject({ state: "foreign", action: "none" }); const unknown = new RecoveryPolicy({ startupGraceMs: 0 }); diff --git a/tests/windows/windows-recovery-repair.test.ts b/tests/windows/windows-recovery-repair.test.ts index b3e65738832..2a2c2585dc5 100644 --- a/tests/windows/windows-recovery-repair.test.ts +++ b/tests/windows/windows-recovery-repair.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, test } from "bun:test"; -import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, lstatSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { tmpdir } from "node:os"; import { repoPath } from "../helpers/repo-root"; @@ -250,4 +250,24 @@ describe("ocx recovery guardian isolated GLM repair", () => { expect(readFileSync(candidate, "utf8")).toBe("keep-existing-candidate\n"); expect(readFileSync(join(paths.projectRoot, "src", "lib", "runtime-diagnostics.ts"), "utf8")).toBe("export const value = 1;\n"); }); + + test.if(process.platform === "win32")("refuses an incident directory reached through a junction, before any request is sent", async () => { + const paths = fixture(); + const via = join(dirname(paths.incidentDir), "linked-incident"); + symlinkSync(paths.incidentDir, via, "junction"); + // Pins the refusal and the runtime fact that makes the `isSymbolicLink()` walk enough + // here: bun 1.3.14 reports a junction as a symlink. If a future runtime calls it a + // plain directory, this assertion goes red first and the guard then needs the realpath + // comparison that was measured unnecessary on 2026-09-24. + expect(lstatSync(via).isSymbolicLink()).toBe(true); + let asked = 0; + const result = await request({ + projectRoot: paths.projectRoot, + incidentDir: via, + fetchFn: async () => { asked += 1; return new Response("{}"); }, + }); + expect(result).toMatchObject({ outcome: "failed", failureClass: "SAFETY_REJECTED" }); + expect(asked).toBe(0); + expect(existsSync(join(via, "candidate"))).toBe(false); + }); }); diff --git a/tests/windows/windows-tray-restart-hardening.test.ts b/tests/windows/windows-tray-restart-hardening.test.ts index 5bf57dc1357..dfbb39a838f 100644 --- a/tests/windows/windows-tray-restart-hardening.test.ts +++ b/tests/windows/windows-tray-restart-hardening.test.ts @@ -1,9 +1,20 @@ import { describe, expect, test } from "bun:test"; -import { readFileSync } from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; import { join } from "node:path"; import { repoPath } from "../helpers/repo-root"; +import { writeRecoveryIntentIfGuardianEnabled } from "../../src/lib/recovery-intent"; const source = readFileSync(repoPath("src/tray/windows-tray.ps1"), "utf8"); +const cli = readFileSync(repoPath("src/cli/index.ts"), "utf8"); + +/** The declared body of one `src/cli/index.ts` function, for ordering assertions. */ +function cliFunction(name: string): string { + const start = cli.indexOf(`async function ${name}(`); + expect(start).toBeGreaterThan(-1); + const end = cli.indexOf("\nasync function ", start + 1); + return cli.slice(start, end < 0 ? cli.length : end); +} describe("Windows tray restart process hardening", () => { test("fails a pending action when tracked process state cannot be inspected", () => { @@ -32,4 +43,72 @@ describe("Windows tray restart process hardening", () => { expect(source).toContain('$stopProcess = Start-OcxCommand @("stop") -TrackExit'); expect(source).toContain('$script:pendingProcess = $stopProcess'); }); + + test("clears the restart maintenance fence unconditionally", () => { + // The guardian reader never compares `until` to now, so a fence left behind by a + // failed restart keeps this home from ever recovering on its own. + const restart = cliFunction("handleTrayProxyRestart"); + expect(restart).toContain('writeRecoveryIntentIfGuardianEnabled("maintenance"'); + expect(restart).toMatch(/finally\s*\{[\s\S]*?writeRecoveryIntentIfGuardianEnabled\("running"\)/); + expect(restart).not.toMatch(/if \(restarted\)/); + // The parameter that existed only to dodge this write is gone, and with it the + // second `running` signature one tray Start used to produce. + expect(cli).not.toContain("writeRunningIntent"); + expect(cliFunction("handleTrayProxyStart")).not.toContain("writeRecoveryIntentIfGuardianEnabled"); + }); + + test("re-affirms the running intent on every path that brings a proxy up", () => { + // Without this, a durable `stopped` from an earlier manual stop silently disables + // crash recovery for the whole life of the process this command started. + for (const name of ["handleStart", "handleEnsure"]) { + expect(cliFunction(name)).toContain('writeRecoveryIntentIfGuardianEnabled("running")'); + } + }); + + test("stop takes the ownership lease before it writes the intent, and fails cleanly", () => { + const stop = cliFunction("handleStop"); + expect(stop.indexOf("acquireOwnershipMutationLease(serviceStatePaths())")) + .toBeLessThan(stop.indexOf('writeRecoveryIntentIfGuardianEnabled("stopped")')); + // A malformed marker must reach the operator as a line, not as a stack out of the + // CLI top level with the proxy still running. + expect(stop).toMatch(/catch \(error\) \{[\s\S]*?Stop refused[\s\S]*?process\.exitCode = 1;[\s\S]*?ok: false/); + }); + + test("refuses a maintenance fence the guardian reader would reject", async () => { + const home = mkdtempSync(join(tmpdir(), "ocx-restart-fence-")); + try { + writeFileSync(join(home, "recovery-guardian.json"), JSON.stringify({ version: 1, enabled: true })); + const at = 1_760_000_000_000; + await expect(writeRecoveryIntentIfGuardianEnabled("maintenance", { home, at, until: at + 180_001 })) + .rejects.toThrow("Recovery intent maintenance deadline is invalid."); + expect(existsSync(join(home, "recovery-intent.json"))).toBe(false); + await expect(writeRecoveryIntentIfGuardianEnabled("maintenance", { home, at, until: at + 180_000 })) + .resolves.toBe(true); + } finally { + rmSync(home, { recursive: true, force: true }); + } + }); + + test("does not re-sign a running intent the tray already wrote", async () => { + // Every signature change restarts the guardian's stabilisation window, so a second + // `running` write charges the fallback gateway a fresh recoveryStableMs for a proxy + // that was already on its way. Proven against scripts/ocx-recovery-guardian/main.cjs. + const home = mkdtempSync(join(tmpdir(), "ocx-restart-running-")); + try { + writeFileSync(join(home, "recovery-guardian.json"), JSON.stringify({ version: 1, enabled: true })); + const intentPath = join(home, "recovery-intent.json"); + const at = 1_760_000_000_000; + await expect(writeRecoveryIntentIfGuardianEnabled("stopped", { home, at })).resolves.toBe(true); + await expect(writeRecoveryIntentIfGuardianEnabled("running", { home, at: at + 1000 })).resolves.toBe(true); + expect(JSON.parse(readFileSync(intentPath, "utf8"))).toEqual({ version: 1, mode: "running", at: at + 1000 }); + await expect(writeRecoveryIntentIfGuardianEnabled("running", { home, at: at + 2000 })).resolves.toBe(true); + expect(JSON.parse(readFileSync(intentPath, "utf8"))).toEqual({ version: 1, mode: "running", at: at + 1000 }); + // A fence still has to clear it. + await expect(writeRecoveryIntentIfGuardianEnabled("maintenance", { home, at: at + 3000, until: at + 6000 })) + .resolves.toBe(true); + expect(JSON.parse(readFileSync(intentPath, "utf8")).mode).toBe("maintenance"); + } finally { + rmSync(home, { recursive: true, force: true }); + } + }); }); diff --git a/tests/windows/windows-tray.test.ts b/tests/windows/windows-tray.test.ts index 58ee6b67f57..14892aca88c 100644 --- a/tests/windows/windows-tray.test.ts +++ b/tests/windows/windows-tray.test.ts @@ -10,6 +10,7 @@ import { writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; +import { createServer } from "node:net"; import { helperPath, repoPath, repoRoot } from "../helpers/repo-root"; import { join } from "node:path"; import { @@ -22,6 +23,7 @@ import { readWindowsTrayRunValueWithAsyncRunner, readWindowsTrayRunValueWithRunner, replaceWindowsTrayOwnedFile, + trayHomeRequiresRecoveryIntentHelper, windowsPowerShellPath, windowsTrayProcessArgs, windowsTrayRunValue, @@ -368,7 +370,7 @@ describe("Windows tray packaging and command safety", () => { expect(cli).toContain("isProxyReplacement(previous, live)"); expect(cli).toContain("process.exitCode = result.ok ? 0 : 1"); expect(cli).toContain("waitForProxy(40_000)"); - expect(cli).toContain("await handleProxyRestart(() => handleTrayProxyStart(false, false))"); + expect(cli).toContain("await handleProxyRestart(() => handleTrayProxyStart(false))"); expect(cli).toContain("function detachedStartEnvironment()"); expect(cli).toContain("delete env.OCX_SERVICE"); expect(cli).not.toContain("OCX_KEEP_ROUTING"); @@ -382,6 +384,126 @@ describe("Windows tray packaging and command safety", () => { expect(source).not.toContain("Stop-Process"); }); + test("writes the recovery intent before latching a pending menu action", () => { + // A refused intent write throws out of the click handler: PowerShell keeps the tray + // alive (probed on 5.1) but never runs the rest of the body, so a latch set first + // would lock these three items for its whole budget and then balloon a failure for + // an action the tray never dispatched. + const source = readFileSync(repoPath("src", "tray", "windows-tray.ps1"), "utf8"); + const start = source.indexOf('$startItem.add_Click'); + const stop = source.indexOf('$stopItem.add_Click'); + const restart = source.indexOf('$restartItem.add_Click'); + const logs = source.indexOf('$logsItem.add_Click'); + expect(start).toBeGreaterThan(-1); + expect(stop).toBeGreaterThan(start); + expect(logs).toBeGreaterThan(restart); + for (const [body, mode, action] of [ + [source.slice(start, stop), "running", "Start Proxy"], + [source.slice(stop, restart), "stopped", "Stop Proxy"], + ] as const) { + expect(body).toContain(`Set-RecoveryIntent -OpenCodexHome $OpenCodexHome -Mode "${mode}"`); + expect(body.indexOf("Set-RecoveryIntent")).toBeLessThan(body.indexOf(`Set-PendingAction "${action}"`)); + } + // The restart fence belongs to `ocx __tray-restart`, which signs it and clears it in + // the same function; a second signature here would reset primaryReady per restart. + expect(source.slice(restart, logs)).not.toContain("Set-RecoveryIntent"); + }); + + test("reads health status without a synchronous socket wait on the UI thread", () => { + const source = readFileSync(repoPath("src", "tray", "windows-tray.ps1"), "utf8"); + // GetResponse() measured 700-730 ms per 3 s tick on a home whose proxy is down or + // wedged (probe on PS 5.1): the request must go out on a task, and the tick must + // wait only briefly for it. + expect(source).not.toContain("$response = $request.GetResponse()"); + expect(source).toContain("$task = $request.GetResponseAsync()"); + expect(source).toContain("-not $task.Wait(100)"); + // The task-based call ignores Timeout, so the re-issue ceiling is what stops a + // wedged proxy being reported as online from the cache forever. + expect(source).toContain("-lt 3000"); + // The ceiling path is the only handle on a request nobody will observe again. + expect(source).toContain("$script:jsonRequest.Abort()"); + }); + + test("recovers after a faulted status read instead of going blind for good", async () => { + if (process.platform !== "win32") return; + const source = readFileSync(repoPath("src", "tray", "windows-tray.ps1"), "utf8"); + const lines = source.split(/\r?\n/); + // Execute the real function, extracted verbatim: a faulted task whose clearing sits + // below the throwing `Wait()` makes every later tick rethrow, so the tray reads + // Offline permanently and never notices the proxy coming back. + const start = lines.findIndex(line => line.startsWith("function Read-JsonUrl")); + expect(start).toBeGreaterThan(-1); + const end = lines.findIndex((line, index) => index > start && line === "}"); + expect(end).toBeGreaterThan(start); + const listener = createServer(); + await new Promise(resolve => listener.listen(0, "127.0.0.1", resolve)); + const port = (listener.address() as { port: number }).port; + await new Promise(resolve => listener.close(() => resolve())); + const root = mkdtempSync(join(tmpdir(), "ocx-tray-blindness-")); + try { + const refusing = "http://127.0.0.1:9/healthz"; + const live = `http://127.0.0.1:${port}/healthz`; + const script = join(root, "ticks.ps1"); + writeFileSync(script, [ + "$ErrorActionPreference = 'Stop'", + lines.slice(start, end + 1).join("\r\n"), + "$script:jsonTask = $null; $script:jsonRequest = $null; $script:jsonTaskStarted = 0L; $script:jsonPayload = $null", + "function Tick($u) { $t = ''; $v = $null; try { $v = Read-JsonUrl $u } catch { $t = $_.Exception.GetType().Name }; return [pscustomobject]@{ thrown = $t; value = $v } }", + "$throws = 0", + `for ($n = 1; $n -le 6; $n++) { if ((Tick '${refusing}').thrown) { $throws++ }; Start-Sleep -Milliseconds 250 }`, + "$listener = New-Object System.Net.HttpListener", + `$listener.Prefixes.Add('http://127.0.0.1:${port}/'); $listener.Start()`, + "$ctx = $listener.GetContextAsync(); $seen = 0", + `for ($n = 1; $n -le 10; $n++) { $r = Tick '${live}'; if ($r.thrown) { $throws++ }; if ($null -ne $r.value) { $seen++ }`, + " if ($ctx.Wait(400)) { $c = $ctx.Result; $b = [Text.Encoding]::UTF8.GetBytes('{\"status\":\"ok\",\"service\":\"opencodex\",\"port\":" + + port + "}')", + " $c.Response.OutputStream.Write($b, 0, $b.Length); $c.Response.Close(); $ctx = $listener.GetContextAsync() }", + " Start-Sleep -Milliseconds 250 }", + "$listener.Stop()", + "[Console]::Out.WriteLine(('THROWS={0} SEEN={1}' -f $throws, $seen))", + ].join("\r\n")); + const run = Bun.spawnSync( + [windowsPowerShellPath(), "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File", script], + { stdout: "pipe", stderr: "pipe", timeout: 60_000 }, + ); + const out = Buffer.from(run.stdout).toString(); + expect(run.exitCode, Buffer.from(run.stderr).toString()).toBe(0); + expect(out).toMatch(/THROWS=0 SEEN=[1-9]/); + } finally { + removeTreeWithRetry(root); + } + }, 90_000); + + test("derives the intent-helper ownership rule from the home, not the source tree", () => { + const home = mkdtempSync(join(tmpdir(), "ocx-tray-helper-")); + try { + // No marker: a legacy home, and the installed tray dispatches without a helper. + expect(trayHomeRequiresRecoveryIntentHelper(home)).toBe(false); + writeFileSync(join(home, "recovery-guardian.json"), JSON.stringify({ version: 1, enabled: false })); + expect(trayHomeRequiresRecoveryIntentHelper(home)).toBe(false); + writeFileSync(join(home, "recovery-guardian.json"), JSON.stringify({ version: 1, enabled: true })); + expect(trayHomeRequiresRecoveryIntentHelper(home)).toBe(true); + // A marker the tray script cannot read is fail-closed there too, so an install + // that cannot see scripts/ must not certify a helper it never installed. + writeFileSync(join(home, "recovery-guardian.json"), "{ nope"); + expect(trayHomeRequiresRecoveryIntentHelper(home)).toBe(true); + // A published install ships no scripts/ directory and therefore no helper: the same + // marker must not demand one, or the home is stale forever and reinstall cannot + // clear what status just insisted on. + expect(trayHomeRequiresRecoveryIntentHelper(home, false)).toBe(false); + writeFileSync(join(home, "recovery-guardian.json"), JSON.stringify({ version: 1, enabled: true })); + expect(trayHomeRequiresRecoveryIntentHelper(home, false)).toBe(false); + } finally { + removeTreeWithRetry(home); + } + const tray = readFileSync(repoPath("src", "tray", "windows.ts"), "utf8"); + expect(tray).toContain("trayHomeRequiresRecoveryIntentHelper() ? [installedTrayRecoveryIntentPath()]"); + expect(tray).not.toContain("existsSync(sourceTrayRecoveryIntentPath()) ? [installedTrayRecoveryIntentPath()]"); + // One rule for both sides: the copy is gated on the same shipped test status applies. + expect(tray).toContain("if (trayRecoveryIntentHelperShipped()) {"); + expect(tray).toContain("helperShipped = trayRecoveryIntentHelperShipped()"); + }); + test("tray reads restart safety through the CLI instead of the admin-gated /api endpoint", () => { const source = readFileSync(repoPath("src", "tray", "windows-tray.ps1"), "utf8"); // The management API is admin-token gated, and the tray runs without that token, From 184f75bff8a5b8f0f1cef5d55a5dc4c5bb4b8836 Mon Sep 17 00:00:00 2001 From: Codex Developer Date: Fri, 25 Sep 2026 02:07:55 +0800 Subject: [PATCH 5/7] fix(bridge): bound delivery receipts and stop carrying prompt text on context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review fixes on the carried ChatGPT bridge, each with a mutation-proven test. - store: an idempotent replay must survive pruning, so `pruneOperations` now refuses to drop receipts inside the `OPERATION_REPLAY_MS` window it exists to answer (the row cap still binds outside it), and the host-target uniqueness guard moved into a partial unique index because a SELECT-then-INSERT only held within one process. A chat conversation keeps its one binding row for life by schema; the guard mirrors that instead of filtering by lifecycle. - dsh host: an unclaimed delivery no longer holds the TARGET_ACTIVE gate forever. It now releases once the claim is older than the same 120 s stale window the core store uses, while a claimed-but-unfinished turn still refuses. - mcp client: a DevSpace restart retires the minted session id, so a 404 drops it and the next call re-initializes instead of failing for the life of the client; a shared SSE frame is matched by request id, not by which frame arrives first. - contracts/provider: retire the codes and the `replace` action this module no longer emits, add `BINDING_EXISTS`, hoist `MAX_PROMPT_CHARS` into the shared contract, and drop `promptPreview` from the turn context — a serialized slice of the conversation is request content, and any transport that logs the context would then log the body. - tests: pin the chat-side uniqueness guard (it had no discriminating assertion), and make the runTurn-only skip list derived-and-checked instead of a hand-maintained allowlist over seven registry-wide loops. Verification: `chatgpt-bridge-core` 17/0, `chatgpt-bridge-codex-host` and `chatgpt-bridge-dsh-host` green, `adapter-tool-conformance` 9/0 with both mutation directions firing. `bun x tsc --noEmit` green. --- extensions/dsh-chatgpt-bridge/src/host.ts | 157 ++++++---- src/chatgpt-bridge/contracts/index.ts | 8 +- src/chatgpt-bridge/core/store.ts | 271 +++++++++++------- .../hosts/codex/devspace-mcp-client.ts | 27 +- .../hosts/codex/legacy-registry.ts | 32 +-- src/chatgpt-bridge/provider/adapter.ts | 5 +- .../adapters/adapter-tool-conformance.test.ts | 16 +- .../chatgpt-bridge-codex-host.test.ts | 55 ++++ .../chatgpt-bridge-core.test.ts | 175 ++++++++++- .../chatgpt-bridge-dsh-host.test.ts | 149 +++++++++- 10 files changed, 685 insertions(+), 210 deletions(-) diff --git a/extensions/dsh-chatgpt-bridge/src/host.ts b/extensions/dsh-chatgpt-bridge/src/host.ts index 03b9a258a62..9992cd477b1 100644 --- a/extensions/dsh-chatgpt-bridge/src/host.ts +++ b/extensions/dsh-chatgpt-bridge/src/host.ts @@ -10,8 +10,9 @@ * correlates an inbox item to its turn; * - ctx.on("session/event", (session, event)) streams * assistant/message{turn} / turn/end{turn} for attribution; - * - subagent-owned sessions are rejected on three layers (origin, lineage, - * runtime ownership) — bridging never hijacks a child session; + * - subagent-owned sessions are rejected on both durable layers the session + * header carries (`origin`, `parentSession`) before anything is enqueued — + * bridging never hijacks a child session; * - ctx.storage persists the binding map; ctx.webServer.register exposes the * control endpoints to the OpenCodex core only (loopback + token). */ @@ -22,15 +23,9 @@ export interface DshUserMessage { readonly content: string; } -export interface DshAgent { - readonly id: string; - followup(message: DshUserMessage): void; - whenIdle(): Promise; -} - export interface DshSessionHeader { readonly id: string; - readonly origin?: "user" | "subagent"; + readonly origin?: "subagent"; readonly parentSession?: string; readonly agentPreset?: string; } @@ -39,9 +34,15 @@ export interface DshSession { readonly header: DshSessionHeader; } +export interface DshAgent { + /** Session-backed identity: the same id `ctx.agents.get` takes and `session/event` carries. */ + readonly id: string; + readonly session: DshSession; + followup(message: DshUserMessage): void; +} + export interface DshAgentsRegistry { get(id: string): DshAgent | undefined; - isOwnedBy(childId: string, parentId: string): boolean; } export interface DshPluginContext { @@ -87,6 +88,7 @@ export interface BridgeBindingState { boundAt: string; /** Latest correlated turn per delivered inbox item. */ lastDeliveredInboxItemId: string | null; + lastDeliveredAt: string | null; lastClaimedTurn: number | null; lastAssistantMessageId: string | null; lastTurnEnded: number | null; @@ -94,6 +96,14 @@ export interface BridgeBindingState { export const TOKEN_HEADER = "x-bridge-token"; +/** + * How long an unclaimed delivery may hold the gate. Same boundary the core store + * uses to call a pending send outcome-unknown: past it the host assumes the inbox + * item was dropped, because `agent/inbox/discarded` is not an event this plugin + * can prove the host ever emits. + */ +const OUTSTANDING_DELIVERY_MAX_MS = 120_000; + export interface DshBridgePluginConfig { /** Credential reference resolved by ctx.credentials; never a plaintext secret here. */ controlTokenRef?: string; @@ -103,6 +113,10 @@ export interface DshBridgePluginConfig { export function createBridgePlugin(ctx: DshPluginContext, config: DshBridgePluginConfig) { let controlToken = config.controlToken ?? ""; const states = new Map(); + // A delivery's read-modify-write spans awaits, so two concurrent control calls + // for one session would both mint state and both enqueue: at most one runs at + // a time, and the loser is refused rather than silently dropped. + const delivering = new Set(); const loadState = async (sessionId: string): Promise => { if (states.has(sessionId)) return states.get(sessionId); @@ -123,6 +137,14 @@ export function createBridgePlugin(ctx: DshPluginContext, config: DshBridgePlugi state.lastClaimedTurn = turn; await saveState(String(agent.id)); }); + // A delivery that is cancelled before it is claimed never ends a turn, so + // without this the outstanding-item gate below would wedge the binding. + ctx.on("agent/inbox/discarded", async ({ agent, message }) => { + const state = await loadState(String(agent.id)); + if (!state || state.lastDeliveredInboxItemId !== message.id || state.lastClaimedTurn !== null) return; + state.lastDeliveredInboxItemId = null; + await saveState(String(agent.id)); + }); ctx.on("session/event", async (session, event) => { if (event.type !== "assistant/message" && event.type !== "turn/end") return; const state = await loadState(String(session.header.id)); @@ -136,10 +158,12 @@ export function createBridgePlugin(ctx: DshPluginContext, config: DshBridgePlugi }); /** - * Three-layer rejection mirroring hasApiSessionSubagentOwner (0.1.2-rc.1): - * durable origin, durable parent lineage, and runtime ownership. A child - * session is rejected even when its parent agent is no longer live — the - * lineage alone is disqualifying for bridging. + * Both durable rejection layers of a session header, mirroring + * hasApiSessionSubagentOwner (0.1.2-rc.1): durable origin and durable parent + * lineage. A child session is rejected even when its parent agent is no + * longer live — the lineage alone is disqualifying for bridging. Runtime + * ownership is the third layer there and cannot be consulted here: it needs + * the exact parent Agent, which a session-id-keyed control call never carries. */ const isSubagentOwned = (header: DshSessionHeader): boolean => { if (header.origin === "subagent") return true; @@ -151,6 +175,66 @@ export function createBridgePlugin(ctx: DshPluginContext, config: DshBridgePlugi return request.headers[TOKEN_HEADER] === controlToken; }; + const deliverToSession = async ( + sessionId: string, + request: BridgeControlRequest, + ): Promise => { + const body = request.body as { message?: string; chatConversationId?: string; inboxItemId?: string } | undefined; + const message = body?.message ?? ""; + if (!message.trim()) return { status: 400, body: { ok: false, code: "EMPTY_PROMPT" } }; + const agent = ctx.agents.get(sessionId); + if (!agent) return { status: 404, body: { ok: false, code: "TARGET_NOT_FOUND" } }; + if (isSubagentOwned(agent.session.header)) { + return { status: 409, body: { ok: false, code: "CONTEXT_INCOMPATIBLE" } }; + } + + const state: BridgeBindingState = (await loadState(sessionId)) ?? { + sessionId, + chatConversationId: body?.chatConversationId ?? "", + boundAt: new Date().toISOString(), + lastDeliveredInboxItemId: null, + lastDeliveredAt: null, + lastClaimedTurn: null, + lastAssistantMessageId: null, + lastTurnEnded: null, + }; + if (body?.chatConversationId && state.chatConversationId && state.chatConversationId !== body.chatConversationId) { + return { status: 409, body: { ok: false, code: "BINDING_CHANGED" } }; + } + + // One delivery is outstanding until its turn ends: an item that has not + // been claimed yet may still start one, and overwriting it would leave + // that turn unattributable. Refuse instead of steering or queueing on top. + if (state.lastDeliveredInboxItemId !== null && state.lastTurnEnded === null) { + // A claimed turn ends on its own. An item that was never claimed only + // clears through `agent/inbox/discarded`, so without this age-out a host + // that drops the item quietly refuses every later delivery forever. + const parsed = state.lastDeliveredAt ? Date.parse(state.lastDeliveredAt) : 0; + // A marker this plugin did not write (an epoch number, a hand-edited row) parses to + // NaN, and every comparison against NaN is false: treating it as "very old" is the + // only reading that cannot wedge the session forever. + const deliveredAt = Number.isFinite(parsed) ? parsed : 0; + const dropped = state.lastClaimedTurn === null && Date.now() - deliveredAt > OUTSTANDING_DELIVERY_MAX_MS; + if (!dropped) return { status: 409, body: { ok: false, code: "TARGET_ACTIVE" } }; + } + + const inboxItemId = body?.inboxItemId ?? `bridge-${crypto.randomUUID()}`; + const userMessage: DshUserMessage = { id: inboxItemId, role: "user", content: message }; + state.lastDeliveredInboxItemId = inboxItemId; + state.lastDeliveredAt = new Date().toISOString(); + state.lastClaimedTurn = null; + state.lastAssistantMessageId = null; + state.lastTurnEnded = null; + if (body?.chatConversationId) state.chatConversationId = body.chatConversationId; + states.set(sessionId, state); + agent.followup(userMessage); + await saveState(sessionId); + return { + status: 202, + body: { ok: true, state: "SUBMITTED_UNVERIFIED", inboxItemId }, + }; + }; + const handler = async (request: BridgeControlRequest): Promise => { if (!requireToken(request)) return { status: 401, body: { ok: false, code: "AUTH_REQUIRED" } }; const sessionId = request.path.split("/").filter(Boolean)[1] ?? ""; @@ -162,45 +246,13 @@ export function createBridgePlugin(ctx: DshPluginContext, config: DshBridgePlugi } if (request.method === "POST" && request.path.endsWith("/deliver")) { - const body = request.body as { message?: string; chatConversationId?: string; inboxItemId?: string } | undefined; - const message = body?.message ?? ""; - if (!message.trim()) return { status: 400, body: { ok: false, code: "EMPTY_PROMPT" } }; - const agent = ctx.agents.get(sessionId); - if (!agent) return { status: 404, body: { ok: false, code: "TARGET_NOT_FOUND" } }; - - const state: BridgeBindingState = (await loadState(sessionId)) ?? { - sessionId, - chatConversationId: body?.chatConversationId ?? "", - boundAt: new Date().toISOString(), - lastDeliveredInboxItemId: null, - lastClaimedTurn: null, - lastAssistantMessageId: null, - lastTurnEnded: null, - }; - if (body?.chatConversationId && state.chatConversationId && state.chatConversationId !== body.chatConversationId) { - return { status: 409, body: { ok: false, code: "BINDING_CHANGED" } }; + if (delivering.has(sessionId)) return { status: 409, body: { ok: false, code: "TARGET_ACTIVE" } }; + delivering.add(sessionId); + try { + return await deliverToSession(sessionId, request); + } finally { + delivering.delete(sessionId); } - - // A prior delivery whose turn was claimed but not yet ended means the - // host is mid-turn: refuse instead of steering or queueing on top. - if (state.lastClaimedTurn !== null && state.lastTurnEnded === null) { - return { status: 409, body: { ok: false, code: "TARGET_ACTIVE" } }; - } - - const inboxItemId = body?.inboxItemId ?? `bridge-${crypto.randomUUID()}`; - const userMessage: DshUserMessage = { id: inboxItemId, role: "user", content: message }; - state.lastDeliveredInboxItemId = inboxItemId; - state.lastClaimedTurn = null; - state.lastAssistantMessageId = null; - state.lastTurnEnded = null; - if (body?.chatConversationId) state.chatConversationId = body.chatConversationId; - states.set(sessionId, state); - agent.followup(userMessage); - await saveState(sessionId); - return { - status: 202, - body: { ok: true, state: "SUBMITTED_UNVERIFIED", inboxItemId }, - }; } return { status: 404, body: { ok: false, code: "TARGET_NOT_FOUND" } }; @@ -212,7 +264,6 @@ export function createBridgePlugin(ctx: DshPluginContext, config: DshBridgePlugi ctx.webServer.register({ kind: "prefix", path: "/chatgpt-bridge", handler }); }, handler, - isSubagentOwned, setControlToken(token: string) { controlToken = token; }, diff --git a/src/chatgpt-bridge/contracts/index.ts b/src/chatgpt-bridge/contracts/index.ts index 54d0cbaf43d..06de0441716 100644 --- a/src/chatgpt-bridge/contracts/index.ts +++ b/src/chatgpt-bridge/contracts/index.ts @@ -3,7 +3,6 @@ import { z } from "zod"; /** Wire-level error codes shared by Bridge.* contracts. */ export const BRIDGE_ERROR_CODES = [ "BINDING_CHANGED", - "REVISION_CONFLICT", "BINDING_REVISION_CONFLICT", "OPERATION_ID_CONFLICT", "AUTH_REQUIRED", @@ -12,6 +11,7 @@ export const BRIDGE_ERROR_CODES = [ "TARGET_ACTIVE", "TARGET_NOT_FOUND", "BINDING_NOT_FOUND", + "BINDING_EXISTS", "HOST_OFFLINE", "ATTACHMENT_UNAVAILABLE", "ATTACHMENT_REQUIRED", @@ -24,9 +24,7 @@ export const BRIDGE_ERROR_CODES = [ "INVALID_CHATGPT_URL", "INVALID_REGISTRY", "PROTOCOL_UNSUPPORTED", - "ENVIRONMENT_UNTRUSTED", "CONTEXT_INCOMPATIBLE", - "LOOP_DETECTED", ] as const; export type BridgeErrorCode = (typeof BRIDGE_ERROR_CODES)[number]; @@ -127,6 +125,9 @@ export type BridgeBinding = z.infer; export const BRIDGE_PROTOCOL_VERSION = 1; +/** Largest prompt a delivery may carry; the ceiling the Codex host sends at. */ +export const MAX_PROMPT_CHARS = 512 * 1024; + /** Management actions accepted by the unified manage entry point. */ export const MANAGEMENT_ACTIONS = [ "create", @@ -136,7 +137,6 @@ export const MANAGEMENT_ACTIONS = [ "renew", "revoke", "unbind", - "replace", ] as const; export type BridgeManagementAction = (typeof MANAGEMENT_ACTIONS)[number]; diff --git a/src/chatgpt-bridge/core/store.ts b/src/chatgpt-bridge/core/store.ts index 7fe3f6b6369..f6b9af910af 100644 --- a/src/chatgpt-bridge/core/store.ts +++ b/src/chatgpt-bridge/core/store.ts @@ -10,6 +10,7 @@ import { parseChatGptConversationUrl, DEFINITE_NON_DELIVERY_CODES, type BridgeErrorCode, + MAX_PROMPT_CHARS, } from "../contracts"; /** @@ -21,6 +22,11 @@ import { const RESERVATION_STALE_MS = 120_000; const DUPLICATE_GUARD_MS = 120_000; const MAX_OPERATIONS_PER_BINDING = 40; +// Age-free ceiling, so a wrong clock at install cannot make the table grow forever. +const MAX_OPERATIONS_HARD_CAP = 400; +// A dashboard reload replays the pending action minutes later, so a receipt must stay +// replayable longer than the send path's own stale window. +const OPERATION_REPLAY_MS = 15 * 60_000; export interface CreateBindingInput { bindingId?: string; @@ -38,7 +44,6 @@ export interface ManageBindingInput { action: Exclude; operationId: string; expectedRevision: number; - replacement?: CreateBindingInput; capabilityHash?: string | null; capabilityExpiresAt?: string | null; } @@ -79,7 +84,7 @@ export class BridgeBindingStore { constructor( database: Database | string, - private readonly options: { now?: () => string; reservationStaleMs?: number; duplicateGuardMs?: number } = {}, + private readonly options: { reservationStaleMs?: number; duplicateGuardMs?: number } = {}, ) { this.db = typeof database === "string" ? new Database(database) : database; this.db.exec("PRAGMA journal_mode = WAL;"); @@ -114,7 +119,6 @@ export class BridgeBindingStore { proof_chat_read TEXT, proof_checked_at TEXT, last_receipt_id TEXT, - replaced_by TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL ); @@ -144,6 +148,12 @@ export class BridgeBindingStore { settled_at TEXT ); CREATE INDEX IF NOT EXISTS idx_bridge_deliveries_binding ON chatgpt_bridge_deliveries(binding_id, state); + CREATE INDEX IF NOT EXISTS idx_bridge_deliveries_digest ON chatgpt_bridge_deliveries(binding_id, prompt_digest, state); + -- The chat side is UNIQUE in schema; the host side was only checked in-process, + -- so a second process on this file could bind one Codex target twice. + CREATE UNIQUE INDEX IF NOT EXISTS idx_bridge_bindings_host_target + ON chatgpt_bridge_bindings(host_instance_id, host_target_id) + WHERE lifecycle IN ('active', 'paused'); `); this.db .prepare( @@ -156,6 +166,11 @@ export class BridgeBindingStore { this.db.close(); } + /** Every multi-statement write below runs in one transaction: a receipt must not outlive its row. */ + private tx(write: () => T): T { + return this.db.transaction(write)(); + } + private rowToBinding(row: Record): BridgeBinding { return { schemaVersion: 1, @@ -248,10 +263,30 @@ export class BridgeBindingStore { VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, ) .run(operationId, bindingId, action, requestDigest, revision, outcome, errorCode ?? null, created); + this.pruneOperations(bindingId); return { operationId, bindingId, action, revision, outcome, errorCode, createdAt: created }; } + /** + * Trim each binding's receipt history, but never inside the replay window: a + * pruned row turns a late retry of `create` into a second insert (or a + * `BINDING_EXISTS`) instead of the `alreadyApplied` replay it asked for. + * + * The age condition can be defeated by the clock: a device that booted with a wrong + * RTC writes future `created_at` values that never age out, so the second statement + * is an age-free backstop. Growth is then `MAX_OPERATIONS_PER_BINDING` in steady + * state and never more than `MAX_OPERATIONS_HARD_CAP` even with a hostile clock. + */ private pruneOperations(bindingId: string): void { + const cutoff = new Date(Date.now() - OPERATION_REPLAY_MS).toISOString(); + this.db + .prepare( + `DELETE FROM chatgpt_bridge_operations WHERE binding_id = ? AND created_at < ? AND operation_id NOT IN ( + SELECT operation_id FROM chatgpt_bridge_operations WHERE binding_id = ? + ORDER BY created_at DESC LIMIT ? + )`, + ) + .run(bindingId, cutoff, bindingId, MAX_OPERATIONS_PER_BINDING); this.db .prepare( `DELETE FROM chatgpt_bridge_operations WHERE binding_id = ? AND operation_id NOT IN ( @@ -259,7 +294,7 @@ export class BridgeBindingStore { ORDER BY created_at DESC LIMIT ? )`, ) - .run(bindingId, bindingId, MAX_OPERATIONS_PER_BINDING); + .run(bindingId, bindingId, MAX_OPERATIONS_HARD_CAP); } private assertOperationIdempotent( @@ -300,13 +335,24 @@ export class BridgeBindingStore { const bindingId = input.bindingId ?? randomUUID(); if (this.getBinding(bindingId)) { - throw new BridgeCoreError("OPERATION_ID_CONFLICT", "bindingId already exists"); + throw new BridgeCoreError("BINDING_EXISTS", "bindingId already exists"); } const duplicateChat = this.db .prepare("SELECT binding_id FROM chatgpt_bridge_bindings WHERE chat_conversation_id = ?") .get(chat.conversationId) as { binding_id: string } | undefined; if (duplicateChat) { - throw new BridgeCoreError("OPERATION_ID_CONFLICT", "chat already bound to another target"); + throw new BridgeCoreError("BINDING_EXISTS", "chat already bound to another target"); + } + // The mirror of the chat guard: two bindings on one host target would both + // deliver into the same task, and the per-binding SEND_IN_PROGRESS and + // DUPLICATE_PROMPT guards cannot see each other across them. + const duplicateHost = this.db + .prepare( + "SELECT binding_id FROM chatgpt_bridge_bindings WHERE host_instance_id = ? AND host_target_id = ? AND lifecycle IN ('active','paused')", + ) + .get(input.host.instanceId, input.host.targetId) as { binding_id: string } | undefined; + if (duplicateHost) { + throw new BridgeCoreError("BINDING_EXISTS", "host target already bound to another chat"); } const created = now(); @@ -328,32 +374,34 @@ export class BridgeBindingStore { createdAt: created, updatedAt: created, }; - this.db - .prepare( - `INSERT INTO chatgpt_bridge_bindings - (binding_id, host_kind, host_instance_id, host_target_id, host_workspace_ref, - chat_conversation_id, chat_canonical_url, owner_ref, source_kind, source_locator, - revision, epoch, lifecycle, attachment_state, capability_hash, capability_expires_at, - proof_attachment, proof_host_read, proof_chat_read, proof_checked_at, last_receipt_id, - created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'bridge-v1', ?, 0, 0, 'active', 'pending', ?, ?, NULL, NULL, NULL, NULL, NULL, ?, ?)`, - ) - .run( - binding.bindingId, - binding.host.kind, - binding.host.instanceId, - binding.host.targetId, - binding.host.workspaceRef, - binding.chat.conversationId, - binding.chat.canonicalUrl, - binding.ownerRef, - binding.source.locator, - binding.capabilityHash, - binding.capabilityExpiresAt, - binding.createdAt, - binding.updatedAt, - ); - const receipt = this.recordOperation(input.operationId, bindingId, "create", digest, 0, "applied"); + const receipt = this.tx(() => { + this.db + .prepare( + `INSERT INTO chatgpt_bridge_bindings + (binding_id, host_kind, host_instance_id, host_target_id, host_workspace_ref, + chat_conversation_id, chat_canonical_url, owner_ref, source_kind, source_locator, + revision, epoch, lifecycle, attachment_state, capability_hash, capability_expires_at, + proof_attachment, proof_host_read, proof_chat_read, proof_checked_at, last_receipt_id, + created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'bridge-v1', ?, 0, 0, 'active', 'pending', ?, ?, NULL, NULL, NULL, NULL, NULL, ?, ?)`, + ) + .run( + binding.bindingId, + binding.host.kind, + binding.host.instanceId, + binding.host.targetId, + binding.host.workspaceRef, + binding.chat.conversationId, + binding.chat.canonicalUrl, + binding.ownerRef, + binding.source.locator, + binding.capabilityHash, + binding.capabilityExpiresAt, + binding.createdAt, + binding.updatedAt, + ); + return this.recordOperation(input.operationId, bindingId, "create", digest, 0, "applied"); + }); return { binding, receipt }; } @@ -379,13 +427,15 @@ export class BridgeBindingStore { const updated = now(); const nextState: BridgeBinding["attachmentState"] = binding.attachmentState === "readable" ? "readable" : "attached"; - this.db - .prepare( - `UPDATE chatgpt_bridge_bindings SET attachment_state = ?, proof_attachment = ?, - revision = revision + 1, updated_at = ? WHERE binding_id = ?`, - ) - .run(nextState, input.attachmentProof, updated, bindingId); - const receipt = this.recordOperation(input.operationId, bindingId, "attach", digest, binding.revision + 1, "applied"); + const receipt = this.tx(() => { + this.db + .prepare( + `UPDATE chatgpt_bridge_bindings SET attachment_state = ?, proof_attachment = ?, + revision = revision + 1, updated_at = ? WHERE binding_id = ?`, + ) + .run(nextState, input.attachmentProof, updated, bindingId); + return this.recordOperation(input.operationId, bindingId, "attach", digest, binding.revision + 1, "applied"); + }); return { binding: this.getBinding(bindingId)!, receipt }; } @@ -394,8 +444,6 @@ export class BridgeBindingStore { bindingId: input.bindingId, action: input.action, capabilityHash: input.capabilityHash ?? null, - replacementHost: input.replacement?.host ?? null, - replacementChat: input.replacement ? parseChatGptConversationUrl(input.replacement.chatUrl) : null, }); const replay = this.assertOperationIdempotent(input.operationId, input.action, digest); if (replay) { @@ -423,8 +471,6 @@ export class BridgeBindingStore { return lifecycle === "revoked" || lifecycle === "active" || lifecycle === "paused" ? "unbound" : null; case "renew": return lifecycle === "active" || lifecycle === "paused" ? lifecycle : null; - case "replace": - return lifecycle === "active" || lifecycle === "paused" ? "revoked" : null; default: return null; } @@ -434,35 +480,33 @@ export class BridgeBindingStore { } const updated = now(); - const epochBump = input.action === "revoke" || input.action === "replace" ? 1 : 0; - const replacedBy = input.action === "replace" ? input.replacement?.bindingId ?? null : null; - this.db - .prepare( - `UPDATE chatgpt_bridge_bindings SET lifecycle = ?, epoch = epoch + ?, - revision = revision + 1, - capability_hash = COALESCE(?, capability_hash), - capability_expires_at = COALESCE(?, capability_expires_at), - replaced_by = COALESCE(?, replaced_by), - updated_at = ? WHERE binding_id = ?`, - ) - .run( - nextLifecycle, - epochBump, - input.capabilityHash ?? null, - input.capabilityExpiresAt ?? null, - replacedBy, - updated, + const epochBump = input.action === "revoke" ? 1 : 0; + const receipt = this.tx(() => { + this.db + .prepare( + `UPDATE chatgpt_bridge_bindings SET lifecycle = ?, epoch = epoch + ?, + revision = revision + 1, + capability_hash = COALESCE(?, capability_hash), + capability_expires_at = COALESCE(?, capability_expires_at), + updated_at = ? WHERE binding_id = ?`, + ) + .run( + nextLifecycle, + epochBump, + input.capabilityHash ?? null, + input.capabilityExpiresAt ?? null, + updated, + input.bindingId, + ); + return this.recordOperation( + input.operationId, input.bindingId, + input.action, + digest, + binding.revision + 1, + "applied", ); - this.pruneOperations(input.bindingId); - const receipt = this.recordOperation( - input.operationId, - input.bindingId, - input.action, - digest, - binding.revision + 1, - "applied", - ); + }); return { binding: this.getBinding(input.bindingId), receipt }; } @@ -475,6 +519,15 @@ export class BridgeBindingStore { } const prompt = input.prompt; if (prompt.length === 0) throw new BridgeCoreError("EMPTY_PROMPT", "prompt is empty"); + if (prompt.length > MAX_PROMPT_CHARS) { + throw new BridgeCoreError("PROMPT_TOO_LARGE", `prompt exceeds ${MAX_PROMPT_CHARS} characters`); + } + if (binding.attachmentState === "pending") { + throw new BridgeCoreError("ATTACHMENT_REQUIRED", "binding has no attachment proof yet"); + } + if (binding.capabilityExpiresAt && Date.parse(binding.capabilityExpiresAt) <= Date.now()) { + throw new BridgeCoreError("CAPABILITY_EXPIRED", "capability expired before this send"); + } const pending = this.db .prepare( @@ -510,23 +563,26 @@ export class BridgeBindingStore { const reservationId = randomUUID(); const reservedAt = now(); - this.db - .prepare( - `INSERT INTO chatgpt_bridge_deliveries - (reservation_id, binding_id, operation_id, direction, source_message_id, prompt_digest, - binding_epoch, state, reserved_at) - VALUES (?, ?, ?, ?, ?, ?, ?, 'reserved', ?)`, - ) - .run( - reservationId, - input.bindingId, - input.operationId, - input.direction, - input.sourceMessageId, - sha256(prompt), - binding.epoch, - reservedAt, - ); + this.tx(() => { + this.db + .prepare( + `INSERT INTO chatgpt_bridge_deliveries + (reservation_id, binding_id, operation_id, direction, source_message_id, prompt_digest, + binding_epoch, state, reserved_at) + VALUES (?, ?, ?, ?, ?, ?, ?, 'reserved', ?)`, + ) + .run( + reservationId, + input.bindingId, + input.operationId, + input.direction, + input.sourceMessageId, + sha256(prompt), + binding.epoch, + reservedAt, + ); + this.pruneDeliveries(input.bindingId); + }); return { reservationId, bindingId: input.bindingId, @@ -536,7 +592,21 @@ export class BridgeBindingStore { }; } - settleDelivery(input: SettleDeliveryInput): { state: BridgeDeliveryState; receiptId: string } { + /** + * Terminal rows older than the duplicate-guard window are invisible to every + * reader by construction — the guard only looks inside the window, and a + * reserved or unknown row is a fence that must survive until reconciled. + */ + private pruneDeliveries(bindingId: string): void { + const cutoff = new Date(Date.now() - (this.options.duplicateGuardMs ?? DUPLICATE_GUARD_MS)).toISOString(); + this.db + .prepare( + "DELETE FROM chatgpt_bridge_deliveries WHERE binding_id = ? AND state IN ('delivered','not-delivered') AND settled_at < ?", + ) + .run(bindingId, cutoff); + } + + settleDelivery(input: SettleDeliveryInput): { state: BridgeDeliveryState; receiptId: string | null } { const row = this.db .prepare("SELECT * FROM chatgpt_bridge_deliveries WHERE reservation_id = ?") .get(input.reservationId) as Record | undefined; @@ -556,15 +626,22 @@ export class BridgeBindingStore { if (input.outcome === "unknown" && !input.operator) { throw new BridgeCoreError("DELIVERY_UNKNOWN", "manual unknown resolution requires operator identity"); } - const receiptId = input.receiptId ?? randomUUID(); - this.db - .prepare( - "UPDATE chatgpt_bridge_deliveries SET state = ?, failure_code = ?, receipt_id = ?, resolved_by = ?, settled_at = ? WHERE reservation_id = ?", - ) - .run(input.outcome, input.failureCode ?? null, receiptId, input.operator ?? null, now(), input.reservationId); - this.db - .prepare("UPDATE chatgpt_bridge_bindings SET last_receipt_id = ?, updated_at = ? WHERE binding_id = ?") - .run(receiptId, now(), row.binding_id as string); + // A receipt attests delivery: a not-delivered or unknown settlement records + // a failure code, never a receipt the caller could present as proof. + const receiptId = input.outcome === "delivered" ? input.receiptId ?? randomUUID() : null; + const settledAt = now(); + this.tx(() => { + this.db + .prepare( + "UPDATE chatgpt_bridge_deliveries SET state = ?, failure_code = ?, receipt_id = ?, resolved_by = ?, settled_at = ? WHERE reservation_id = ?", + ) + .run(input.outcome, input.failureCode ?? null, receiptId, input.operator ?? null, settledAt, input.reservationId); + if (receiptId) { + this.db + .prepare("UPDATE chatgpt_bridge_bindings SET last_receipt_id = ?, updated_at = ? WHERE binding_id = ?") + .run(receiptId, settledAt, row.binding_id as string); + } + }); return { state: input.outcome, receiptId }; } diff --git a/src/chatgpt-bridge/hosts/codex/devspace-mcp-client.ts b/src/chatgpt-bridge/hosts/codex/devspace-mcp-client.ts index 4fdda50d4d1..4e4fb10eed2 100644 --- a/src/chatgpt-bridge/hosts/codex/devspace-mcp-client.ts +++ b/src/chatgpt-bridge/hosts/codex/devspace-mcp-client.ts @@ -1,4 +1,4 @@ -import { BridgeCoreError, type BridgeErrorCode } from "../../contracts"; +import { BridgeCoreError, type BridgeErrorCode, MAX_PROMPT_CHARS } from "../../contracts"; /** * Minimal MCP client for the local DevSpace control plane (legacy bridge tools). @@ -59,7 +59,9 @@ export class DevSpaceMcpClient { if (this.sessionId) { await this.fetchImpl(this.config.baseUrl, { method: "POST", - headers: this.headers(false), + // Streamable HTTP scopes every frame after initialize to the minted + // session, so this notification must carry the id it announces. + headers: this.headers(true), body: JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" }), signal: this.signal(), }); @@ -93,6 +95,13 @@ export class DevSpaceMcpClient { if (response.status === 401 || response.status === 403) { throw new BridgeCoreError("AUTH_REQUIRED", `devspace rejected credentials (${response.status})`); } + if (response.status === 404) { + // A DevSpace restart retires the minted session id, and every later call + // would then be rejected with the same 404. Drop it so the next call + // re-initializes instead of failing for the life of this client. + this.sessionId = null; + this.sessionReady = null; + } if (!response.ok) { const detail = (await response.text().catch(() => "")).slice(0, 300); throw new BridgeCoreError("HOST_OFFLINE", `devspace mcp ${response.status}${detail ? `: ${detail}` : ""}`); @@ -103,13 +112,14 @@ export class DevSpaceMcpClient { /** Invoke a DevSpace MCP tool and unwrap the bridge result envelope. */ async callTool>(tool: string, args: Record = {}): Promise { await this.ensureSession(); + const id = this.nextId++; const response = await this.postRpc({ jsonrpc: "2.0", - id: this.nextId++, + id, method: "tools/call", params: { name: tool, arguments: args }, }); - const payload = await this.parseRpcResponse(response); + const payload = await this.parseRpcResponse(response, id); if (payload.error) { throw new BridgeCoreError("HOST_OFFLINE", `devspace rpc error ${payload.error.code}: ${payload.error.message}`); } @@ -123,8 +133,11 @@ export class DevSpaceMcpClient { /** * Streamable HTTP servers may answer a POST with an SSE stream carrying the * JSON-RPC frame; unwrap either transport into a single response object. + * A shared stream multiplexes every request, so a frame is only an answer + * when it carries this request's id — the first response frame may belong to + * an earlier call. */ - private async parseRpcResponse(response: Response): Promise { + private async parseRpcResponse(response: Response, id: number | string): Promise { const contentType = response.headers.get("content-type") ?? ""; if (contentType.includes("text/event-stream")) { const raw = await response.text(); @@ -134,7 +147,7 @@ export class DevSpaceMcpClient { if (!chunk) continue; try { const frame = JSON.parse(chunk) as JsonRpcResponse; - if (frame.id !== null && frame.id !== undefined && (frame.result || frame.error)) return frame; + if (frame.id === id && (frame.result || frame.error)) return frame; } catch { // ignore keep-alive comments / non-JSON frames } @@ -250,7 +263,7 @@ export class CodexHostAdapter { async send(controllerId: string, prompt: string): Promise { if (prompt.length === 0) throw new BridgeCoreError("EMPTY_PROMPT", "prompt is empty"); - if (prompt.length > 512 * 1024) throw new BridgeCoreError("PROMPT_TOO_LARGE", "prompt exceeds 512 KiB"); + if (prompt.length > MAX_PROMPT_CHARS) throw new BridgeCoreError("PROMPT_TOO_LARGE", "prompt exceeds 512 KiB"); const result = await this.client.callTool("codex_bridge_send", { controllerId, prompt }); return this.toStatus(result); } diff --git a/src/chatgpt-bridge/hosts/codex/legacy-registry.ts b/src/chatgpt-bridge/hosts/codex/legacy-registry.ts index 46b4ff37465..4492988bc58 100644 --- a/src/chatgpt-bridge/hosts/codex/legacy-registry.ts +++ b/src/chatgpt-bridge/hosts/codex/legacy-registry.ts @@ -1,4 +1,4 @@ -import { readFileSync, statSync } from "node:fs"; +import { readFileSync } from "node:fs"; import { BridgeCoreError, CHATGPT_CONVERSATION_URL_PATTERN } from "../../contracts"; /** @@ -8,11 +8,15 @@ import { BridgeCoreError, CHATGPT_CONVERSATION_URL_PATTERN } from "../../contrac * - byte-read + JSON.parse only: importing bridge-lib would run mkdir/icacls * side effects, so that module must never be loaded here; * - this reader never writes, never locks, and never touches *.cap files; - * - snapshots are invalidated by managementRevision + mtime so OC always sees - * the legacy registry as the single source of truth for existing bindings. + * - every read goes to the file: the legacy registry is the single source of + * truth for existing bindings, so any snapshot this class kept would be a + * stale answer, and a same-length rewrite can hide inside mtime granularity. */ +// Windows-only by construction: the legacy bridge writes this under %USERPROFILE%. +// Anywhere else the literal `~` is never expanded, so the read below reports the +// empty snapshot rather than a foreign path. export const LEGACY_REGISTRY_PATH_DEFAULT = - `${process.env.USERPROFILE ?? "~"}\\.codex\\state\\chatgpt-codex-live-bridge\\bindings.json`.replace(/~/g, process.env.USERPROFILE ?? "~"); + `${process.env.USERPROFILE ?? "~"}\\.codex\\state\\chatgpt-codex-live-bridge\\bindings.json`; export interface LegacyBindingSnapshot { source: "legacy-codex"; @@ -40,40 +44,27 @@ export interface LegacyRegistrySnapshot { bindings: LegacyBindingSnapshot[]; } -interface CacheEntry { - mtimeMs: number; - size: number; - snapshot: LegacyRegistrySnapshot; -} - export class LegacyBindingRegistryReader { - private cache: CacheEntry | null = null; - constructor(private readonly registryPath: string = LEGACY_REGISTRY_PATH_DEFAULT) {} read(): LegacyRegistrySnapshot { - let stat: { mtimeMs: number; size: number }; + let raw: string; try { - const s = statSync(this.registryPath); - stat = { mtimeMs: s.mtimeMs, size: s.size }; + raw = readFileSync(this.registryPath, "utf8"); } catch (error) { const code = (error as NodeJS.ErrnoException).code; if (code === "ENOENT" || code === "ENOTDIR") { - this.cache = null; return { managementRevision: -1, version: 0, readAt: new Date().toISOString(), bindings: [] }; } throw new BridgeCoreError("INVALID_REGISTRY", `legacy registry unreadable: ${String(error)}`); } - if (this.cache && this.cache.mtimeMs === stat.mtimeMs && this.cache.size === stat.size) { - return this.cache.snapshot; - } let parsed: { version?: number; managementRevision?: number; bindings?: Record>; }; try { - parsed = JSON.parse(readFileSync(this.registryPath, "utf8")); + parsed = JSON.parse(raw); } catch (error) { throw new BridgeCoreError("INVALID_REGISTRY", `legacy registry parse failed: ${String(error)}`); } @@ -89,7 +80,6 @@ export class LegacyBindingRegistryReader { readAt: new Date().toISOString(), bindings, }; - this.cache = { ...stat, snapshot }; return snapshot; } diff --git a/src/chatgpt-bridge/provider/adapter.ts b/src/chatgpt-bridge/provider/adapter.ts index 2317c2999df..26acf00db71 100644 --- a/src/chatgpt-bridge/provider/adapter.ts +++ b/src/chatgpt-bridge/provider/adapter.ts @@ -13,7 +13,9 @@ import type { IncomingMeta, ProviderAdapter } from "../../adapters/base"; export interface ChatGptWebTurnContext { modelId: string; effort?: string; - promptPreview: string; + // Deliberately no prompt or context projection: a serialized slice of the + // conversation is request content, and any transport that logs this context + // would then log the body. AGENTS.md forbids that at the privacy boundary. } export interface ChatGptWebTransport { @@ -75,7 +77,6 @@ export function createChatGptWebAdapter( { modelId: parsed.modelId, effort: (parsed.options as { reasoningEffort?: string } | undefined)?.reasoningEffort, - promptPreview: JSON.stringify(parsed.context ?? {}).slice(0, 200), }, incoming, emit, diff --git a/tests/adapters/adapter-tool-conformance.test.ts b/tests/adapters/adapter-tool-conformance.test.ts index 94715811077..70c3872e7e2 100644 --- a/tests/adapters/adapter-tool-conformance.test.ts +++ b/tests/adapters/adapter-tool-conformance.test.ts @@ -420,11 +420,19 @@ describe("registry-derived routed tool conformance", () => { }); const TOOL_LESS_ADAPTERS = new Set(["codebuddy", "qoder"]); - // The Devin adapter is runTurn-only: it streams Connect-RPC from runTurn, so + // Devin and chatgpt-web are runTurn-only: Devin streams Connect-RPC from + // runTurn, and chatgpt-web delegates every turn to an injected transport, so // buildRequest returns a placeholder and tools never travel the wire path. - // Both Devin provider rows share it and differ only in where the credential - // came from. - const RUN_TURN_ONLY_WIRES = new Set(["devin"]); + // Both Devin provider rows share an adapter and differ only in where the + // credential came from. + const RUN_TURN_ONLY_WIRES = new Set(["devin", "chatgpt-web"]); + // The skips above must stay derived from the driver table, never merely declared. + test("runTurn-only skips are exactly the reported wires with no conformance driver", () => { + const drivers = new Set(Object.keys(TOOL_WIRE_DRIVERS)); + for (const wire of RUN_TURN_ONLY_WIRES) expect(drivers.has(wire), wire).toBe(false); + const reported = new Set(adapterDefinitions().map(([adapterId]) => effectiveAdapterContract(adapterId).wire)); + for (const wire of reported) if (!drivers.has(wire)) expect(RUN_TURN_ONLY_WIRES.has(wire), wire).toBe(true); + }); test("every registered adapter keeps the nested apply_patch helper in its final request", async () => { for (const [adapterId] of adapterDefinitions()) { diff --git a/tests/chatgpt-bridge/chatgpt-bridge-codex-host.test.ts b/tests/chatgpt-bridge/chatgpt-bridge-codex-host.test.ts index 7decdc7df49..91376816517 100644 --- a/tests/chatgpt-bridge/chatgpt-bridge-codex-host.test.ts +++ b/tests/chatgpt-bridge/chatgpt-bridge-codex-host.test.ts @@ -117,4 +117,59 @@ describe("devspace mcp client", () => { await expect(adapter.send("c1", "x".repeat(512 * 1024 + 1))).rejects.toThrow(BridgeCoreError); expect(calls).toHaveLength(0); }); + + test("a retired session id is dropped: the next call re-initializes instead of failing forever", async () => { + const sent: string[] = []; + let retired = false; + let minted = 0; + const fetchImpl = (async (_input: string | URL | Request, init?: RequestInit) => { + const headers = init?.headers as Record; + const body = JSON.parse(String(init?.body ?? "{}")); + sent.push(`${body.method}:${headers["mcp-session-id"] ?? "none"}`); + if (body.method === "initialize") { + minted += 1; + return new Response(JSON.stringify({ jsonrpc: "2.0", id: body.id, result: {} }), { + status: 200, + headers: { "mcp-session-id": `session-${minted}` }, + }); + } + if (body.method === "notifications/initialized") return new Response(null, { status: 202 }); + // A DevSpace restart retires the minted id, so any call still presenting + // session-1 is rejected until the client mints a new one. + if (retired && headers["mcp-session-id"] === "session-1") return new Response("no such session", { status: 404 }); + retired = true; + return new Response(JSON.stringify({ jsonrpc: "2.0", id: body.id, result: toolResult({ ok: true, state: "READABLE" }) }), { status: 200 }); + }) as typeof fetch; + const client = new DevSpaceMcpClient({ baseUrl: "http://127.0.0.1:17676/mcp", bearerToken: "x", fetchImpl }); + await client.callTool("codex_bridge_status", { controllerId: "c1" }); + await expect(client.callTool("codex_bridge_status", { controllerId: "c1" })).rejects.toThrow(BridgeCoreError); + await client.callTool("codex_bridge_status", { controllerId: "c1" }); + expect(sent).toEqual([ + "initialize:none", + "notifications/initialized:session-1", + "tools/call:session-1", + "tools/call:session-1", + "initialize:none", + "notifications/initialized:session-2", + "tools/call:session-2", + ]); + }); + + test("a shared SSE stream is matched by request id, not by the first response frame", async () => { + const fetchImpl = (async (_input: string | URL | Request, init?: RequestInit) => { + const body = JSON.parse(String(init?.body ?? "{}")); + if (body.method === "initialize") { + return new Response(JSON.stringify({ jsonrpc: "2.0", id: body.id, result: {} }), { status: 200 }); + } + const other = { jsonrpc: "2.0", id: 99, result: toolResult({ ok: true, state: "ANSWER-TO-ANOTHER-CALL" }) }; + const mine = { jsonrpc: "2.0", id: body.id, result: toolResult({ ok: true, state: "READABLE" }) }; + return new Response( + `data: ${JSON.stringify(other)}\n\ndata: ${JSON.stringify(mine)}\n\n`, + { status: 200, headers: { "content-type": "text/event-stream" } }, + ); + }) as typeof fetch; + const client = new DevSpaceMcpClient({ baseUrl: "http://127.0.0.1:17676/mcp", bearerToken: "x", fetchImpl }); + const status = await new CodexHostAdapter(client).status("c1"); + expect(status.state).toBe("READABLE"); + }); }); diff --git a/tests/chatgpt-bridge/chatgpt-bridge-core.test.ts b/tests/chatgpt-bridge/chatgpt-bridge-core.test.ts index ee4daabc9e5..1671e62afc0 100644 --- a/tests/chatgpt-bridge/chatgpt-bridge-core.test.ts +++ b/tests/chatgpt-bridge/chatgpt-bridge-core.test.ts @@ -25,12 +25,29 @@ function makeStore(): BridgeBindingStore { } function createBinding(store: BridgeBindingStore, chatUrl: string = CHAT_URL) { - return store.createBinding({ + const created = store.createBinding({ ownerRef: "owner:test", host: { ...HOST }, chatUrl, operationId: crypto.randomUUID(), }); + // A binding has to be attached before it can carry a prompt, so the shared + // helper returns the state a delivery test actually starts from. + return store.attachBinding(created.binding.bindingId, { + operationId: crypto.randomUUID(), + expectedRevision: created.binding.revision, + attachmentProof: "proof:test", + }); +} + +function reserve(store: BridgeBindingStore, bindingId: string, prompt: string) { + return store.reserveDelivery({ + bindingId, + operationId: crypto.randomUUID(), + direction: "chat-to-host", + sourceMessageId: "m1", + prompt, + }); } /** bun:test toThrow does not take predicates; assert code explicitly here. */ @@ -139,6 +156,75 @@ describe("chatgpt-bridge core store: management fencing", () => { "BINDING_CHANGED", ); }); + + test("receipt pruning keeps the replay window and still bounds the table", () => { + const db = new Database(":memory:"); + const store = new BridgeBindingStore(db, { reservationStaleMs: 60_000, duplicateGuardMs: 60_000 }); + const chatUrl = "https://chatgpt.com/c/22222222-90ab-4cde-8f01-234567890abc"; + const createOp = crypto.randomUUID(); + const input = { ownerRef: "owner:prune", host: { ...HOST }, chatUrl, operationId: createOp }; + const created = store.createBinding(input); + let revision = created.binding.revision; + for (let i = 0; i < 45; i += 1) { + const again = store.attachBinding(created.binding.bindingId, { + operationId: crypto.randomUUID(), + expectedRevision: revision, + attachmentProof: `proof:${i}`, + }); + revision = again.binding.revision; + } + // 45 newer receipts have passed the newest-40 window over the create row. A + // late retry must still replay it, not run the create a second time. + expect(store.createBinding(input).receipt.outcome).toBe("alreadyApplied"); + + const rowCount = () => + (db.query("SELECT COUNT(*) AS n FROM chatgpt_bridge_operations").get() as { n: number }).n; + expect(rowCount()).toBe(46); + + // Outside the window the same trim must actually delete, or the table is unbounded. + // The create row itself stays inside the window, so which rows get trimmed is + // deterministic despite the shared backdated timestamp. + db.run("UPDATE chatgpt_bridge_operations SET created_at = '2020-01-01T00:00:00.000Z' WHERE operation_id != ?", [createOp]); + store.attachBinding(created.binding.bindingId, { + operationId: crypto.randomUUID(), + expectedRevision: revision, + attachmentProof: "proof:after-backdate", + }); + expect(rowCount()).toBe(40); + expect(store.getOperation(createOp)?.action).toBe("create"); + }); + + test("the replay window is pinned from both sides, not just present", () => { + const db = new Database(":memory:"); + const store = new BridgeBindingStore(db, { reservationStaleMs: 60_000, duplicateGuardMs: 60_000 }); + const chatUrl = "https://chatgpt.com/c/33333333-90ab-4cde-8f01-234567890abc"; + const createOp = crypto.randomUUID(); + const created = store.createBinding({ ownerRef: "owner:window", host: { ...HOST }, chatUrl, operationId: createOp }); + let revision = created.binding.revision; + const attachOps: string[] = []; + for (let i = 0; i < 45; i += 1) { + const operationId = crypto.randomUUID(); + attachOps.push(operationId); + revision = store.attachBinding(created.binding.bindingId, { + operationId, expectedRevision: revision, attachmentProof: `proof:${i}`, + }).binding.revision; + } + const aged = (minutesAgo: number) => new Date(Date.now() - minutesAgo * 60_000).toISOString(); + const backdate = (operationId: string, minutesAgo: number) => + db.run("UPDATE chatgpt_bridge_operations SET created_at = ? WHERE operation_id = ?", [aged(minutesAgo), operationId]); + // Both are older than the newest 40 receipts, so the age floor is the only thing + // that can keep the 14-minute one. The 16-minute one has to go, or the window is + // longer than the receipts a client can still be retrying inside. + backdate(createOp, 14); + backdate(attachOps[0], 16); + store.attachBinding(created.binding.bindingId, { + operationId: crypto.randomUUID(), expectedRevision: revision, attachmentProof: "proof:trigger", + }); + + expect(store.getOperation(createOp)).not.toBeNull(); + expect(store.getOperation(attachOps[0])).toBeNull(); + expect((db.query("SELECT COUNT(*) AS n FROM chatgpt_bridge_operations").get() as { n: number }).n).toBe(46); + }); }); describe("chatgpt-bridge core store: delivery guarantees", () => { @@ -200,13 +286,7 @@ describe("chatgpt-bridge core store: delivery guarantees", () => { test("settle not-delivered requires a definite non-delivery code", () => { const store = makeStore(); const { binding } = createBinding(store); - const reservation = store.reserveDelivery({ - bindingId: binding.bindingId, - operationId: crypto.randomUUID(), - direction: "chat-to-host", - sourceMessageId: "m1", - prompt: "definite check", - }); + const reservation = reserve(store, binding.bindingId, "definite check"); expectBridgeError( () => store.settleDelivery({ reservationId: reservation.reservationId, outcome: "not-delivered", failureCode: "PIPE_TIMEOUT" as never }), "DELIVERY_UNKNOWN", @@ -218,6 +298,85 @@ describe("chatgpt-bridge core store: delivery guarantees", () => { }); expect(settled.state).toBe("not-delivered"); }); + + test("a settlement that did not deliver records no receipt", () => { + const store = makeStore(); + const { binding } = createBinding(store); + const notDelivered = store.settleDelivery({ + reservationId: reserve(store, binding.bindingId, "never arrived").reservationId, + outcome: "not-delivered", + failureCode: "SEND_PAUSED", + }); + expect(notDelivered.receiptId).toBeNull(); + expect(store.getBinding(binding.bindingId)?.lastReceiptId).toBeNull(); + const delivered = store.settleDelivery({ + reservationId: reserve(store, binding.bindingId, "arrived").reservationId, + outcome: "delivered", + }); + expect(delivered.receiptId).not.toBeNull(); + expect(store.getBinding(binding.bindingId)?.lastReceiptId).toBe(delivered.receiptId); + }); + + test("reserve enforces the attachment, capability and size gates", () => { + const store = makeStore(); + const pending = store.createBinding({ + ownerRef: "owner:test", + host: { ...HOST }, + chatUrl: CHAT_URL, + operationId: crypto.randomUUID(), + }); + expectBridgeError(() => reserve(store, pending.binding.bindingId, "too early"), "ATTACHMENT_REQUIRED"); + + // A second store: one live binding per host target and per chat is the + // invariant under test elsewhere, so the gated binding needs its own pair. + const gated = makeStore(); + const { binding } = createBinding(gated); + gated.manageBinding({ + bindingId: binding.bindingId, + action: "renew", + operationId: crypto.randomUUID(), + expectedRevision: binding.revision, + capabilityExpiresAt: "2020-01-01T00:00:00.000Z", + }); + expectBridgeError(() => reserve(gated, binding.bindingId, "after expiry"), "CAPABILITY_EXPIRED"); + expectBridgeError( + () => reserve(gated, binding.bindingId, "x".repeat(512 * 1024 + 1)), + "PROMPT_TOO_LARGE", + ); + }); + + test("one host target and one chat each hold at most one live binding", () => { + const store = makeStore(); + createBinding(store); + expectBridgeError( + () => store.createBinding({ + ownerRef: "owner:test", + host: { ...HOST }, + chatUrl: "https://chatgpt.com/c/99999999-90ab-4cde-8f01-234567890abc", + operationId: crypto.randomUUID(), + }), + "BINDING_EXISTS", + ); + const otherTarget = store.createBinding({ + ownerRef: "owner:test", + host: { ...HOST, targetId: "01987654-aaaa-7ccc-9ddd-ffffffffffff" }, + chatUrl: "https://chatgpt.com/c/99999999-90ab-4cde-8f01-234567890abc", + operationId: crypto.randomUUID(), + }); + expect(otherTarget.binding.bindingId).toBeDefined(); + // The create above leaves that chat conversation bound to the second target, so + // this is the only call that reaches the chat guard: the host guard cannot fire here + // (fresh targetId) and the operation replay cannot (fresh operationId). + expectBridgeError( + () => store.createBinding({ + ownerRef: "owner:test", + host: { ...HOST, targetId: "01987654-bbbb-7ccc-9ddd-eeeeeeeeeeee" }, + chatUrl: "https://chatgpt.com/c/99999999-90ab-4cde-8f01-234567890abc", + operationId: crypto.randomUUID(), + }), + "BINDING_EXISTS", + ); + }); }); describe("chatgpt-bridge legacy registry federation", () => { diff --git a/tests/chatgpt-bridge/chatgpt-bridge-dsh-host.test.ts b/tests/chatgpt-bridge/chatgpt-bridge-dsh-host.test.ts index ef20362bc2c..3799f86b3f9 100644 --- a/tests/chatgpt-bridge/chatgpt-bridge-dsh-host.test.ts +++ b/tests/chatgpt-bridge/chatgpt-bridge-dsh-host.test.ts @@ -1,29 +1,45 @@ -import { describe, expect, test } from "bun:test"; -import { createBridgePlugin, TOKEN_HEADER, type DshPluginContext, type DshSessionEvent, type DshUserMessage } from "../../extensions/dsh-chatgpt-bridge/src/host"; +import { describe, expect, setSystemTime, test } from "bun:test"; +import { + createBridgePlugin, + TOKEN_HEADER, + type DshAgent, + type DshPluginContext, + type DshSessionEvent, + type DshSessionHeader, + type DshUserMessage, +} from "../../extensions/dsh-chatgpt-bridge/src/host"; interface Listener { (payload: unknown): void | Promise; } -function makeCtx(agents: Map) { +interface FakeAgent { + followups: DshUserMessage[]; + header?: DshSessionHeader; +} + +function makeCtx(agents: Map) { const listeners = new Map>(); const storage = new Map(); const routes: Array<{ kind: string; path: string; handler: (r: unknown) => Promise }> = []; + const fakeAgent = (id: string, header: DshSessionHeader): DshAgent => ({ + id, + session: { header }, + followup: () => {}, + }); const ctx: DshPluginContext = { agents: { get: (id: string) => { const record = agents.get(id); if (!record) return undefined; return { - id, + ...fakeAgent(id, record.header ?? { id }), followup: (message: DshUserMessage) => { record.followups.push(message); - listeners.get("agent/inbox/inserted")?.forEach(l => l({ agent: { id, followup: () => {} }, message: { id: message.id } })); + listeners.get("agent/inbox/inserted")?.forEach(l => l({ agent: fakeAgent(id, record.header ?? { id }), message: { id: message.id } })); }, - whenIdle: async () => {}, }; }, - isOwnedBy: () => false, }, on: ((event: string, listener: Listener) => { if (!listeners.has(event)) listeners.set(event, new Set()); @@ -43,11 +59,15 @@ function makeCtx(agents: Map) { }; const emitClaimed = (agentId: string, inboxItemId: string, turn: number) => listeners.get("agent/inbox/claimed")?.forEach(l => - l({ agent: { id: agentId, followup: () => {} }, message: { id: inboxItemId }, turn }), + l({ agent: fakeAgent(agentId, { id: agentId }), message: { id: inboxItemId }, turn }), + ); + const emitDiscarded = (agentId: string, inboxItemId: string) => + listeners.get("agent/inbox/discarded")?.forEach(l => + l({ agent: fakeAgent(agentId, { id: agentId }), message: { id: inboxItemId } }), ); const emitSessionEvent = (sessionId: string, event: DshSessionEvent) => listeners.get("session/event")?.forEach(l => l({ header: { id: sessionId } }, event)); - return { ctx, listeners, storage, routes, emitClaimed, emitSessionEvent }; + return { ctx, listeners, storage, routes, emitClaimed, emitDiscarded, emitSessionEvent }; } const TOKEN = "probe-token"; @@ -167,11 +187,112 @@ describe("dsh chatgpt-bridge host plugin", () => { expect((swapped.body as { code: string }).code).toBe("BINDING_CHANGED"); }); - test("subagent-owned sessions are rejected on lineage even when the parent is gone", () => { - const { ctx } = makeCtx(new Map()); + test("subagent-owned sessions are rejected on the deliver path, origin or lineage", async () => { + const { ctx } = makeCtx(new Map([ + ["child-by-origin", { followups: [], header: { id: "child-by-origin", origin: "subagent" as const } }], + ["child-by-lineage", { followups: [], header: { id: "child-by-lineage", parentSession: "parent-1" } }], + ])); + const plugin = createBridgePlugin(ctx, { controlToken: TOKEN }); + for (const sessionId of ["child-by-origin", "child-by-lineage"]) { + const response = await plugin.handler({ + method: "POST", + path: `/chatgpt-bridge/${sessionId}/deliver`, + headers: authHeaders, + body: { message: "go", inboxItemId: "item-1" }, + }); + expect(response.status, sessionId).toBe(409); + expect((response.body as { code: string }).code, sessionId).toBe("CONTEXT_INCOMPATIBLE"); + } + }); + + test("two concurrent deliveries cannot both be accepted: the second loses nothing it delivered", async () => { + const followups: DshUserMessage[] = []; + const { ctx } = makeCtx(new Map([["session-1", { followups }]])); + const plugin = createBridgePlugin(ctx, { controlToken: TOKEN }); + const deliver = (inboxItemId: string) => plugin.handler({ + method: "POST", + path: "/chatgpt-bridge/session-1/deliver", + headers: authHeaders, + body: { message: `prompt ${inboxItemId}`, inboxItemId }, + }); + const [first, second] = await Promise.all([deliver("item-A"), deliver("item-B")]); + const accepted = [first, second].filter(response => response.status === 202); + expect(accepted).toHaveLength(1); + expect(followups).toHaveLength(1); + const refused = [first, second].find(response => response.status !== 202)!; + expect((refused.body as { code: string }).code).toBe("TARGET_ACTIVE"); + const view = await plugin.handler({ method: "GET", path: "/chatgpt-bridge/session-1/binding", headers: authHeaders }); + expect((view.body as { binding: { lastDeliveredInboxItemId: string } }).binding.lastDeliveredInboxItemId) + .toBe((accepted[0]!.body as { inboxItemId: string }).inboxItemId); + }); + + test("a discarded delivery that was never claimed frees the binding for the next send", async () => { + const { ctx, emitDiscarded } = makeCtx(new Map([["session-1", { followups: [] }]])); const plugin = createBridgePlugin(ctx, { controlToken: TOKEN }); - expect(plugin.isSubagentOwned({ id: "s1", origin: "subagent" })).toBe(true); - expect(plugin.isSubagentOwned({ id: "s2", parentSession: "parent-1" })).toBe(true); - expect(plugin.isSubagentOwned({ id: "s3" })).toBe(false); + const deliver = (inboxItemId: string) => plugin.handler({ + method: "POST", + path: "/chatgpt-bridge/session-1/deliver", + headers: authHeaders, + body: { message: `prompt ${inboxItemId}`, inboxItemId }, + }); + await deliver("item-1"); + const blocked = await deliver("item-2"); + expect((blocked.body as { code: string }).code).toBe("TARGET_ACTIVE"); + emitDiscarded("session-1", "item-1"); + const retry = await deliver("item-2"); + expect(retry.status).toBe(202); + }); + + test("an unclaimed delivery stops holding the gate once it is older than the stale window", async () => { + // The host may never emit `agent/inbox/discarded`; that must not wedge the + // session for the rest of its life. + const session1 = { followups: [] as DshUserMessage[] }; + const { ctx } = makeCtx(new Map([["session-1", session1]])); + const plugin = createBridgePlugin(ctx, { controlToken: TOKEN }); + const deliver = (inboxItemId: string) => plugin.handler({ + method: "POST", + path: "/chatgpt-bridge/session-1/deliver", + headers: authHeaders, + body: { message: `prompt ${inboxItemId}`, inboxItemId }, + }); + const at = (seconds: number) => setSystemTime(new Date(Date.UTC(2026, 0, 1, 0, 0, seconds))); + try { + at(0); + await deliver("item-1"); + expect((await deliver("item-2")).status).toBe(409); + // Pin the ceiling from below as well: anything <= 119 s would already let item-2 + // through here, and the stale window the core uses is 120 s. + at(119); + expect((await deliver("item-2")).status).toBe(409); + at(121); + expect((await deliver("item-2")).status).toBe(202); + // A 202 must mean work was actually enqueued, not just a status code. + expect(session1.followups.map(message => message.id)).toEqual(["item-1", "item-2"]); + + // A delivery whose turn the host did claim still refuses: that turn may be + // running, and overwriting it would leave the answer unattributable. + const { ctx: claimedCtx, emitClaimed } = makeCtx(new Map([["session-2", { followups: [] }]])); + const claimed = createBridgePlugin(claimedCtx, { controlToken: TOKEN }); + const toClaimed = (inboxItemId: string) => claimed.handler({ + method: "POST", path: "/chatgpt-bridge/session-2/deliver", + headers: authHeaders, body: { message: `prompt ${inboxItemId}`, inboxItemId }, + }); + at(0); + expect((await toClaimed("item-3")).status).toBe(202); + emitClaimed("session-2", "item-3", 11); + await Bun.sleep(0); + const view = await claimed.handler({ + method: "GET", path: "/chatgpt-bridge/session-2/binding", headers: authHeaders, + }); + // Without this the assertion below could be the unclaimed branch answering. + expect((view.body as { binding: { lastClaimedTurn: number | null } }).binding.lastClaimedTurn).toBe(11); + at(121); + expect((await toClaimed("item-4")).status).toBe(409); + expect((await claimed.handler({ + method: "GET", path: "/chatgpt-bridge/session-2/binding", headers: authHeaders, + })).body).toMatchObject({ binding: { lastDeliveredInboxItemId: "item-3" } }); + } finally { + setSystemTime(null); + } }); }); From ff8951222f9fbe6cfe7bf88bcd549f68f1dcaa38 Mon Sep 17 00:00:00 2001 From: Codex Developer Date: Fri, 25 Sep 2026 02:09:37 +0800 Subject: [PATCH 6/7] refactor(diagnostics): drop the unused module-level sync-operation instrument `beginRuntimeSyncOperation` was a second entry into the same instrumentation the start scope already owns, and it needed three module globals plus their teardown assignments to reach them. Nothing outside the module called it, and a second live diagnostics instance would have had those globals point at whichever child started last. Verification: `server-runtime-diagnostics` green; `bun x tsc --noEmit` green. --- src/lib/runtime-diagnostics.ts | 41 +------------------ .../server/server-runtime-diagnostics.test.ts | 15 +++++-- 2 files changed, 13 insertions(+), 43 deletions(-) diff --git a/src/lib/runtime-diagnostics.ts b/src/lib/runtime-diagnostics.ts index 630e717456b..5d0a3234a07 100644 --- a/src/lib/runtime-diagnostics.ts +++ b/src/lib/runtime-diagnostics.ts @@ -1,13 +1,11 @@ -import { fork, type ChildProcess, type ForkOptions } from "node:child_process"; +import { fork, type ForkOptions } from "node:child_process"; import { fileURLToPath } from "node:url"; // Opt-in, scalar-only diagnostics. A separate process keeps writing even when the // server's event loop is blocked in a synchronous native call. No request data crosses IPC. export type RuntimeCounters = Record; -let child: ChildProcess | null = null; type DiagnosticDeliveryCallback = () => void; type RuntimeDiagnosticSender = (message: object, onDelivered?: DiagnosticDeliveryCallback) => boolean; -let sendToChild: RuntimeDiagnosticSender | null = null; let sequence = 0; const MAX_DIAGNOSTIC_IPC_IN_FLIGHT = 4; @@ -156,30 +154,6 @@ export function createBoundedRuntimeDiagnosticSender(target: DiagnosticIpcTarget }; } -let completedSlowOperationSink: ((operation: CompletedSlowOperation) => void) | null = null; - -export function beginRuntimeSyncOperation(): () => void { - const send = sendToChild; - if (!child?.connected || !send) return () => {}; - const id = ++sequence; - const startedAt = performance.now(); - // Keep code locations, never absolute user paths, arguments, or error messages. - const sites = (new Error().stack ?? "").split("\n").flatMap(line => { - const match = /[\\/](src[\\/][\w./\\-]+:\d+:\d+)/.exec(line); - return match ? [match[1]!.replaceAll("\\", "/")] : []; - }).slice(1, 9); - const started = send({ kind: "sync-start", at: Date.now(), id, sites }); - const complete = completedSlowOperationSink; - return () => { - const elapsedMs = Math.max(0, performance.now() - startedAt); - if (elapsedMs >= SLOW_SYNC_OPERATION_MS && complete) { - complete({ sequence: ++sequence, id, elapsedMs, sites }); - } else if (started) { - send({ kind: "sync-end", at: Date.now(), id }); - } - }; -} - export function startRuntimeDiagnostics( path: string, sample: () => RuntimeCounters, @@ -195,14 +169,8 @@ export function startRuntimeDiagnostics( detached: process.platform === "win32", }; const target = fork(fileURLToPath(new URL("./runtime-diagnostics-child.ts", import.meta.url)), [], launchOptions); - child = target; const send = createBoundedRuntimeDiagnosticSender(target); - sendToChild = send; const completedSlowOperations = createCompletedSlowOperationRingForTests(); - const enqueueCompletedSlowOperation = (operation: CompletedSlowOperation) => { - completedSlowOperations.enqueue(operation); - }; - completedSlowOperationSink = enqueueCompletedSlowOperation; let lastSampleAt = performance.now(); let lastCpu = process.cpuUsage(); let sampleFailures = 0; @@ -224,7 +192,7 @@ export function startRuntimeDiagnostics( return () => { const elapsedMs = Math.max(0, performance.now() - startedAt); if (elapsedMs >= SLOW_SYNC_OPERATION_MS) { - enqueueCompletedSlowOperation({ sequence: ++sequence, id, elapsedMs, sites: [phase] }); + completedSlowOperations.enqueue({ sequence: ++sequence, id, elapsedMs, sites: [phase] }); } else if (started) { send({ kind: "sync-end", at: Date.now(), id }); } @@ -293,9 +261,6 @@ export function startRuntimeDiagnostics( if (closedSettled) return; closedSettled = true; if (timer) clearInterval(timer); - if (child === target) child = null; - if (sendToChild === send) sendToChild = null; - if (completedSlowOperationSink === enqueueCompletedSlowOperation) completedSlowOperationSink = null; rejectReady(); resolveClosed(); }; @@ -364,8 +329,6 @@ export function startRuntimeDiagnostics( return { ready, closed, stop() { stopRequested = true; clearInterval(timer); - if (child === target) child = null; - if (sendToChild === send) sendToChild = null; if (target.connected) { // Do not extend shutdown, but make one final bounded attempt to carry a // completed slow operation that ended between ordinary heartbeats. diff --git a/tests/server/server-runtime-diagnostics.test.ts b/tests/server/server-runtime-diagnostics.test.ts index 2eec935c956..b36f368f9dd 100644 --- a/tests/server/server-runtime-diagnostics.test.ts +++ b/tests/server/server-runtime-diagnostics.test.ts @@ -5,13 +5,13 @@ import { join } from "node:path"; import { fork } from "node:child_process"; import { fileURLToPath } from "node:url"; import { - beginRuntimeSyncOperation, createBoundedRuntimeDiagnosticSender, createCompletedSlowOperationRingForTests, disconnectAfterRuntimeDiagnosticsStop, startRuntimeDiagnostics, } from "../../src/lib/runtime-diagnostics"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { resolvedImportEdges } from "../helpers/import-graph"; test("independent recorder observes a blocked parent and stops with its owner", async () => { const dir = mkdtempSync(join(tmpdir(), "ocx-runtime-diagnostics-")); @@ -22,9 +22,7 @@ test("independent recorder observes a blocked parent and stops with its owner", try { await Promise.race([recorder.ready, Bun.sleep(3000).then(() => { throw new Error("recorder did not start"); })]); await Bun.sleep(100); - const end = beginRuntimeSyncOperation(); Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 400); - end(); await Bun.sleep(100); } finally { recorder.stop(); @@ -34,7 +32,6 @@ test("independent recorder observes a blocked parent and stops with its owner", const content = readFileSync(path, "utf8"); const records = content.trim().split("\n").map(line => JSON.parse(line)); expect(records.some(r => r.kind === "event-loop-stall" && r.heartbeatGapMs >= 100)).toBe(true); - expect(records.some(r => r.kind === "slow-sync-operation" && r.elapsedMs >= 350)).toBe(true); expect(records.some(r => r.kind === "event-loop-recovered")).toBe(true); const delay = records.find(r => r.kind === "event-loop-delay"); expect(delay.timerDelayMs).toBeGreaterThanOrEqual(300); @@ -429,3 +426,13 @@ test("stopping before child initialization rejects ready and closes without an o await expect(ready).rejects.toThrow("runtime diagnostics recorder closed before ready"); } finally { removeTreeWithRetry(dir); } }); + +test("the server module reaches the recorder only through its opt-in gate", () => { + // The recorder stays behind the env gate: only the desktop launcher opts in, so a static + // edge would have loaded this module on every install's server start for a branch that + // most installs never take. + const edges = resolvedImportEdges("src/server/background-lifecycle.ts") + .filter(edge => edge.spec.includes("runtime-diagnostics")); + expect(edges.map(edge => ({ spec: edge.spec, dynamic: edge.dynamic }))) + .toEqual([{ spec: "../lib/runtime-diagnostics", dynamic: true }]); +}); From 98084190837c9851175c379a16843ddc7461c7be Mon Sep 17 00:00:00 2001 From: Codex Developer Date: Fri, 25 Sep 2026 02:10:30 +0800 Subject: [PATCH 7/7] fix(bridge): stop publishing a chatgpt-web dashboard tile that can only 503 `src/adapters/registry.ts` does not construct the adapter with a browser transport yet, so every turn through a preset-selected chatgpt-web model answers CHATGPT_WEB_TRANSPORT_UNAVAILABLE. A published preset therefore promised a capability the shipped build cannot perform; the registry entry and its note stay, only the preset flag goes off until the transport is attached. Also register `src/chatgpt-bridge/` in the structure source-area map, which the manifest guard requires for a new production directory. --- src/providers/registry/entries-core.ts | 5 ++++- structure/INDEX.md | 1 + structure/manifest.json | 1 + structure/providers-and-adapters.md | 1 + 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/providers/registry/entries-core.ts b/src/providers/registry/entries-core.ts index 55d23d44b9c..a7a1a6fc5a4 100644 --- a/src/providers/registry/entries-core.ts +++ b/src/providers/registry/entries-core.ts @@ -93,7 +93,10 @@ export const PROVIDER_REGISTRY_CORE: readonly ProviderRegistryEntry[] = [ baseUrl: "https://chatgpt.com", authKind: "local", featured: false, - dashboardPreset: true, + // Off until `src/adapters/registry.ts` constructs this adapter with a browser transport + // in `deps`: without one every turn 503s with CHATGPT_WEB_TRANSPORT_UNAVAILABLE, so a + // published tile promises something the adapter cannot do. + dashboardPreset: false, note: "Web ChatGPT models served through the chatgpt-bridge module (browser transport). Turns fail with CHATGPT_WEB_TRANSPORT_UNAVAILABLE until the module browser transport is enabled; no keys are stored and web login lives in the managed browser profile.", models: ["chatgpt-web/luna", "chatgpt-web/instant", "chatgpt-web/medium", "chatgpt-web/high"], liveModels: false, diff --git a/structure/INDEX.md b/structure/INDEX.md index 129e4c392d4..8731def50f3 100644 --- a/structure/INDEX.md +++ b/structure/INDEX.md @@ -100,6 +100,7 @@ A source area can be described by more than one doc, because these docs are orga | `scripts/` | [`overview.md`](overview.md)
[`ops/docs-and-release.md`](ops/docs-and-release.md) | | `src/adapters/` | [`runtime.md`](runtime.md)
[`transports/byte-accounting.md`](transports/byte-accounting.md)
[`transports/responses.md`](transports/responses.md)
[`transports/inventory.md`](transports/inventory.md)
[`data-planes/inbound-compat.md`](data-planes/inbound-compat.md)
[`providers-and-adapters.md`](providers-and-adapters.md)
[`providers/cursor.md`](providers/cursor.md)
[`providers/chat-compat.md`](providers/chat-compat.md)
[`adapters/registry.md`](adapters/registry.md) | | `src/chat/` | [`runtime.md`](runtime.md)
[`transports/inventory.md`](transports/inventory.md)
[`data-planes/inbound-compat.md`](data-planes/inbound-compat.md)
[`providers-and-adapters.md`](providers-and-adapters.md) | +| `src/chatgpt-bridge/` | [`providers-and-adapters.md`](providers-and-adapters.md) | | `src/claude/` | [`runtime.md`](runtime.md)
[`clients/claude-desktop.md`](clients/claude-desktop.md) | | `src/cli.ts` | [`runtime.md`](runtime.md)
[`ops/docs-and-release.md`](ops/docs-and-release.md) | | `src/cli/` | [`runtime.md`](runtime.md)
[`config.md`](config.md)
[`clients/claude-desktop.md`](clients/claude-desktop.md)
[`ops/docs-and-release.md`](ops/docs-and-release.md) | diff --git a/structure/manifest.json b/structure/manifest.json index 702372d40b5..052f58355ee 100644 --- a/structure/manifest.json +++ b/structure/manifest.json @@ -227,6 +227,7 @@ "documents": [ "src/adapters/", "src/chat/", + "src/chatgpt-bridge/", "src/combos/", "src/oauth/", "src/providers/", diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index bf4271d54fe..b719b1d68af 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -28,6 +28,7 @@ only canonical Fable, Opus, or Sonnet labels after removing terminal controls; u | `src/adapters/devin.ts`, `src/adapters/devin/cloud-direct/` | Devin runTurn transport over Cognition Connect-RPC. `GetChatMessage` uses the Responses provider executor and shared physical-send budget; catalog and JWT support RPCs remain outside inference-send accounting. Provider-stated 429 reset delays are surfaced to the client rather than slept inside an admitted turn, so they cannot retain shared active-turn capacity. A recorded tenant host is used only for the stored account whose credential owns the transmitted key, searched in the configured provider id and then its deprecated alias; a configured, forwarded, or unmatched key uses the configured base URL or the US default. | | `src/adapters/kiro.ts` and `src/adapters/kiro/` | Kiro event/tool/thinking/truncation/retry handling. The original path is a facade over leaves for wire identity, reasoning, conversation state, token estimation, payload assembly, streaming, and the adapter. | | `src/adapters/mimo-free.ts` | Mimo Free transport (client identity + JWT). | +| `src/chatgpt-bridge/` | ChatGPT web bridge. `src/chatgpt-bridge/provider/adapter.ts` is the `chatgpt-web` runTurn seam, and it fails every turn with `CHATGPT_WEB_TRANSPORT_UNAVAILABLE` until a browser transport is injected at construction. `src/chatgpt-bridge/core/store.ts` is the durable binding and delivery ledger (digests, ids and receipts only, never prompt text) over the vocabulary in `src/chatgpt-bridge/contracts/index.ts`; `src/chatgpt-bridge/hosts/codex/` is a read-only federation view of the legacy DevSpace registry and its MCP control plane. None of it is on a request path: the adapter registry is the only production importer. | | `src/adapters/image.ts`, `src/adapters/anthropic-image-guard.ts`, `src/adapters/anthropic-image-normalize.ts`, `src/adapters/anthropic-image-codec.ts` | Image conversion for adapter ingress and Anthropic-specific normalization/limits. An image's ladder position is pinned to its own identity (content hash + media type), so appending a newer image cannot re-encode older ones and bust Anthropic's prompt prefix cache (#4532). | | `src/adapters/run-turn-queue.ts`, `src/adapters/tool-catalog-nudge.ts`, `src/adapters/identity.ts`, `src/adapters/upstream-http-error.ts` | Shared adapter execution support: turn queueing, tool-catalog nudging, client identity, upstream error normalization. |