diff --git a/docs-site/src/content/docs/reference/cli/lifecycle.md b/docs-site/src/content/docs/reference/cli/lifecycle.md index 21dbee3153a..d2f6d1aa219 100644 --- a/docs-site/src/content/docs/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/reference/cli/lifecycle.md @@ -164,6 +164,15 @@ default provider, Codex autostart setting, service state, shim state, and the re home. Only the explicit, high-confidence Windows Orca runtime-home signature adds an actionable App-home mismatch warning; it never changes `CODEX_HOME` automatically. +When a live proxy has already passed the identity/liveness check, `ocx status` prefers that process's +attested startup-health report for restart safety and service viability. This avoids false negatives +from a shell-local service-manager probe that lacks the running service's manager environment. The +live report is schema-validated; if it is unavailable or malformed, status falls back to the local +service and shim diagnostics. `ocx doctor` uses the same live-first rule for its **Codex restart safety** +section, so the two commands should agree on restart protection. If you are diagnosing a discrepancy, +compare the reported live startup verdict with the local service details rather than treating the shell +probe as more authoritative. + Human output also includes an **OAuth health** block after the OAuth logins summary: `OAuth health: ok` when every known account is healthy, or `OAuth health: warning` with one redacted line per non-healthy account (provider, masked account id, status such as reauthentication required, rate or diff --git a/src/cli/doctor.ts b/src/cli/doctor.ts index 42482a2d9cf..01b97e6af0c 100644 --- a/src/cli/doctor.ts +++ b/src/cli/doctor.ts @@ -12,7 +12,7 @@ import { homedir } from "node:os"; import { dirname, join } from "node:path"; import { getConfigDir, getConfigPath, readConfigDiagnostics } from "../config"; import { readPid } from "../config/process-state"; -import { probeUncleanExitState } from "./status"; +import { fetchLiveStartupHealth, probeUncleanExitState, selectStatusStartupHealth } from "./status"; import { findLiveProxy, probeHostname, type LiveProxy } from "../server/proxy-liveness"; import { directLocalHttpFetch } from "../server/direct-local-http"; import { BUN_RUNTIME_SOURCES } from "../lib/bun-runtime"; @@ -1354,7 +1354,14 @@ export async function runDoctor(args: string[] = []): Promise { diagnoseCodexShim(), serviceTokenPresent, ); - const startup = collectStartupHealth(doctorConfig); + // Use the same attested live startup verdict as `ocx status` when the proxy is already + // identity-verified. A shell-local systemd probe can be a false negative for a system-wide + // service because the shell does not inherit the manager-owned environment. + const live = await findLiveProxy({ + configFn: () => ({ port: doctorConfig.port, hostname: doctorConfig.hostname }), + }); + const liveStartup = live ? await fetchLiveStartupHealth(live) : null; + const startup = selectStatusStartupHealth(liveStartup, () => collectStartupHealth(doctorConfig)); console.log("\nCodex restart safety"); console.log(` ${startup.rebootSafe ? "ok " : "!! "} ${startupHealthSummary(startup)}`); console.log(` ${formatStartupRoutingDetail(startup)}`); @@ -1393,12 +1400,6 @@ export async function runDoctor(args: string[] = []): Promise { } } - // #618: identity-verified liveness first so pid-file absence does not hide a live service. - // Reuse the diagnostics config already loaded above so doctor stays read-only on malformed JSON. - const live = await findLiveProxy({ - configFn: () => ({ port: doctorConfig.port, hostname: doctorConfig.hostname }), - }); - // Mirrors `ocx status` through the same comparison rather than a second implementation: // two diagnostics disagreeing about whether an install is stale is worse than one (#2701). // No extra probe -- findLiveProxy already carried the version back. diff --git a/src/cli/status.ts b/src/cli/status.ts index 9079a2b5f90..25ddffa52ee 100644 --- a/src/cli/status.ts +++ b/src/cli/status.ts @@ -29,6 +29,8 @@ import { tokenCollidesWithAdmin } from "../lib/admin-secrets"; export { proxyHealthFailureReason, isConnectionRefused, isUncleanExitEvidence, probeUncleanExitState } from "./status-probes"; export type { ListenTarget } from "./status-probes"; import { checkProxyHealth, probeUncleanExitState, type ListenTarget } from "./status-probes"; +import { LOCAL_MANAGEMENT_READ_PATHS } from "../lib/local-management-capability"; +import { fetchBoundLocalManagementRead } from "../server/local-management-read-client"; /** * The state of the data-plane admission secret the SERVICE will use. State only -- never the value. @@ -233,6 +235,84 @@ function statusDashboardUrl(config: StatusListenConfig, hostname: string | undef return `http://${dashboardHostname}:${port}/`; } +const STARTUP_HEALTH_BOOLEAN_FIELDS = [ + "routingInjected", "localRoutingDependency", "autostartEnabled", "rebootSafe", + "serviceInstalled", "serviceViable", "serviceEnabled", "serviceRunning", + "serviceStale", "serviceConflict", "shimInstalled", "shimHealthy", + "serviceSupported", "diagnosticStale", +] as const; + +export async function fetchLiveStartupHealth( + live: NonNullable>>, + deps: Parameters[2] = {}, +): Promise { + const result = await fetchBoundLocalManagementRead( + live, LOCAL_MANAGEMENT_READ_PATHS.startupHealth, { timeoutMs: 1_500, ...deps }, + ); + if (result.kind !== "response" || !result.response.ok) return null; + let payload: unknown; + try { payload = await result.response.json(); } catch { return null; } + if (!payload || typeof payload !== "object" || Array.isArray(payload)) return null; + const row = payload as Record; + if (row.status !== "native" && row.status !== "protected" && row.status !== "at-risk") return null; + if (row.protection !== "service" && row.protection !== "shim" && row.protection !== "none") return null; + if (row.routingKind !== "native" && row.routingKind !== "opencodex-local" + && row.routingKind !== "custom-local" && row.routingKind !== "custom-remote" && row.routingKind !== "unknown") return null; + if (row.shimCoverage !== "full" && row.shimCoverage !== "cli-only" && row.shimCoverage !== "none") return null; + if (typeof row.platform !== "string") return null; + if (row.recommendedCommand !== null && typeof row.recommendedCommand !== "string") return null; + if (!row.commands || typeof row.commands !== "object" || Array.isArray(row.commands)) return null; + for (const key of ["installService", "repairService", "installShim", "restoreNative"] as const) { + if (typeof (row.commands as Record)[key] !== "string") return null; + } + if (row.routingAdoption !== undefined) { + if (!row.routingAdoption || typeof row.routingAdoption !== "object" || Array.isArray(row.routingAdoption)) return null; + const adoption = row.routingAdoption as Record; + if (adoption.adoption !== "not-applicable" && adoption.adoption !== "adopted" + && adoption.adoption !== "pending-client-restart" && adoption.adoption !== "unknown") return null; + if (adoption.injectedAtMs !== null && typeof adoption.injectedAtMs !== "number") return null; + if (typeof adoption.observedClients !== "number" || !Array.isArray(adoption.staleClients)) return null; + for (const client of adoption.staleClients) { + if (!client || typeof client !== "object" || Array.isArray(client)) return null; + const row = client as Record; + if (typeof row.pid !== "number" || typeof row.startedAtMs !== "number") return null; + } + } + for (const key of STARTUP_HEALTH_BOOLEAN_FIELDS) if (typeof row[key] !== "boolean") return null; + return payload as StartupHealth; +} + +/** Prefer an attested live verdict and evaluate the local fallback only when live state is absent. */ +export function selectStatusStartupHealth( + liveStartup: StartupHealth | null, + fallback: () => StartupHealth, +): StartupHealth { + return liveStartup ?? fallback(); +} + +/** Build the service summary from the same startup source that `ocx status` selected. */ +export function statusServiceSummary( + liveStartup: StartupHealth | null, + service: Pick, "installed" | "summary">, + live: boolean, +): string { + if (liveStartup) { + if (liveStartup.protection === "service" && liveStartup.serviceViable) { + return `running under the live managed service (logs: ${serviceLogPath()})`; + } + const state = [ + liveStartup.serviceInstalled ? "installed" : "absent", + liveStartup.serviceRunning ? "running" : "not running", + liveStartup.serviceViable ? "viable" : "not viable", + ].join(", "); + const action = liveStartup.recommendedCommand ? `; run '${liveStartup.recommendedCommand}'` : ""; + return `live startup reports service ${state}${action} (logs: ${serviceLogPath()})`; + } + return service.installed && !live + ? `${service.summary} — registered but NOT serving; see ${serviceLogPath()} and re-run 'ocx service repair'` + : service.summary; +} + /** * The hub block, or null when this machine is not a hub. * @@ -634,20 +714,19 @@ export async function collectStatus(): Promise { hostname: config.hostname, }); const bunRuntime = durableBunRuntime(); + const liveStartup = live ? await fetchLiveStartupHealth(live) : null; const service = diagnoseService(); // A service can be registered and still not serve: the manager reports the job - // either way. `live` was already identity-probed a few lines above, so cross-check - // rather than print registration as if it were service. - const serviceSummary = service.installed && !live - ? `${service.summary} — registered but NOT serving; see ${serviceLogPath()} and re-run 'ocx service repair'` - : service.summary; + // either way. When the identity-probed live proxy provides an attested startup verdict, + // prefer it over a shell-local service-manager probe that lacks the service environment. + const serviceSummary = statusServiceSummary(liveStartup, service, Boolean(live)); const codexShim = diagnoseCodexShim(); const codexShimSummary = codexShim.summary; - const startup = collectStartupHealth(config, { + const startup = selectStatusStartupHealth(liveStartup, () => collectStartupHealth(config, { service, shim: codexShim, routingKind: getCodexRoutingKind(), - }); + })); const codexPlugins = diagnoseCodexBundledPlugins(); const lastClamp = loadLastEffortClamp(); const clampActive = effortClampAppliesToRuntime(lastClamp, resolvedRuntime.runtime); diff --git a/tests/cli/cli-status-startup-health.test.ts b/tests/cli/cli-status-startup-health.test.ts new file mode 100644 index 00000000000..fd51366b7ad --- /dev/null +++ b/tests/cli/cli-status-startup-health.test.ts @@ -0,0 +1,133 @@ +import { describe, expect, test } from "bun:test"; +import { fetchLiveStartupHealth, selectStatusStartupHealth, statusServiceSummary } from "../../src/cli/status"; +import type { StartupHealth } from "../../src/codex/autostart-health"; + +const LIVE = { + pid: 4242, + port: 10101, + hostname: "127.0.0.1", + source: "runtime" as const, +}; + +const SECRET = "A".repeat(43); +const NONCE = "B".repeat(43); + +function startupPayload() { + return { + status: "protected", + routingKind: "opencodex-local", + routingInjected: true, + localRoutingDependency: true, + autostartEnabled: true, + rebootSafe: true, + protection: "service", + serviceInstalled: true, + serviceViable: true, + serviceEnabled: true, + serviceRunning: true, + serviceStale: false, + serviceConflict: false, + shimInstalled: true, + shimHealthy: true, + shimCoverage: "cli-only", + serviceSupported: true, + platform: "linux", + diagnosticStale: false, + recommendedCommand: null, + commands: { + installService: "ocx service install", + repairService: "ocx service repair", + installShim: "ocx codex-shim install", + restoreNative: "ocx restore", + }, + }; +} + +function deps(body: unknown) { + return { + readRuntime: () => ({ + pid: LIVE.pid, + port: LIVE.port, + hostname: LIVE.hostname, + attestationSecret: SECRET, + }), + createNonce: () => NONCE, + now: () => 1_000, + fetchImpl: async () => new Response(JSON.stringify(body), { + status: 200, + headers: { "content-type": "application/json" }, + }), + }; +} + +describe("ocx status live startup health", () => { + test("uses an attested live startup verdict when the shell-local service probe would disagree", async () => { + const observed = await fetchLiveStartupHealth(LIVE, deps(startupPayload())); + expect(observed?.status).toBe("protected"); + expect(observed?.rebootSafe).toBe(true); + expect(observed?.serviceViable).toBe(true); + expect(observed?.protection).toBe("service"); + }); + + test("rejects malformed live startup payloads", async () => { + for (const malformed of [ + { ...startupPayload(), serviceRunning: "yes" }, + (() => { const row = { ...startupPayload() } as Record; delete row.platform; return row; })(), + { ...startupPayload(), recommendedCommand: 7 }, + { ...startupPayload(), commands: { installService: "ok" } }, + { ...startupPayload(), routingAdoption: { adoption: "adopted", injectedAtMs: 1, staleClients: "bad", observedClients: 1 } }, + { ...startupPayload(), routingAdoption: { adoption: "adopted", injectedAtMs: 1, staleClients: [{ pid: "bad", startedAtMs: 1 }], observedClients: 1 } }, + ]) { + expect(await fetchLiveStartupHealth(LIVE, deps(malformed))).toBeNull(); + } + }); + + test("selection prefers the attested live verdict and does not evaluate the conflicting fallback", () => { + const live = startupPayload() as StartupHealth; + let fallbackCalls = 0; + const selected = selectStatusStartupHealth(live, () => { + fallbackCalls += 1; + return { ...live, status: "at-risk", rebootSafe: false, protection: "none" } as StartupHealth; + }); + expect(selected.status).toBe("protected"); + expect(selected.rebootSafe).toBe(true); + expect(fallbackCalls).toBe(0); + expect(statusServiceSummary(live, { installed: false, summary: "systemd not found" }, true)) + .toContain("running under the live managed service"); + }); + + test("selection falls back to local startup diagnostics when the live read is unavailable", () => { + const local = { ...startupPayload(), status: "at-risk", rebootSafe: false, protection: "none" } as StartupHealth; + let fallbackCalls = 0; + const selected = selectStatusStartupHealth(null, () => { fallbackCalls += 1; return local; }); + expect(selected).toBe(local); + expect(fallbackCalls).toBe(1); + expect(statusServiceSummary(null, { installed: true, summary: "registered" }, false)) + .toContain("registered but NOT serving"); + }); + + test("service summary never contradicts a present negative live startup verdict", () => { + const live = { + ...startupPayload(), + status: "at-risk", + rebootSafe: false, + protection: "none", + serviceInstalled: false, + serviceRunning: false, + serviceViable: false, + recommendedCommand: "ocx service repair", + } as StartupHealth; + const summary = statusServiceSummary(live, { installed: true, summary: "healthy local service" }, true); + expect(summary).toContain("live startup reports service absent, not running, not viable"); + expect(summary).toContain("ocx service repair"); + expect(summary).not.toContain("healthy local service"); + }); + + test("fails closed when the runtime attestation cannot bind the live PID", async () => { + const observed = await fetchLiveStartupHealth(LIVE, { + ...deps(startupPayload()), + readRuntime: () => null, + }); + expect(observed).toBeNull(); + }); +}); \ No newline at end of file