diff --git a/plugins/codex-security/mcp-app/helpers-main.ts b/plugins/codex-security/mcp-app/helpers-main.ts index 15d5e62481..fabe6490ba 100644 --- a/plugins/codex-security/mcp-app/helpers-main.ts +++ b/plugins/codex-security/mcp-app/helpers-main.ts @@ -1,4 +1,7 @@ -import { closeSync, readFileSync } from "node:fs"; +import { closeSync, readFileSync, existsSync, realpathSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +export { parseCanonicalScanDraft } from "./src/artifact-scan-draft.js"; +export { resumeSelectedDeepScan } from "./src/deep-scan/finalization.js"; import { resolveSecurityMdCommand } from "./src/helpers/resolve-security-md"; import { decodePosixBytes } from "./src/helpers/posix-path"; import { windowsBinding } from "./src/native"; @@ -9,58 +12,72 @@ import { rankShardsCommand } from "./src/helpers/rank-shards"; import { rankPoolCommand } from "./src/helpers/rank-pool"; import { bindRepoScopesCommand } from "./src/helpers/bind-repo-scopes"; -let commandLine = process.argv.slice(2); -if (process.platform === "win32") { - const original = windowsBinding().windowsArguments(); - commandLine = original - .slice(original.length - commandLine.length) - .map((argument) => argument.toString("utf16le")); -} -let posixHome = process.env.HOME; -if (commandLine[0] === "--helper") { +// Importing the bundled helper from the SDK does not invoke its CLI adapter. +const entryPath = import.meta.url.startsWith("file:") + ? fileURLToPath(import.meta.url) + : import.meta.url; +const invokedPath = process.argv[1]; +if ( + invokedPath && + existsSync(invokedPath) && + realpathSync(invokedPath) === realpathSync(entryPath) +) + runHelper(); + +function runHelper(): void { + let commandLine = process.argv.slice(2); if (process.platform === "win32") { - commandLine = commandLine.slice(1); + const original = windowsBinding().windowsArguments(); + commandLine = original + .slice(original.length - commandLine.length) + .map((argument) => argument.toString("utf16le")); + } + let posixHome = process.env.HOME; + if (commandLine[0] === "--helper") { + if (process.platform === "win32") { + commandLine = commandLine.slice(1); + } else { + const encoded = readFileSync(3, "ascii"); + closeSync(3); + const [homeSet, home, ...args] = decodePosixBytes( + Buffer.from(encoded.trim(), "hex"), + ) + .split("\0") + .slice(0, -1); + posixHome = homeSet ? home : undefined; + commandLine = args; + } + } + const [command, ...args] = commandLine; + if (command === "resolve-security-md") { + process.exitCode = resolveSecurityMdCommand(args, posixHome); + } else if (command === "normalize-candidates") { + process.exitCode = normalizeCandidatesCommand(args, posixHome); + } else if (command === "validate-patch-risk-assessment") { + process.exitCode = validatePatchRiskAssessmentCommand(args); + } else if ( + command === "copy-deep-review-input" || + command === "select-deep-review-input" + ) { + process.exitCode = deepReviewInputCommand(command, args, posixHome); + } else if ( + command === "make-rank-shards" || + command === "validate-rank-shard" || + command === "merge-rank-outputs" + ) { + process.exitCode = rankShardsCommand(command, args, posixHome); + } else if ( + command === "make-rank-pool-plan" || + command === "validate-rank-worker" || + command === "validate-rank-pool" + ) { + process.exitCode = rankPoolCommand(command, args, posixHome); + } else if (command === "bind-repo-scopes") { + process.exitCode = bindRepoScopesCommand(args, posixHome); } else { - const encoded = readFileSync(3, "ascii"); - closeSync(3); - const [homeSet, home, ...args] = decodePosixBytes( - Buffer.from(encoded.trim(), "hex"), - ) - .split("\0") - .slice(0, -1); - posixHome = homeSet ? home : undefined; - commandLine = args; + console.error( + "Usage: launch_codex_security_mcp[.cmd] --helper [options]", + ); + process.exitCode = 2; } } -const [command, ...args] = commandLine; -if (command === "resolve-security-md") { - process.exitCode = resolveSecurityMdCommand(args, posixHome); -} else if (command === "normalize-candidates") { - process.exitCode = normalizeCandidatesCommand(args, posixHome); -} else if (command === "validate-patch-risk-assessment") { - process.exitCode = validatePatchRiskAssessmentCommand(args); -} else if ( - command === "copy-deep-review-input" || - command === "select-deep-review-input" -) { - process.exitCode = deepReviewInputCommand(command, args, posixHome); -} else if ( - command === "make-rank-shards" || - command === "validate-rank-shard" || - command === "merge-rank-outputs" -) { - process.exitCode = rankShardsCommand(command, args, posixHome); -} else if ( - command === "make-rank-pool-plan" || - command === "validate-rank-worker" || - command === "validate-rank-pool" -) { - process.exitCode = rankPoolCommand(command, args, posixHome); -} else if (command === "bind-repo-scopes") { - process.exitCode = bindRepoScopesCommand(args, posixHome); -} else { - console.error( - "Usage: launch_codex_security_mcp[.cmd] --helper [options]", - ); - process.exitCode = 2; -} diff --git a/plugins/codex-security/mcp-app/package.json b/plugins/codex-security/mcp-app/package.json index b8702ce722..4a5a632353 100644 --- a/plugins/codex-security/mcp-app/package.json +++ b/plugins/codex-security/mcp-app/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "tsc --noEmit", "build:mcp": "node scripts/build_mcp_app.mjs --output .preview/mcp", - "test:mcp": "node --experimental-strip-types --test --test-concurrency=2 --test-reporter=./scripts/test_reporter.mts \"tests/test_*.ts\"", + "test:mcp": "node --experimental-strip-types --test --test-concurrency=2 --test-reporter=./scripts/test_reporter.mts \"tests/test_*.ts\" \"tests/test_*.mjs\"", "typecheck": "tsc --noEmit" }, "dependencies": { diff --git a/plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs b/plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs index aa1372c602..dce1cd421a 100644 --- a/plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs +++ b/plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs @@ -75,6 +75,7 @@ export async function buildMcpApp({ output, native = "universal" }) { loader: { ".md": "text" }, logLevel: "info", logOverride: { "empty-import-meta": "silent" }, + nodePaths: [join(root, "node_modules")], outfile: bundle, platform: "node", target: "node20", @@ -145,5 +146,6 @@ const runtimeModule = new Module(loaderPath); runtimeModule.filename = loaderPath; runtimeModule.paths = Module._nodeModulePaths(dirname(loaderPath)); runtimeModule._compile(runtimeSource, loaderPath); +export default runtimeModule.exports; `; } diff --git a/plugins/codex-security/mcp-app/server.ts b/plugins/codex-security/mcp-app/server.ts index 19f9c7a5ef..e8abf293e4 100644 --- a/plugins/codex-security/mcp-app/server.ts +++ b/plugins/codex-security/mcp-app/server.ts @@ -31,6 +31,12 @@ import { AsyncLock, startOrJoinDeepScanCoordinator, } from "./src/deep-scan/registry.js"; +import { + captureDeepScanExecutionSettings, + loadDeepScanExecutionSettings, + restoredDeepScanWorkerSettings, + type DeepScanLegacySettingsContext, +} from "./src/deep-scan/recovery-settings.js"; import { CodexSdkWorkerExecutor } from "./src/deep-scan/executor.js"; import { CODEX_SANDBOX_STATE_META_CAPABILITY, @@ -1066,7 +1072,7 @@ export function createCodexSecurityServer(): McpServer { { title: "Start or Join Codex Security Deep Scan", description: - "Run or rejoin independent Standard security scans and semantically merge their validated findings. Pass scanId and its handoffClaimToken to resume, or targetPath to start headlessly. The call blocks until the aggregate draft is ready, fails, or is canceled. On success, use the returned scanId and scanDir; manifestPath identifies the canonical parent scan-manifest.json. Call complete_codex_security_scan once.", + "Run or rejoin independent Standard security scans and semantically merge their validated findings. Pass scanId and its handoffClaimToken to resume, or targetPath to start headlessly. The call blocks until the aggregate draft is ready, fails, or is canceled. On success, use the returned scanId and scanDir; manifestPath identifies the canonical parent scan-manifest.json; native scans can call complete_codex_security_scan once. SDK-managed scans leave completion to the SDK after the scan turn ends.", inputSchema: startDeepScanSchema, annotations: writingAnnotations, _meta: modelActionMeta, @@ -1116,7 +1122,16 @@ export function createCodexSecurityServer(): McpServer { } const preparation = await deepScanStartLock .run(async () => { + // A joining observer does not need a usable current home. A new run, + // however, must save its original settings before creation can commit. + const executionSettings = await captureDeepScanExecutionSettings( + modelSettings, + parentSandbox, + process.env, + { threadId, startedAt: new Date().toISOString() }, + ).catch(() => null); const begun = await deepScanStore.begin({ + executionSettings, scanId, targetPath, scope: hasTarget ? (scope ?? ".") : undefined, @@ -1132,18 +1147,71 @@ export function createCodexSecurityServer(): McpServer { threadId, }); } - const immediate = deepScanTerminalResult(begun); - if (immediate) return { begun, immediate }; + // Recipes identify SDK executions, including runs predating usageOwner. + const sdkOwned = + begun.usageOwner?.dedicated ?? + (await runWorkbench(["get-scan", "--scan-id", begun.scanId])) + .recipe != null; + const immediate = deepScanTerminalResult(begun, sdkOwned); + const completingLocally = + begun.status === "succeeded" && + begun.finalizationInput && + deepScanCoordinators.get(begun.scanId); + const recoverableSelection = + begun.status === "succeeded" && + begun.finalizationInput && + !sdkOwned && + begun.ownerThreadId === threadId; + if (immediate && !completingLocally && !recoverableSelection) + return { begun, immediate, sdkOwned }; const started = await startOrJoinDeepScanCoordinator({ run: begun, registry: deepScanCoordinators, options: { store: deepScanStore, + prepareExecutor: async (run) => + new CodexSdkWorkerExecutor({ + ...restoredDeepScanWorkerSettings( + await loadDeepScanExecutionSettings( + run.scanDir, + run, + async () => { + const context = await runWorkbench([ + "get-scan", + "--scan-id", + run.scanId, + ]); + const recipe = context.recipe as + | Pick + | undefined; + const scan = context.scan as { + executionAttribution?: { + owner: DeepScanRunState["usageOwner"]; + }; + }; + return { + config: recipe?.config, + usageOwner: scan.executionAttribution?.owner, + }; + }, + ), + parentSandbox, + ), + artifactContext: { + pluginRoot: PLUGIN_ROOT, + scanRoot: run.scanDir, + repoRoot: run.targetPath, + scanId: run.scanId, + scope: run.scope, + pythonCommand: await resolvePythonCommand(), + }, + }), executor: new CodexSdkWorkerExecutor({ ...modelSettings, parentSandbox, artifactContext: { pluginRoot: PLUGIN_ROOT, + scanRoot: begun.scanDir, repoRoot: begun.targetPath, scanId: begun.scanId, scope: begun.scope, @@ -1154,7 +1222,7 @@ export function createCodexSecurityServer(): McpServer { log: logDeepScanEvent, handoffClaimToken, threadId, - onComplete: async (draft, signal) => { + onComplete: async (draft, signal, publication) => { const context = await createScanArtifactContext( begun.scanId, runWorkbench, @@ -1175,8 +1243,45 @@ export function createCodexSecurityServer(): McpServer { }, runWorkbench, signal, + publication, ); }, + onFinalized: async (run, signal) => { + // Active owners finish through the completion tool after this call; + // detached native scans still publish without another observer. + if (sdkOwned || !abortSignalFromExtra(extra)?.aborted) return; + try { + await runWorkbench( + [ + "complete-scan", + "--scan-id", + run.scanId, + "--thread-id", + threadId, + ...optionalArg("--claim-token", handoffClaimToken), + ], + undefined, + false, + false, + signal, + ); + } catch (error) { + if (!signal.aborted) { + await deepScanStore + .releaseCoordinator(run.scanId) + .catch((releaseError) => { + logDeepScanEvent({ + event: "coordinator_release_failed", + scanId: run.scanId, + reason: boundedErrorData(releaseError).message, + }); + }); + } + throw new Error(deepScanInvocationFailureMessage(error), { + cause: error, + }); + } + }, onStopped: async (run) => { await runWorkbench([ "preserve-scan-results", @@ -1193,7 +1298,7 @@ export function createCodexSecurityServer(): McpServer { }, }, }); - return { begun, ...started }; + return { begun, sdkOwned, ...started }; }) .catch((error: unknown) => ({ invocationFailure: toolErrorResult( @@ -1202,16 +1307,14 @@ export function createCodexSecurityServer(): McpServer { })); if ("invocationFailure" in preparation) return preparation.invocationFailure; + // SDK completion belongs to the host; native selected results use the existing lease. if (preparation.immediate) return preparation.immediate; const { begun, coordinator, joined } = preparation; if (joined) { - logDeepScanEvent({ - event: "coordinator_joined", - scanId: begun.scanId, - }); + logDeepScanEvent({ event: "coordinator_joined", scanId: begun.scanId }); } const terminal = await coordinator.wait(abortSignalFromExtra(extra)); - const result = deepScanTerminalResult(terminal); + const result = deepScanTerminalResult(terminal, preparation.sdkOwned); if (!result) { return toolErrorResult( deepScanInvocationFailureMessage( @@ -1670,21 +1773,39 @@ export function createCodexSecurityServer(): McpServer { { title: "Complete Codex Security Scan", description: - "Finalization only: validate and seal already-authored scan-manifest.json, findings.json, and coverage.json, generate report.md, index findings, and mark the scan complete. For an app-backed running scan, scan-manifest.json is an unsealed draft and must omit scan.sealedAt and scan.artifacts; this tool supplies the exact workbench timestamps, seal, artifact digests, and derived finding identities. Call only after those canonical files exist; this tool does not create missing artifacts or run skipped phases. If it fails, surface the exact error and stop the current response without retrying completion or returning a final, no-findings, structured, or benchmark response.", + "Finalization only: validate and seal already-authored scan-manifest.json, findings.json, and coverage.json, generate report.md, index findings, and mark the scan complete. For an app-backed running scan, scan-manifest.json is an unsealed draft and must omit scan.sealedAt and scan.artifacts; this tool supplies the exact workbench timestamps, seal, artifact digests, and derived finding identities. Call only after those canonical files exist; this tool does not create missing artifacts or run skipped phases. SDK-managed Deep scans leave sealing to the SDK after the scan turn ends. If it fails, surface the exact error and stop the current response without retrying completion or returning a final, no-findings, structured, or benchmark response.", inputSchema: completeScanSchema, annotations: writingAnnotations, _meta: modelActionMeta, }, async ({ scanId, handoffClaimToken }) => { try { + const context = await runWorkbench(["get-scan", "--scan-id", scanId]); + const scan = isJsonObject(context.scan) ? context.scan : undefined; + const progress = isJsonObject(scan?.progress) + ? scan.progress + : undefined; + if ( + scan?.mode === "deep" && + context.recipe != null && + progress?.status === "running" + ) { + // A tool call is still inside the owner turn; its final usage has not arrived. + return scanActionResult( + redactHandoffClaimToken(context), + "The SDK completes this Deep Scan after the scan turn finishes. Continue any remaining scan work, then end the turn without calling completion again. The SDK will account for the turn, enforce its budget, and seal and publish the scan.", + ); + } + const result = await runWorkbench([ + "complete-scan", + "--scan-id", + scanId, + ...optionalArg("--claim-token", handoffClaimToken), + ]); + const coverage = result.coverageSummary as JsonObject; return scanActionResult( - await runWorkbench([ - "complete-scan", - "--scan-id", - scanId, - ...optionalArg("--claim-token", handoffClaimToken), - ]), - "Validated and indexed the completed Codex Security scan.", + result, + `Validated and indexed the completed Codex Security scan. Canonical coverage: ${coverage.completeness}; ${coverage.surfaceCount} surfaces; ${coverage.deferredCount} deferred items; ${coverage.explicitExclusionCount} explicit exclusions.`, ); } catch (error) { throw new Error(completionFailureMessage(error)); @@ -2311,10 +2432,10 @@ function boundedErrorData(error: unknown): { message: string; name: string } { }; } -function deepScanTerminalResult(run: DeepScanRunState) { +function deepScanTerminalResult(run: DeepScanRunState, sdkOwned = false) { if (run.status === "succeeded") { if (!run.manifestPath) return undefined; - const instructions = `Deep Scan discovery completed. Independent Standard scans have already performed validation and attack-path analysis and have been consolidated into the canonical scan-manifest.json, findings.json, and coverage.json under ${run.scanDir}. The returned manifestPath is the canonical scan-manifest.json, not a legacy discovery manifest. Any instructions requiring parent candidate listing, centralized validation, attack-path analysis, or another draft apply only to the old discovery-only workflow and must be skipped. The authoritative scan ID is ${run.scanId}. Immediately call complete_codex_security_scan once using that scan ID to seal and publish the scan. Return output only after completion succeeds and generated report.md exists. If completion fails, surface that exact error and return no final, no-findings, structured, or benchmark response.`; + const instructions = `Deep Scan discovery completed. Independent Standard scans have already performed validation and attack-path analysis and have been consolidated into the canonical scan-manifest.json, findings.json, and coverage.json under ${run.scanDir}. The returned manifestPath is the canonical scan-manifest.json, not a legacy discovery manifest. Any instructions requiring parent candidate listing, centralized validation, attack-path analysis, or another draft apply only to the old discovery-only workflow and must be skipped. The authoritative scan ID is ${run.scanId}. ${sdkOwned ? "Leave the canonical artifacts unchanged and end this scan turn without calling a completion tool. The SDK will account for the completed turn, enforce its budget, and seal and publish the scan." : "Immediately call complete_codex_security_scan once using that scan ID to seal and publish the scan. Return output only after completion succeeds and generated report.md exists. If completion fails, surface that exact error and return no final, no-findings, structured, or benchmark response."}`; return { content: [{ type: "text" as const, text: instructions }], structuredContent: { @@ -2366,11 +2487,21 @@ function logDeepScanEvent(event: { async function runWorkbench( args: string[], input?: string | Buffer, + selectFinalization = false, + withExecutionSettings = false, + signal?: AbortSignal, ): Promise { let pythonCommand: string | undefined; try { pythonCommand = await resolvePythonCommand(); - return await executeWorkbenchWithStateSelection(pythonCommand, args, input); + return await executeWorkbenchWithStateSelection( + pythonCommand, + args, + input, + selectFinalization, + withExecutionSettings, + signal, + ); } catch (error) { const launchError = pythonCommand ? missingPythonHelperMessage(error, pythonCommand) @@ -2389,12 +2520,31 @@ async function executeWorkbenchWithStateSelection( pythonCommand: string, args: string[], input?: string | Buffer, + selectFinalization = false, + withExecutionSettings = false, + signal?: AbortSignal, ): Promise { if (WORKBENCH_COMMANDS_WITHOUT_DATABASE.has(args[0] ?? "")) { - return await executeWorkbench(pythonCommand, args, undefined, input); + return await executeWorkbench( + pythonCommand, + args, + undefined, + input, + selectFinalization, + withExecutionSettings, + signal, + ); } if (CONFIGURED_WORKBENCH_STATE_DIR) { - return await executeWorkbench(pythonCommand, args, undefined, input); + return await executeWorkbench( + pythonCommand, + args, + undefined, + input, + selectFinalization, + withExecutionSettings, + signal, + ); } if (fallbackWorkbenchStateDir) { return await executeWorkbench( @@ -2402,10 +2552,21 @@ async function executeWorkbenchWithStateSelection( args, await fallbackWorkbenchStateDir, input, + selectFinalization, + withExecutionSettings, + signal, ); } if (persistentWorkbenchStateSucceeded) { - return await executeWorkbench(pythonCommand, args, undefined, input); + return await executeWorkbench( + pythonCommand, + args, + undefined, + input, + selectFinalization, + withExecutionSettings, + signal, + ); } return await workbenchStateSelectionLock.run(async () => { if (fallbackWorkbenchStateDir) { @@ -2414,10 +2575,21 @@ async function executeWorkbenchWithStateSelection( args, await fallbackWorkbenchStateDir, input, + selectFinalization, + withExecutionSettings, + signal, ); } if (persistentWorkbenchStateSucceeded) { - return await executeWorkbench(pythonCommand, args, undefined, input); + return await executeWorkbench( + pythonCommand, + args, + undefined, + input, + selectFinalization, + withExecutionSettings, + signal, + ); } try { const result = await executeWorkbench( @@ -2425,6 +2597,9 @@ async function executeWorkbenchWithStateSelection( args, undefined, input, + selectFinalization, + withExecutionSettings, + signal, ); persistentWorkbenchStateSucceeded = true; return result; @@ -2437,6 +2612,9 @@ async function executeWorkbenchWithStateSelection( args, fallbackStateDir, input, + selectFinalization, + withExecutionSettings, + signal, ); } }); @@ -2447,6 +2625,9 @@ async function executeWorkbench( args: string[], stateDir?: string, input?: string | Buffer, + selectFinalization = false, + withExecutionSettings = false, + signal?: AbortSignal, ): Promise { const userContextIndex = args.indexOf("--user-context"); const userContext = @@ -2456,45 +2637,55 @@ async function executeWorkbench( workbenchArgs.splice(userContextIndex, 2, "--user-context-stdin"); } const workbenchInput = input ?? userContext; - const execution = execFileAsync( - pythonCommand, - [workbenchScriptPath(), ...workbenchArgs], - { - cwd: PLUGIN_ROOT, - env: stateDir - ? { ...process.env, CODEX_SECURITY_STATE_DIR: stateDir } - : process.env, - encoding: "utf8" as const, - // Artifact bytes are base64-encoded here; retain the existing file-size behavior. - maxBuffer: args[0] === "read-artifact" ? Infinity : 4 * 1024 * 1024, - timeout: [ - "begin-deep-scan", - "claim-deep-scan-dedup", - "commit-deep-scan-dedup", - "complete-scan", - "export-findings", - "finish-deep-scan", - "get-scan", - "get-deep-scan", - "get-workspace", - "inspect-setup", - "list-findings", - "preserve-scan-results", - "recover-scan-results", - "request-finding-remediation", - "request-finding-remediation-action", - "save-workspace", - "set-finding-triage", - "set-finding-remediation", - "start-headless-standard-scan", - "start-prompt-only-scan", - "start-scan", - "upsert-deep-scan-worker", - ].includes(args[0] ?? "") - ? 300_000 - : 30_000, - }, - ); + const internalInvocation = selectFinalization + ? "select_finalization=True" + : withExecutionSettings + ? "with_execution_settings=True" + : undefined; + const pythonArgs = internalInvocation + ? [ + "-c", + `import runpy, sys; script = sys.argv.pop(1); runpy.run_path(script)['main'](${internalInvocation})`, + workbenchScriptPath(), + ...workbenchArgs, + ] + : [workbenchScriptPath(), ...workbenchArgs]; + const execution = execFileAsync(pythonCommand, pythonArgs, { + signal, + cwd: PLUGIN_ROOT, + env: stateDir + ? { ...process.env, CODEX_SECURITY_STATE_DIR: stateDir } + : process.env, + encoding: "utf8" as const, + // Artifact bytes are base64-encoded here; retain the existing file-size behavior. + maxBuffer: args[0] === "read-artifact" ? Infinity : 4 * 1024 * 1024, + timeout: [ + "begin-deep-scan", + "claim-deep-scan-dedup", + "commit-deep-scan-dedup", + "complete-scan", + "export-findings", + "finish-deep-scan", + "get-scan", + "get-deep-scan", + "get-workspace", + "inspect-setup", + "list-findings", + "preserve-scan-results", + "recover-scan-results", + "request-finding-remediation", + "request-finding-remediation-action", + "save-workspace", + "set-finding-triage", + "set-finding-remediation", + "start-headless-standard-scan", + "start-prompt-only-scan", + "start-scan", + "upsert-deep-scan-worker", + ].includes(args[0] ?? "") + ? 300_000 + : 30_000, + }); if (workbenchInput !== undefined) { execution.child.stdin!.on("error", () => { // The workbench may exit before consuming stdin; surface its process error. diff --git a/plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts b/plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts index 368d89c6ff..96055ed356 100644 --- a/plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts +++ b/plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts @@ -1,4 +1,4 @@ -import { join } from "node:path"; +import { dirname, join, relative, sep } from "node:path"; import type { ZodType } from "zod/v4"; import commonSchema from "../../schemas/definitions/artifact-common.schema.json"; import reducerSchema from "../../schemas/tools/deep-reducer.schema.json"; @@ -25,7 +25,9 @@ import { type DeepScanArtifacts, } from "./deep-scan/artifacts.js"; import { + deepReductionForPersistence, parseDeepReduction, + projectDiscoveryCoverage, parseStoredScanDraft, reconcileDeepReduction, type DeepReductionInput, @@ -60,6 +62,22 @@ interface BoundReducer { /** Read the findings and scan context assigned to this reducer. */ export async function getCodexSecurityDeepReducerInputs( context: ArtifactContext, +): Promise { + const inputs = await readDeepReductionSources(context); + const { sourceCoverage: _coverage, ...previous } = inputs.previous ?? {}; + return { + discoveries: inputs.discoveries.map(({ workerId, result }) => ({ + workerId, + result, + })), + previous: + inputs.previous === null ? null : (previous as DeepReductionInput), + }; +} + +/** Capture host coverage alongside the reducer's immutable finding inputs. */ +export async function readDeepReductionSources( + context: ArtifactContext, ): Promise { return withLogicalReducerErrors(context, async () => { const bound = bindDeepReducer(context); @@ -88,8 +106,22 @@ export async function getCodexSecurityDeepReducerInputs( sourceFindingIds: [`${worker.id}:${index}`], }, })); - const { coverage: _coverage, ...reduction } = result; - return { workerId: worker.id, result: reduction }; + const { coverage, ...reduction } = result; + return { + workerId: worker.id, + ...(worker.attempt === undefined ? {} : { attempt: worker.attempt }), + coverage: projectDiscoveryCoverage( + coverage, + worker, + relative( + bound.artifacts.scanDir, + worker.artifactDir ?? dirname(worker.resultPath), + ) + .split(sep) + .join("/"), + ), + result: reduction, + }; }), ); const previous = await readPreviousReduction(bound); @@ -125,7 +157,7 @@ export async function recordCodexSecurityDeepReduction( throw new Error( "Deep reduction is only a checkpoint, not a complete result.", ); - const inputs = await getCodexSecurityDeepReducerInputs(context); + const inputs = await readDeepReductionSources(context); const expectedScanId = bound.scanId ?? inputs.previous?.scanId ?? @@ -141,8 +173,12 @@ export async function recordCodexSecurityDeepReduction( inputs.previous, ); - await saveScanDraftCheckpoint(context, reduction); - await writeJsonAtomic(bound.resultPath, reduction); + const persisted = deepReductionForPersistence( + reduction, + bound.state.persistSourceCoverage, + ); + await saveScanDraftCheckpoint(context, persisted); + await writeJsonAtomic(bound.resultPath, persisted); const documentWarning = await saveThreatModelDocument( context, reduction.threatModel, diff --git a/plugins/codex-security/mcp-app/src/artifact-io.ts b/plugins/codex-security/mcp-app/src/artifact-io.ts index d8e19a61db..923a1f352f 100644 --- a/plugins/codex-security/mcp-app/src/artifact-io.ts +++ b/plugins/codex-security/mcp-app/src/artifact-io.ts @@ -7,12 +7,16 @@ import { dirname, isAbsolute, join, sep } from "node:path"; export interface DeepReducerWorkerContext { id: string; resultPath: string; + /** Original output owner for relative evidence, including accepted checkpoints. */ + artifactDir?: string; + attempt?: number; } export interface DeepReducerContext { scanRoot: string; claimedWorkers: DeepReducerWorkerContext[]; previousReducerResultPath?: string; + persistSourceCoverage?: boolean; } /** diff --git a/plugins/codex-security/mcp-app/src/artifact-scan-draft.ts b/plugins/codex-security/mcp-app/src/artifact-scan-draft.ts index d0e661a410..64c60b9535 100644 --- a/plugins/codex-security/mcp-app/src/artifact-scan-draft.ts +++ b/plugins/codex-security/mcp-app/src/artifact-scan-draft.ts @@ -7,6 +7,8 @@ import { isDeepStrictEqual } from "node:util"; import type * as z from "zod/v4"; import commonSchema from "../../schemas/definitions/artifact-common.schema.json"; import scanDraftDocument from "../../schemas/tools/scan-draft.schema.json"; +import scanManifestDocument from "../../schemas/scan-manifest.schema.json"; +import coverageDocument from "../../schemas/coverage.schema.json"; import type { ArtifactContext } from "./artifact-context.js"; import type { RunArtifactWorkbench } from "./artifact-context.js"; import { @@ -20,18 +22,10 @@ import { loadArtifactZodSchema, type SchemaDocument, } from "./artifact-schema-loader.js"; +import type { ScanDraftInput } from "../../../../sdk/typescript/src/accepted-audit.js"; +export type { ScanDraftInput } from "../../../../sdk/typescript/src/accepted-audit.js"; import { saveThreatModelDocument } from "./threat-model-document.js"; -export interface ScanDraftInput { - scanId: string; - complete?: boolean; - handoffClaimToken?: string; - scope?: JsonObject; - threatModel?: JsonObject; - findings: JsonObject[]; - coverage: JsonObject; -} - export interface CompletedScanInput { scanId: string; handoffClaimToken?: string; @@ -57,6 +51,11 @@ interface PreparedScanDraft { coverage: JsonObject; } +/** Host-selected Deep aggregate, separate from model-authored draft fields. */ +export interface DeepScanPublication { + coordinatorGeneration?: number; + resultPath: string | null; +} interface SavedScanDraft { input: ScanDraftInput; modifiedMs: number; @@ -78,6 +77,29 @@ export const scanDraftInputSchema = loadArtifactZodSchema( "scanDraftInput", ) as z.ZodType; +// Sealed documents retain the existing public ID contract; live drafts require UUIDs. +const canonicalScanDraftInputSchema = loadArtifactZodSchema( + [ + commonSchema, + { + ...scanDraftDocument, + $defs: { + ...scanDraftDocument.$defs, + scanId: scanManifestDocument.properties.scan.properties.id, + surface: { + ...scanDraftDocument.$defs.surface, + properties: { + ...scanDraftDocument.$defs.surface.properties, + id: coverageDocument.properties.surfaces.items.properties.id, + }, + }, + }, + }, + ] as SchemaDocument[], + scanDraftDocument.$id, + "scanDraftInput", +) as z.ZodType; + export const completedScanInputSchema = loadArtifactZodSchema( schemaDocuments, scanDraftDocument.$id, @@ -207,6 +229,7 @@ export async function recordCodexSecurityScanDraftViaWorkbench( input: ScanDraftInput, runWorkbench: RunArtifactWorkbench, signal?: AbortSignal, + publication?: DeepScanPublication, ): Promise { return recordCodexSecurityScanDraft( context, @@ -226,7 +249,12 @@ export async function recordCodexSecurityScanDraftViaWorkbench( const { handoffClaimToken: _claim, ...snapshot } = checkpoint; await Promise.all([ replaceArtifactJson(checkpointPath, snapshot), - replaceArtifactJson(draftPath, draft), + replaceArtifactJson(draftPath, { + ...draft, + ...(publication === undefined + ? {} + : { deepScanPublication: publication }), + }), ]); const arguments_ = [ "write-scan-draft", @@ -1254,7 +1282,6 @@ async function readPreviousScanDraft( contents[2]!, "previous scan draft coverage", ); - const scan = requireObject(manifest.scan, "previous scan draft.scan"); return { digest, // File-authored coverage may leave its manifest unchanged; tool writes have a head. @@ -1262,12 +1289,10 @@ async function readPreviousScanDraft( (await readCheckpointHead(context, "current")) === undefined ? saved[2]!.modifiedMs : Math.min(...saved.map((record) => record!.modifiedMs)), - input: parsePersistedCheckpoint({ + input: parseCanonicalScanDraft({ scanId: context.scanId, - ...(scan.complete === false ? { complete: false } : {}), - ...(isObject(scan.scope) ? { scope: scan.scope } : {}), - ...(isObject(scan.threatModel) ? { threatModel: scan.threatModel } : {}), - findings: findings.findings, + manifest, + findings, coverage, }), }; @@ -1819,8 +1844,42 @@ export async function getCodexSecurityCompletedScan( return { scanId: parsed.scanId, manifest, findings, coverage }; } +/** Project canonical documents through the same semantic parser as worker drafts. */ +export function parseCanonicalScanDraft(input: { + scanId?: string; + manifest: JsonObject; + findings: JsonObject; + coverage: JsonObject; +}): ScanDraftInput { + const scan = requireObject(input.manifest.scan, "scan draft manifest.scan"); + for (const scanId of [ + scan.id, + input.findings.scanId, + input.coverage.scanId, + ]) { + if (scanId !== undefined && scanId !== input.scanId) { + throw new Error( + "scan draft: canonical documents belong to a different scan.", + ); + } + } + return parsePersistedCheckpoint( + { + scanId: input.scanId, + ...(scan.complete === undefined ? {} : { complete: scan.complete }), + ...(scan.scope === undefined ? {} : { scope: scan.scope }), + ...(scan.threatModel === undefined + ? {} + : { threatModel: scan.threatModel }), + findings: input.findings.findings, + coverage: input.coverage, + }, + canonicalScanDraftInputSchema, + ); +} + export function parseScanDraft(input: ScanDraftInput): ScanDraftInput { - const parsed = parseScanDraftDocument(input); + const parsed = parseSemanticScanDraft(input, scanDraftInputSchema); const deferredIds = new Set(); const ambiguousIds = ambiguousGenericDeferredIds([parsed]); for (const row of parsed.coverage.deferred as JsonObject[]) { @@ -1842,8 +1901,11 @@ export function parseScanDraft(input: ScanDraftInput): ScanDraftInput { return parsed; } -function parseScanDraftDocument(input: unknown): ScanDraftInput { - const parsed = scanDraftInputSchema.parse(input); +function parseSemanticScanDraft( + input: unknown, + schema: z.ZodType, +): ScanDraftInput { + const parsed = schema.parse(input); validateFindingSemantics(parsed.findings); validateCoverageSemantics(parsed.coverage); return parsed; @@ -1852,20 +1914,26 @@ function parseScanDraftDocument(input: unknown): ScanDraftInput { /** Validate saved drafts before reconciling interrupted writes and older findings. */ export function parsePersistedScanDraft( input: Record, +): ScanDraftInput { + return parsePersistedDraft(input, scanDraftInputSchema); +} + +function parsePersistedDraft( + input: Record, + schema: z.ZodType, ): ScanDraftInput { const compatible = structuredClone(input); - if (!Array.isArray(compatible.findings)) { - return parseScanDraftDocument(compatible); - } - for (const finding of compatible.findings) { - if (!isObject(finding)) continue; - normalizePersistedFindingDetails(finding); + if (Array.isArray(compatible.findings)) { + for (const finding of compatible.findings) { + if (isObject(finding)) normalizePersistedFindingDetails(finding); + } } - return parseScanDraftDocument(compatible); + return parseSemanticScanDraft(compatible, schema); } function parsePersistedCheckpoint( input: Record, + schema = scanDraftInputSchema, ): ScanDraftInput { const compatible = structuredClone(input); if (isObject(compatible.scope)) { @@ -1894,7 +1962,7 @@ function parsePersistedCheckpoint( delete finding.fingerprints; } } - return parsePersistedScanDraft(compatible); + return parsePersistedDraft(compatible, schema); } function normalizePersistedFindingDetails(finding: JsonObject): void { diff --git a/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts b/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts index 7888b34cde..79b470fac4 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts @@ -1,3 +1,4 @@ +import { join } from "node:path"; import { isDeepStrictEqual } from "node:util"; import { parsePersistedScanDraft, @@ -14,10 +15,18 @@ import { } from "./artifacts.js"; import type { DeepScanArtifacts } from "./artifacts.js"; -export type DeepReductionInput = Omit; +export type DeepReductionInput = Omit & { + /** Host projection of accepted source coverage; never supplied by the reducer. */ + sourceCoverage?: ScanDraftInput["coverage"]; +}; export interface DeepReductionSources { - discoveries: { workerId: string; result: DeepReductionInput }[]; + discoveries: { + workerId: string; + attempt?: number; + coverage?: ScanDraftInput["coverage"]; + result: DeepReductionInput; + }[]; previous: DeepReductionInput | null; } @@ -26,6 +35,23 @@ export interface ReducerArtifactValidation { result: DeepReductionInput; } +export function deepReductionToScanDraft( + result: DeepReductionInput, +): ScanDraftInput { + const { sourceCoverage, ...draft } = structuredClone(result); + return { ...draft, coverage: sourceCoverage ?? unknownSourceCoverage() }; +} + +/** Older workflow readers reject the host field; retain their persisted shape. */ +export function deepReductionForPersistence( + result: DeepReductionInput, + persistSourceCoverage = false, +): DeepReductionInput { + if (persistSourceCoverage) return result; + const { sourceCoverage: _coverage, ...legacy } = result; + return legacy; +} + /** * Check reducer findings with the Standard scan validator. * It requires coverage, so add an empty value and remove it after validation. @@ -34,8 +60,9 @@ export function parseDeepReduction( input: Record, persisted = false, ): DeepReductionInput { + const { sourceCoverage, ...submitted } = input; const standard = { - ...input, + ...submitted, coverage: { completeness: "complete", surfaces: [], @@ -46,6 +73,15 @@ export function parseDeepReduction( const { coverage: _coverage, ...parsed } = persisted ? parsePersistedScanDraft(standard) : parseScanDraft(standard as unknown as ScanDraftInput); + if (persisted && sourceCoverage !== undefined) { + return { + ...parsed, + sourceCoverage: parsePersistedScanDraft({ + ...standard, + coverage: sourceCoverage, + }).coverage, + }; + } return parsed; } @@ -54,6 +90,23 @@ export async function validateDiscoveryArtifacts( artifacts: DeepScanArtifacts, resultPath: string, expectedScanId: string, +): Promise { + const result = await readDiscoveryAuditDraft( + artifacts, + resultPath, + expectedScanId, + ); + if (result.complete === false) + throw new Error( + "Standard scan worker wrote only a checkpoint; its audit is not complete.", + ); + return result; +} + +export async function readDiscoveryAuditDraft( + artifacts: DeepScanArtifacts, + resultPath: string, + expectedScanId: string, ): Promise { await requireRegularFile(resultPath, artifacts.workersRoot); const result = parseStoredScanDraft( @@ -62,13 +115,32 @@ export async function validateDiscoveryArtifacts( expectedScanId, parsePersistedScanDraft, ); - if (result.complete === false) - throw new Error( - "Standard scan worker wrote only a checkpoint; its audit is not complete.", - ); + if (result.complete !== false) await validateWriteupFiles(artifacts, result); return result; } +async function validateWriteupFiles( + artifacts: DeepScanArtifacts, + result: Pick, +): Promise { + for (const [index, finding] of result.findings.entries()) { + const reportPath = (finding.writeup as { reportPath: string } | undefined) + ?.reportPath; + if (!reportPath) continue; + try { + await requireRegularFile( + join(artifacts.scanDir, reportPath), + artifacts.scanDir, + ); + } catch (cause) { + throw new Error( + `findings[${index}].writeup.reportPath: expected a file inside the scan directory: ${reportPath}`, + { cause }, + ); + } + } +} + /** Validate the complete aggregate and derive convergence from stable finding identities. */ export async function validateReducerArtifacts( input: { @@ -78,6 +150,7 @@ export async function validateReducerArtifacts( reducerId: string; previousReducerResultPath?: string; sources?: DeepReductionSources; + persistSourceCoverage?: boolean; }, expectedScanId?: string, ): Promise { @@ -118,13 +191,20 @@ export async function validateReducerArtifacts( input.sources.discoveries, input.sources.previous, ); + } else { + validateRetainedFindings(result, [], previous); + } + await validateWriteupFiles(artifacts, result); + if (input.sources) { + const persisted = deepReductionForPersistence( + result, + input.persistSourceCoverage, + ); await saveScanDraftCheckpoint( { root: artifactDir, repoRoot: artifacts.scanDir, layout: "reducer" }, - result, + persisted, ); - await writeJsonAtomic(resultPath, result); - } else { - validateRetainedFindings(result, [], previous); + await writeJsonAtomic(resultPath, persisted); } const previousFindingIds = new Set( (previous?.findings ?? []).map(scanFindingIdentity), @@ -144,6 +224,11 @@ export function reconcileDeepReduction( previous: DeepReductionInput | null, ): DeepReductionInput { const result = structuredClone(input); + result.sourceCoverage = aggregateSourceCoverage(discoveries, previous); + if (result.complete === false) + throw new Error( + "Deep reduction is only a checkpoint, not a complete result.", + ); for (const source of [ ...discoveries.map((discovery) => discovery.result), ...(previous ? [previous] : []), @@ -207,6 +292,138 @@ export function reconcileDeepReduction( return result; } +/** Keep independent reviews separate: matching labels do not resolve another pass's proof gap. */ +export function aggregateSourceCoverage( + discoveries: DeepReductionSources["discoveries"], + previous: DeepReductionInput | null, +): ScanDraftInput["coverage"] { + const sources = [ + ...(previous ? [previous.sourceCoverage ?? unknownSourceCoverage()] : []), + ...discoveries.map((source) => source.coverage ?? unknownSourceCoverage()), + ]; + const result: ScanDraftInput["coverage"] = { + completeness: "complete", + surfaces: [], + explicitExclusions: [], + deferred: [], + reviews: [], + }; + for (const field of [ + "surfaces", + "explicitExclusions", + "deferred", + "openQuestions", + "reviews", + ]) { + const entries = sources.flatMap( + (source) => (source[field] as unknown[] | undefined) ?? [], + ); + if (entries.length || field !== "openQuestions") + result[field] = structuredClone(entries); + } + if ( + sources.some((source) => source.completeness === "partial") || + (result.deferred as unknown[]).length > 0 || + (result.surfaces as Record[]).some( + (surface) => surface.disposition === "needs_follow_up", + ) + ) { + result.completeness = "partial"; + } else if (sources.some((source) => source.completeness === "unknown")) { + result.completeness = "unknown"; + } + return result; +} + +function unknownSourceCoverage(): ScanDraftInput["coverage"] { + return { + completeness: "unknown", + surfaces: [], + explicitExclusions: [], + deferred: [], + }; +} + +/** Qualify worker-local IDs and receipt paths before combining accepted coverage. */ +export function projectDiscoveryCoverage( + coverage: ScanDraftInput["coverage"], + worker: { id: string; attempt?: number }, + artifactPrefix: string, +): ScanDraftInput["coverage"] { + const provenance = { + workerId: worker.id, + ...(worker.attempt === undefined ? {} : { attempt: worker.attempt }), + }; + const prefix = `${worker.id}-attempt-${worker.attempt ?? "unknown"}`; + const surfaces = coverage.surfaces as Record[]; + const surfaceIds = new Map( + surfaces.map((surface, index) => [ + surface.id, + `${prefix}-surface-${index + 1}`, + ]), + ); + const project = (item: Record) => { + const projected = structuredClone(item); + const sourceProvenance = + typeof projected.provenance === "object" && + projected.provenance !== null && + !Array.isArray(projected.provenance) + ? (projected.provenance as Record) + : {}; + // Preserve source descriptions while deriving identity only from the accepted owner. + for (const key of ["workerId", "attempt", "sourceId", "candidateId"]) + delete sourceProvenance[key]; + projected.provenance = { + ...sourceProvenance, + ...provenance, + ...(item.id === undefined ? {} : { sourceId: item.id }), + ...(item.candidateId === undefined + ? {} + : { candidateId: item.candidateId }), + }; + return projected; + }; + return { + completeness: coverage.completeness, + reviews: [{ ...provenance, completeness: coverage.completeness }], + surfaces: surfaces.map((surface, index) => ({ + ...project(surface), + id: `${prefix}-surface-${index + 1}`, + receiptRefs: ((surface.receiptRefs as string[] | undefined) ?? []).map( + (ref) => `${artifactPrefix}/${ref}`, + ), + })), + explicitExclusions: ( + coverage.explicitExclusions as Record[] + ).map(project), + deferred: (coverage.deferred as Record[]).map( + (item, index) => ({ + ...project(item), + id: `${prefix}-deferred-${index + 1}`, + ...(item.candidateId === undefined + ? {} + : { candidateId: `${prefix}-candidate-${index + 1}` }), + ...(item.surfaceIds === undefined + ? {} + : { + surfaceIds: (item.surfaceIds as string[]).map( + (id) => surfaceIds.get(id) ?? id, + ), + }), + }), + ), + ...(coverage.openQuestions === undefined + ? {} + : { + openQuestions: ( + coverage.openQuestions as (string | Record)[] + ).map((question) => + project(typeof question === "string" ? { question } : question), + ), + }), + }; +} + function findingSourceIds(finding: Record): string[] { const provenance = finding.provenance as Record; const ids = provenance.sourceFindingIds; diff --git a/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts b/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts index 11d5bb6d46..b2b3edff11 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts @@ -1,19 +1,23 @@ /// +import { publishSelectedDeepScan } from "./finalization.js"; import { randomUUID } from "node:crypto"; -import { promises as fs } from "node:fs"; -import { join } from "node:path"; +import { basename, dirname, join } from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { createDeepScanArtifacts, ensureDeepScanDirectories, } from "./artifacts.js"; import { + aggregateSourceCoverage, + deepReductionToScanDraft, validateDiscoveryArtifacts, validateReducerArtifacts, type DeepReductionInput, } from "./artifact-validation.js"; +import { readDeepReductionSources } from "../artifact-deep-reducer.js"; import { scanDraftInputSchema, + type DeepScanPublication, type ScanDraftInput, } from "../artifact-scan-draft.js"; import type { DeepScanArtifacts } from "./artifacts.js"; @@ -56,6 +60,7 @@ interface SchedulerResult { reason: DeepScanTerminalReason; omittedWorkerIds: string[]; result?: DeepReductionInput; + resultPath?: string; } export interface CoordinatorOptions { @@ -72,7 +77,13 @@ export interface CoordinatorOptions { threadId?: string; heartbeatIntervalMs?: number; observeReplacement?: (run: DeepScanRunState) => Promise; - onComplete?: (draft: ScanDraftInput, signal: AbortSignal) => Promise; + onComplete?: ( + draft: ScanDraftInput, + signal: AbortSignal, + publication: DeepScanPublication, + ) => Promise; + /** Complete the enclosing scan after selected publication, before local waiters settle. */ + onFinalized?: (run: DeepScanRunState, signal: AbortSignal) => Promise; onStopped?: (run: DeepScanRunState) => Promise; } @@ -156,7 +167,7 @@ export class DeepScanCoordinator { if (this.started) return; this.started = true; this.log({ event: "coordinator_started", scanId: this.state.scanId }); - this.scheduleDiscoveryDeadline(); + if (!this.state.finalizationInput) this.scheduleDiscoveryDeadline(); this.scheduleHeartbeat(); void this.run().catch((error: unknown) => { this.log({ @@ -280,21 +291,30 @@ export class DeepScanCoordinator { await ensureDeepScanDirectories(this.artifacts); if (this.canceled || this.externallyFailed) return; + if (this.state.finalizationInput) { + await this.completeSelectedFinalization(); + return; + } this.setupComplete = true; const schedulerResult = await this.runScheduler(); if (this.canceled || this.externallyFailed) return; + if (this.state.workflowVersion === "deep-security-scan/v2") { + if (!this.options.store.selectFinalization) + throw new Error( + "The Deep Scan store cannot select finalization input.", + ); + this.state = await this.options.store.selectFinalization({ + scanId: this.state.scanId, + reason: schedulerResult.reason, + manifestPath: join(this.state.scanDir, "scan-manifest.json"), + resultPath: schedulerResult.resultPath, + omittedWorkerIds: schedulerResult.omittedWorkerIds, + }); + await this.completeSelectedFinalization(); + return; + } const draft = schedulerResult.result - ? { - ...structuredClone(schedulerResult.result), - // Readers require coverage.json. The coordinator has accepted this - // result, so mark it complete and leave review notes empty. - coverage: { - completeness: "complete", - surfaces: [], - explicitExclusions: [], - deferred: [], - }, - } + ? deepReductionToScanDraft(schedulerResult.result) : scanDraftInputSchema.parse({ scanId: this.state.scanId, findings: [], @@ -318,6 +338,10 @@ export class DeepScanCoordinator { await this.options.onComplete?.( draft, this.publicationAbortController.signal, + { + coordinatorGeneration: this.state.coordinatorGeneration, + resultPath: schedulerResult.resultPath ?? null, + }, ); if (this.canceled || this.externallyFailed) return; this.state = await this.finishWithReplay(schedulerResult); @@ -340,6 +364,16 @@ export class DeepScanCoordinator { ) { return; } + if (this.state.finalizationInput) { + // Publication can be retried from the committed input without model work. + this.log({ + event: "coordinator_publication_pending", + scanId: this.state.scanId, + reason: errorKind(error), + }); + if (this.stopLocally()) this.rejectTerminal(error); + return; + } const message = errorMessage(error); const persistedMessage = boundedDeepScanErrorMessage(error); if (!this.setupComplete) { @@ -453,6 +487,24 @@ export class DeepScanCoordinator { } } + private async completeSelectedFinalization(): Promise { + this.state = await publishSelectedDeepScan({ + run: this.state, + artifacts: this.artifacts, + signal: this.publicationAbortController.signal, + publish: async (...args) => { + await this.options.onComplete?.(...args); + }, + finish: (input) => this.options.store.finish(input), + }); + if (this.canceled || this.externallyFailed) return; + await this.options.onFinalized?.( + cloneState(this.state), + this.publicationAbortController.signal, + ); + if (this.canceled || this.externallyFailed) return; + if (this.stopLocally()) this.resolveTerminal(cloneState(this.state)); + } private stopLocally(): boolean { if (this.terminal) return false; this.terminal = true; @@ -568,7 +620,25 @@ export class DeepScanCoordinator { current.coordinatorGeneration !== undefined && this.state.coordinatorGeneration !== undefined && current.coordinatorGeneration > this.state.coordinatorGeneration); - if (current.status === "running" && !replacementConfirmed) return false; + // finish-deep-scan commits before the enclosing scan's finalizer. Its own + // succeeded generation is still ours until that finalizer settles. + const completingSelectedParent = + current.status === "succeeded" && + this.state.finalizationInput !== undefined && + current.coordinatorGeneration === this.state.coordinatorGeneration; + if ( + !replacementConfirmed && + (current.status === "running" || completingSelectedParent) + ) { + // A selection response can be lost after its transaction commits. + if (current.finalizationInput) + this.state = { + ...this.state, + finalizationInput: current.finalizationInput, + terminalReason: current.terminalReason, + }; + return false; + } this.externallyFailed = true; this.abortController.abort("deep_scan_coordinator_lease_lost"); @@ -605,7 +675,7 @@ export class DeepScanCoordinator { const config = this.state.config; const active = new Map>(); const recovered = await this.recoverAcceptedDiscoveries(); - let acceptedCount = recovered.length; + const accepted: AcceptedDiscovery[] = [...recovered]; const mergedIds = new Set( (this.state.persistedWorkers ?? []) .filter( @@ -615,7 +685,6 @@ export class DeepScanCoordinator { .map((worker) => worker.id), ); const omittedWorkerIds: string[] = []; - await this.readPersistedExecutionPrompts(); const recoveredReducers = await this.recoverCompletedReducers(recovered); let latestResult = recoveredReducers.result; let buffer: AcceptedDiscovery[] = recovered.filter( @@ -657,9 +726,11 @@ export class DeepScanCoordinator { ); } if (reducerFailures >= errorLimit) { - const failure = (this.state.persistedWorkers ?? []).findLast( - (worker) => worker.kind === "dedup" && worker.status === "failed", - ); + const failure = [...(this.state.persistedWorkers ?? [])] + .reverse() + .find( + (worker) => worker.kind === "dedup" && worker.status === "failed", + ); throw reducerErrorLimitError( reducerFailures, errorLimit, @@ -700,23 +771,42 @@ export class DeepScanCoordinator { wakeScheduler = resolvePromise; }); } - return settlements.shift()!; + const settlement = settlements.shift(); + if (!settlement) + throw new Error("Deep Scan scheduler woke without a settled task."); + return settlement; }; - const reconcileRemainingDiscoveries = async (): Promise< - unknown | undefined - > => { - const results = await Promise.allSettled([...active.values()]); + const reconcileRemainingDiscoveries = async ( + succeededState: "buffered" | "omitted", + ): Promise => { + const entries = [...active.entries()]; + const results = await Promise.allSettled( + entries.map(([, promise]) => promise), + ); let firstFailure: unknown | undefined; - for (const result of results) { + for (const [index, result] of results.entries()) { + const workerId = entries[index]?.[0]; + if (workerId) active.delete(workerId); if (result.status === "rejected") { firstFailure ??= result.reason; continue; } const outcome = result.value; if (outcome.status === "failed") { - if (!outcome.replaceableFailureKind) firstFailure ??= outcome.error; + if (!outcome.replaceableFailureKind) { + firstFailure ??= outcome.error; + } } else if (outcome.status === "succeeded") { - omittedWorkerIds.push(outcome.worker.id); + if (!accepted.some((worker) => worker.id === outcome.worker.id)) { + accepted.push(outcome.worker); + } + if (succeededState === "omitted") { + omittedWorkerIds.push(outcome.worker.id); + } else if ( + !buffer.some((worker) => worker.id === outcome.worker.id) + ) { + buffer.push(outcome.worker); + } } } return firstFailure; @@ -725,11 +815,22 @@ export class DeepScanCoordinator { unknown | undefined > => { if (!reducer) return undefined; - const [result] = await Promise.allSettled([reducer]); - if (result.status === "rejected") return result.reason; + const pendingReducer = reducer; + reducer = undefined; + const [result] = await Promise.allSettled([pendingReducer]); + if (!result || result.status === "rejected") { + return result?.status === "rejected" ? result.reason : undefined; + } const outcome = result.value; - if ("status" in outcome) return outcome.error; + if ("status" in outcome) { + buffer = [...outcome.consumed, ...buffer].sort( + compareCompletionSequence, + ); + return outcome.error; + } this.state = outcome.run; + previousReducerResultPath = outcome.resultPath; + latestResult = outcome.result; return undefined; }; @@ -738,7 +839,7 @@ export class DeepScanCoordinator { phase: "discovery", handoffClaimToken: this.options.handoffClaimToken, }); - this.logProgress(acceptedCount); + this.logProgress(accepted.length); while (!stopReason) { if (this.abortController.signal.aborted) { @@ -786,6 +887,7 @@ export class DeepScanCoordinator { label: `dedup-${String(reducerSequence).padStart(4, "0")}`, consumed, previousReducerResultPath, + previousSourceCoverage: latestResult?.sourceCoverage, }), ); observe(reducer); @@ -811,7 +913,7 @@ export class DeepScanCoordinator { if (settlement.status === "rejected") { this.abortController.abort(errorMessage(settlement.error)); await reconcileReducerSettlement(); - await this.settleSchedulerWork(); + await reconcileRemainingDiscoveries("buffered"); throw settlement.error; } const outcome = settlement.outcome; @@ -843,12 +945,12 @@ export class DeepScanCoordinator { ); this.abortController.abort(thresholdError.message); await reconcileReducerSettlement(); - await this.settleSchedulerWork(); + await reconcileRemainingDiscoveries("buffered"); throw thresholdError; } this.abortController.abort(outcome.error.message); await reconcileReducerSettlement(); - await this.settleSchedulerWork(); + await reconcileRemainingDiscoveries("buffered"); throw outcome.error; } if (outcome.status === "canceled") { @@ -862,16 +964,18 @@ export class DeepScanCoordinator { } continue; } - acceptedCount += 1; + accepted.push(outcome.worker); this.state = { ...this.state, consecutiveErrors: 0 }; buffer.push(outcome.worker); - this.logProgress(acceptedCount); + this.logProgress(accepted.length); continue; } reducer = undefined; if ("status" in outcome) { - buffer = [...outcome.consumed, ...buffer]; + buffer = [...outcome.consumed, ...buffer].sort( + compareCompletionSequence, + ); reducerFailures += 1; this.log({ event: "dedup_worker_replaced", @@ -888,7 +992,7 @@ export class DeepScanCoordinator { outcome.error, ); this.abortController.abort(thresholdError.message); - await this.settleSchedulerWork(); + await reconcileRemainingDiscoveries("buffered"); throw thresholdError; } reducerFailures = 0; @@ -907,14 +1011,14 @@ export class DeepScanCoordinator { // Convergence cancels active workers, but their promises must settle before // the manifest records which results completed and which were canceled. - const lateFailure = await reconcileRemainingDiscoveries(); + const lateFailure = await reconcileRemainingDiscoveries("omitted"); // Once Deep reaches saturation, late worker errors cannot fail the scan. if (lateFailure && stopReason !== "saturated") throw lateFailure; if ( !previousReducerResultPath && - !(this.discoveryDeadlineReached && acceptedCount === 0) + !(this.discoveryDeadlineReached && accepted.length === 0) ) { throw new Error( "Deep Scan ended without a successfully reduced Standard scan.", @@ -922,8 +1026,9 @@ export class DeepScanCoordinator { } return { reason: stopReason, - omittedWorkerIds: [...new Set(omittedWorkerIds)], + omittedWorkerIds: unique(omittedWorkerIds), result: latestResult, + resultPath: previousReducerResultPath, }; } @@ -932,21 +1037,35 @@ export class DeepScanCoordinator { for (const worker of this.state.persistedWorkers ?? []) { if (worker.kind !== "discovery" || worker.status !== "succeeded") continue; - if (!worker.resultManifestPath || !worker.completionSequence) { + // Migrated workers can have frozen merge inputs without an attempt record. + const claimedInput = this.state.persistedDedupInputs?.find( + (input) => + input.discoveryWorkerId === worker.id && + (input.attempt === undefined || input.attempt === worker.attempt) && + input.resultManifestPath, + ); + const resultPath = + worker.acceptedResultPath ?? + claimedInput?.resultManifestPath ?? + worker.resultManifestPath; + if (!resultPath || !worker.completionSequence) { throw new Error( `Accepted discovery ${worker.id} has incomplete persisted evidence.`, ); } await validateDiscoveryArtifacts( this.artifacts, - worker.resultManifestPath, + resultPath, this.state.scanId, ); - await fs.readFile(worker.promptPath, "utf8"); recovered.push({ id: worker.id, - resultPath: worker.resultManifestPath, + label: basename(dirname(worker.promptPath)), + artifactDir: worker.artifactDir, + resultPath, completionSequence: worker.completionSequence, + attempt: worker.attempt, + ...(worker.threadId ? { threadId: worker.threadId } : {}), }); } return recovered.sort(compareCompletionSequence); @@ -954,11 +1073,13 @@ export class DeepScanCoordinator { private async recoverCompletedReducers( discoveries: AcceptedDiscovery[], - ): Promise<{ resultPath?: string; result?: DeepReductionInput }> { - const discoveryIds = new Set(discoveries.map((worker) => worker.id)); + ): Promise<{ result?: DeepReductionInput; resultPath?: string }> { + const discoveriesById = new Map( + discoveries.map((worker) => [worker.id, worker]), + ); const inputs = this.state.persistedDedupInputs ?? []; - let resultPath: string | undefined; let latestResult: DeepReductionInput | undefined; + let latestResultPath: string | undefined; const completedReducers = (this.state.persistedWorkers ?? []) .filter( (worker) => worker.kind === "dedup" && worker.status === "succeeded", @@ -970,48 +1091,79 @@ export class DeepScanCoordinator { left.id.localeCompare(right.id), ); for (const worker of completedReducers) { - if (!worker.resultManifestPath) { + // A later merge claim can retain a legacy aggregate's accepted reference. + const resultPath = + worker.acceptedResultPath ?? + this.state.persistedMergeClaims?.find( + (claim) => + claim.previousWorkerId === worker.id && claim.previousResultPath, + )?.previousResultPath ?? + worker.resultManifestPath; + if (!resultPath) { throw new Error( `Completed reducer ${worker.id} has no persisted result manifest.`, ); } const consumed = inputs .filter((input) => input.dedupWorkerId === worker.id) - .map((input) => discoveryIds.has(input.discoveryWorkerId)); + .sort((left, right) => left.inputOrder - right.inputOrder) + .map((input) => { + const discovery = discoveriesById.get(input.discoveryWorkerId); + return ( + discovery && { + ...discovery, + resultPath: input.resultManifestPath ?? discovery.resultPath, + attempt: input.attempt ?? discovery.attempt, + } + ); + }); if (consumed.length === 0 || consumed.some((value) => !value)) { throw new Error( `Completed reducer ${worker.id} has incomplete persisted inputs.`, ); } + const accepted = consumed as AcceptedDiscovery[]; + const claim = this.state.persistedMergeClaims?.find( + (item) => item.workerId === worker.id, + ); const { result } = await validateReducerArtifacts( { artifacts: this.artifacts, artifactDir: worker.artifactDir, - resultPath: worker.resultManifestPath, + resultPath, reducerId: worker.id, - previousReducerResultPath: resultPath, + previousReducerResultPath: claim + ? claim.previousResultPath + : latestResultPath, }, this.state.scanId, ); - latestResult = result; - await fs.readFile(worker.promptPath, "utf8"); - resultPath = worker.resultManifestPath; - } - return { resultPath, result: latestResult }; - } - - private async readPersistedExecutionPrompts(): Promise { - for (const worker of this.state.persistedWorkers ?? []) { - if ( - worker.kind === "setup" || - worker.status === "queued" || - worker.status === "running" || - (worker.status === "canceled" && worker.attempt === 0) - ) { - continue; + if (result.sourceCoverage === undefined) { + const context = { + root: worker.artifactDir, + repoRoot: this.state.targetPath, + scanId: this.state.scanId, + layout: "reducer" as const, + deepReducer: { + scanRoot: this.artifacts.scanDir, + claimedWorkers: accepted.map((source) => ({ + id: source.id, + resultPath: source.resultPath, + artifactDir: source.artifactDir, + attempt: source.attempt, + })), + }, + }; + const sources = await readDeepReductionSources(context); + result.sourceCoverage = aggregateSourceCoverage( + sources.discoveries, + latestResult ?? null, + ); } - await fs.readFile(worker.promptPath, "utf8"); + latestResult = result; + latestResultPath = resultPath; } + return { result: latestResult, resultPath: latestResultPath }; } private reducerReady( @@ -1193,3 +1345,7 @@ function errorKind(error: unknown): string { if (!(error instanceof Error)) return typeof error; return errorNameWithCode(error); } + +function unique(values: string[]): string[] { + return [...new Set(values)]; +} diff --git a/plugins/codex-security/mcp-app/src/deep-scan/executor.ts b/plugins/codex-security/mcp-app/src/deep-scan/executor.ts index 2aa48eeec4..f2f5bc9f42 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/executor.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/executor.ts @@ -15,7 +15,12 @@ import { resolve, win32, } from "node:path"; -import { Codex, type CyberAccessProgram } from "@openai/codex-sdk"; +import { + createCodexClient, + readCodexSessionTurn, +} from "../../../../../sdk/typescript/src/codex-session.js"; +import type { CodexOptions } from "@openai/codex-sdk"; +import type { CyberAccessProgram } from "@openai/codex-sdk"; import { parse as parseToml } from "smol-toml"; import { executablePathForSpawn } from "./executable-path.js"; import { @@ -36,10 +41,13 @@ import type { } from "./types.js"; export interface CodexSdkWorkerModelSettings { + /** Resolved by the execution owner, including when reconstructing a scan. */ + codexOptions?: CodexOptions; model?: string; reasoningEffort?: string; artifactContext?: CodexSdkWorkerArtifactContext; parentSandbox?: DeepWorkerParentSandbox; + runtimeSettings?: CodexSdkWorkerRuntimeSettings; } /** The coordinator supplies scan identity; worker tools never choose paths. */ @@ -48,10 +56,11 @@ export interface CodexSdkWorkerArtifactContext { repoRoot: string; scanId: string; scope?: string; + scanRoot?: string; pythonCommand?: string; } -interface CodexSdkWorkerRuntimeSettings { +export interface CodexSdkWorkerRuntimeSettings { reasoningSummary?: string; serviceTier?: string; cyberAccessProgram?: CyberAccessProgram; @@ -62,6 +71,7 @@ interface CodexSdkWorkerRuntimeSettings { } export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { + private runtimeModelConfig?: Promise>; private runtimeSettings?: Promise; constructor( @@ -77,13 +87,33 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { ); } const workerProfile = workerPermissionProfile(parentSandbox); - const configOverrides = - workerPermissionProfileConfigOverrides(workerProfile); + const resolved = this.modelSettings.codexOptions; const originalCwd = process.cwd(); - const childEnv = await snapshotWorkerEnvironment(); - // Snapshot the SDK's per-scan config once for this coordinator, including resumes. - const runtimeSettings = await (this.runtimeSettings ??= - workerRuntimeSettings(childEnv)); + const childEnv = await snapshotWorkerEnvironment(resolved?.env); + const runtimeSettings = await (this.runtimeSettings ??= this.modelSettings + .runtimeSettings + ? Promise.resolve(this.modelSettings.runtimeSettings) + : workerRuntimeSettings(childEnv)); + if (resolved?.apiKey !== undefined) + childEnv.CODEX_API_KEY = resolved.apiKey; + // Snapshot per-scan selections once; a reconstructed owner can supply them. + // Native account credentials continue to refresh in the selected home. + const modelConfig: NonNullable = { + ...(await (this.runtimeModelConfig ??= resolved?.config + ? Promise.resolve(resolved.config) + : workerModelConfig(childEnv))), + ...(this.modelSettings.model + ? { model: this.modelSettings.model } + : {}), + // The CLI can add effort levels before the pinned SDK widens ThreadOptions. + ...(this.modelSettings.reasoningEffort + ? { model_reasoning_effort: this.modelSettings.reasoningEffort } + : {}), + }; + const configOverrides = [ + ...(resolved?.configOverrides ?? []), + ...workerPermissionProfileConfigOverrides(workerProfile), + ]; const openAiApiKey = environmentVariable( childEnv, "OPENAI_API_KEY", @@ -95,7 +125,9 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { process.platform, )?.trim(); const codexPath = resolveCodexPath( - childEnv, + resolved?.codexPathOverride === undefined + ? childEnv + : { ...childEnv, CODEX_CLI_PATH: resolved.codexPathOverride }, process.platform, process.arch, originalCwd, @@ -104,37 +136,52 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { await preflightDeepScanWorkerPermissionProfile({ codexPath, cwd: request.workingDirectory, - configOverrides, + configOverrides: [ + ...Object.entries(workerModelSelection(modelConfig)).map( + ([key, value]) => `${key}=${tomlInlineValue(value)}`, + ), + ...configOverrides, + ...(resolved?.baseUrl + ? [`openai_base_url=${tomlString(resolved.baseUrl)}`] + : []), + ], expectedProfile: workerProfile, env: childEnv, allowOpenAiApiKeyFallback: Boolean(openAiApiKey && !codexApiKey), signal: request.signal, }); const prompt = await fs.readFile(request.promptPath, "utf8"); - const codex = new Codex({ + const codex = createCodexClient({ + ...resolved, codexPathOverride: executablePathForSpawn(codexPath), env: childEnv, // Codex exec reads CODEX_API_KEY; the SDK maps apiKey to that variable. // Keep native credentials unless the worker has no configured account. ...(useOpenAiApiKey ? { apiKey: openAiApiKey } : {}), config: { - ...(runtimeSettings.reasoningSummary === undefined + ...modelConfig, + ...(modelConfig.model_reasoning_summary !== undefined || + runtimeSettings.reasoningSummary === undefined ? {} : { model_reasoning_summary: runtimeSettings.reasoningSummary }), - ...(runtimeSettings.serviceTier === undefined + ...(modelConfig.service_tier !== undefined || + runtimeSettings.serviceTier === undefined ? {} : { service_tier: runtimeSettings.serviceTier }), - // The CLI can add effort levels before the pinned SDK widens ThreadOptions. - ...(this.modelSettings.reasoningEffort - ? { model_reasoning_effort: this.modelSettings.reasoningEffort } - : {}), mcp_servers: { + ...(isRecord(modelConfig.mcp_servers) + ? modelConfig.mcp_servers + : {}), // Discovery workers use the bundled skills and artifacts, not the parent workbench MCP. // A disabled server still needs a valid transport while Codex resolves plugin configuration. "codex-security": { command: "node", enabled: false }, ...this.compactArtifactServer(request), }, - ...workerSubagentConfig(request.subagents, runtimeSettings.features), + ...workerSubagentConfig( + request.subagents, + modelConfig, + runtimeSettings.features, + ), }, // Structured SDK config cannot preserve literal filesystem keys such as // ":root" or "/repo/.env"; raw overrides keep this inline TOML intact. @@ -150,7 +197,7 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { workingDirectory: request.workingDirectory, } as const; const thread = request.resumeThreadId - ? codex.resumeThread(request.resumeThreadId, threadOptions) + ? codex.resumeThread!(request.resumeThreadId, threadOptions) : codex.startThread(threadOptions); const input = request.resumeThreadId ? (request.continuationPrompt ?? prompt) @@ -168,46 +215,61 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { signal: controller.signal, cyberAccessProgram: runtimeSettings.cyberAccessProgram, }); - let threadId: string | undefined; - let turnCompleted = false; - let lastStreamError: string | undefined; const diagnostics: CodexWorkerDiagnostic[] = []; - for await (const event of events) { - const item = event.type === "item.completed" ? event.item : event; - if (item.type === "error") { - const fallbackError = deepScanPermissionProfileFallbackError( - item.message, - ); - if (fallbackError) { - controller.abort(fallbackError); - throw fallbackError; + const turn = await readCodexSessionTurn({ + thread, + events, + stopOnCompletion: true, + onEvent: async (event) => { + if ( + event.type === "thread.started" && + typeof event.thread_id === "string" + ) { + await request.onThreadStarted?.(event.thread_id); + } else if ( + event.type === "item.completed" && + isRecord(event.item) + ) { + const fallbackError = + event.item.type === "error" && + typeof event.item.message === "string" + ? deepScanPermissionProfileFallbackError(event.item.message) + : undefined; + if (fallbackError) { + controller.abort(fallbackError); + throw fallbackError; + } + appendSafeItemDiagnostic(diagnostics, event.item); + } else if (event.type === "turn.completed") { + request.signal.removeEventListener("abort", forwardAbort); + } else if (event.type === "turn.failed") { + throw new Error((event.error as { message: string }).message); + } else if ( + event.type === "error" && + typeof event.message === "string" + ) { + const fallbackError = deepScanPermissionProfileFallbackError( + event.message, + ); + if (fallbackError) { + controller.abort(fallbackError); + throw fallbackError; + } + // Codex exec emits retry-in-progress notifications as error events. + appendCodeModeFrameDiagnostic(diagnostics, event.message); } - } - if (event.type === "thread.started") { - threadId = event.thread_id; - await request.onThreadStarted?.(threadId); - } else if (event.type === "item.completed") { - appendSafeItemDiagnostic(diagnostics, event.item); - } else if (event.type === "turn.completed") { - turnCompleted = true; - request.signal.removeEventListener("abort", forwardAbort); - break; - } else if (event.type === "turn.failed") { - throw new Error(event.error.message); - } else if (event.type === "error") { - // Codex exec currently emits retry-in-progress notifications as error events. - lastStreamError = event.message; - appendCodeModeFrameDiagnostic(diagnostics, event.message); - } - } - if (!turnCompleted) { - const detail = lastStreamError ? `: ${lastStreamError}` : ""; + }, + }); + if (turn.status !== "completed") { + const detail = turn.lastStreamError + ? `: ${turn.lastStreamError}` + : ""; throw new Error( `Codex worker stream ended before turn.completed${detail}`, ); } return { - threadId: threadId ?? thread.id ?? undefined, + threadId: turn.threadId ?? thread.id ?? undefined, ...(diagnostics.length > 0 ? { diagnostics } : {}), }; } finally { @@ -292,15 +354,24 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { function workerSubagentConfig( subagents: number, + config: NonNullable, inheritedFeatures: CodexSdkWorkerRuntimeSettings["features"], ) { return { // V1 counts children; V2 counts the root plus its children. Keeping its // feature disabled lets the model choose either runtime without rejecting // inherited agents.max_threads configuration. - ...(subagents > 0 ? { agents: { max_threads: subagents } } : {}), + ...(subagents > 0 + ? { + agents: { + ...(isRecord(config.agents) ? config.agents : {}), + max_threads: subagents, + }, + } + : {}), features: { ...inheritedFeatures, + ...(isRecord(config.features) ? config.features : {}), multi_agent_v2: { enabled: false, max_concurrent_threads_per_session: subagents + 1, @@ -316,7 +387,7 @@ function workerSubagentConfig( }; } -type TomlValue = string | number | boolean | TomlObject; +type TomlValue = string | number | boolean | TomlValue[] | TomlObject; type TomlObject = { [key: string]: TomlValue }; function workerPermissionProfile(sandbox: DeepWorkerParentSandbox): TomlObject { @@ -346,6 +417,7 @@ function tomlInlineValue(value: TomlValue): string { if (typeof value === "string") return tomlString(value); if (typeof value === "number") return String(value); if (typeof value === "boolean") return value ? "true" : "false"; + if (Array.isArray(value)) return `[${value.map(tomlInlineValue).join(",")}]`; return `{${Object.entries(value) .map(([key, entry]) => `${tomlKey(key)}=${tomlInlineValue(entry)}`) .join(",")}}`; @@ -471,6 +543,47 @@ function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null; } +// These are the existing non-secret selections written by the SDK preflight +// adapter. Reading only summary left provider selection in a shared home. +function workerModelSelection( + config: NonNullable, +): TomlObject { + const result: TomlObject = {}; + for (const key of [ + "model", + "model_provider", + "model_reasoning_effort", + "model_reasoning_summary", + "service_tier", + "model_providers", + ]) { + const value = config[key]; + if (value !== undefined) result[key] = value; + } + return result; +} + +async function workerModelConfig( + environment: Record, +): Promise> { + const configPath = environmentVariable( + environment, + "CODEX_SECURITY_CONFIG_PATH", + process.platform, + ); + if (!configPath) return {}; + const config = parseToml(await fs.readFile(configPath, "utf8")); + const profiles = config.profiles; + const profile = + typeof config.profile === "string" && isRecord(profiles) + ? profiles[config.profile] + : undefined; + return workerModelSelection({ + ...config, + ...(isRecord(profile) ? profile : {}), + } as NonNullable); +} + async function workerRuntimeSettings( environment: Record, ): Promise { @@ -480,7 +593,14 @@ async function workerRuntimeSettings( process.platform, ); if (!configPath) return {}; - const config = parseToml(await fs.readFile(configPath, "utf8")); + return workerRuntimeSettingsFromConfig( + parseToml(await fs.readFile(configPath, "utf8")), + ); +} + +export function workerRuntimeSettingsFromConfig( + config: Record, +): CodexSdkWorkerRuntimeSettings { const profiles = config.profiles; const profile = typeof config.profile === "string" && isRecord(profiles) @@ -518,9 +638,11 @@ async function workerRuntimeSettings( return settings; } -async function snapshotWorkerEnvironment(): Promise> { +async function snapshotWorkerEnvironment( + source: NodeJS.ProcessEnv = process.env, +): Promise> { const environment = Object.fromEntries( - Object.entries(process.env).filter( + Object.entries(source).filter( (entry): entry is [string, string] => entry[1] !== undefined, ), ) as Record; @@ -533,7 +655,7 @@ async function snapshotWorkerEnvironment(): Promise> { "CODEX_MANAGED_PACKAGE_ROOT", "LOCALAPPDATA", ]) { - const value = process.env[name]; + const value = environmentVariable(source, name, process.platform); for (const key of Object.keys(environment)) { if (key.toUpperCase() === name) delete environment[key]; } diff --git a/plugins/codex-security/mcp-app/src/deep-scan/finalization.ts b/plugins/codex-security/mcp-app/src/deep-scan/finalization.ts new file mode 100644 index 0000000000..8e2fb6f4b6 --- /dev/null +++ b/plugins/codex-security/mcp-app/src/deep-scan/finalization.ts @@ -0,0 +1,183 @@ +import { + deepReductionToScanDraft, + parseDeepReduction, +} from "./artifact-validation.js"; +import { + createScanArtifactContext, + type RunArtifactWorkbench, +} from "../artifact-context.js"; +import { + recordCodexSecurityScanDraftViaWorkbench, + type DeepScanPublication, +} from "../artifact-scan-draft.js"; +import { WorkbenchDeepScanStore } from "./store.js"; +import { createDeepScanArtifacts } from "./artifacts.js"; +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { type ScanDraftInput } from "../artifact-scan-draft.js"; +import { requireRegularFile, type DeepScanArtifacts } from "./artifacts.js"; +import type { + DeepScanFinalizationInput, + DeepScanRunState, + DeepScanStore, +} from "./types.js"; + +export type { DeepScanFinalizationInput } from "./types.js"; + +/** Publish exactly the saved aggregate; the enclosing scan owns public completion. */ +export async function publishSelectedDeepScan(input: { + run: DeepScanRunState; + artifacts: DeepScanArtifacts; + signal: AbortSignal; + publish: ( + draft: ScanDraftInput, + signal: AbortSignal, + publication: DeepScanPublication, + ) => Promise; + finish: DeepScanStore["finish"]; +}): Promise { + const { run, artifacts, signal } = input; + const selection = selectedInput(run); + if (run.status === "succeeded") return run; + if (run.status !== "running") + throw new Error("Stopped Deep Scan finalization cannot become successful."); + signal.throwIfAborted(); + const draft = await readSelectedDeepScanDraft( + artifacts, + run.scanId, + selection, + ); + await input.publish(draft, signal, { + coordinatorGeneration: run.coordinatorGeneration, + resultPath: + selection.resultPath === null + ? null + : join(run.scanDir, selection.resultPath), + }); + signal.throwIfAborted(); + return input.finish({ + scanId: run.scanId, + reason: selection.terminalReason, + manifestPath: join(run.scanDir, "scan-manifest.json"), + omittedWorkerIds: selection.omittedWorkerIds, + }); +} + +/** Recreate the chosen draft without scheduling discovery or reducer work. */ +export async function readSelectedDeepScanDraft( + artifacts: DeepScanArtifacts, + scanId: string, + selection: DeepScanFinalizationInput, +): Promise { + if (selection.version !== 1) + throw new Error("Unsupported Deep Scan finalization input version."); + if (selection.resultPath === null) { + if ( + selection.terminalReason !== "capped" || + selection.resultSha256 !== null + ) { + throw new Error( + "An empty Deep Scan finalization requires the recorded discovery deadline.", + ); + } + return { + scanId, + findings: [], + coverage: { + completeness: "partial", + surfaces: [], + explicitExclusions: [], + deferred: [ + { + reason: + "The configured discovery time limit elapsed before any source review completed.", + }, + ], + }, + }; + } + const resultPath = join(artifacts.scanDir, selection.resultPath); + await requireRegularFile(resultPath, artifacts.scanDir); + const contents = await readFile(resultPath); + if ( + createHash("sha256").update(contents).digest("hex") !== + selection.resultSha256 + ) { + throw new Error( + "The selected Deep Scan finalization input changed after acceptance.", + ); + } + const stored = JSON.parse(contents.toString("utf8")); + const draft = deepReductionToScanDraft(parseDeepReduction(stored, true)); + if (draft.scanId !== scanId || draft.complete === false) { + throw new Error( + "Deep Scan finalization requires the selected complete result for this scan.", + ); + } + return draft; +} + +/** SDK recovery uses the installed plugin's publisher and the original parent scan. */ +export async function resumeSelectedDeepScan(input: { + scanId: string; + threadId: string; + pluginRoot: string; + runWorkbench: RunArtifactWorkbench; + signal: AbortSignal; + handoffClaimToken?: string; +}): Promise { + const store = new WorkbenchDeepScanStore(input.runWorkbench); + const run = await store.get(input.scanId, input.threadId); + selectedInput(run); + try { + await publishSelectedDeepScan({ + run, + artifacts: createDeepScanArtifacts(run.scanDir), + signal: input.signal, + publish: async (draft, signal, publication) => { + const context = await createScanArtifactContext( + input.scanId, + input.runWorkbench, + { + requireRunning: true, + requireClaim: true, + handoffClaimToken: input.handoffClaimToken, + pluginRoot: input.pluginRoot, + }, + ); + await recordCodexSecurityScanDraftViaWorkbench( + context, + draft, + input.runWorkbench, + signal, + publication, + ); + }, + finish: (selection) => + store.finish({ + ...selection, + coordinatorGeneration: run.coordinatorGeneration, + }), + }); + } catch (error) { + // The original coordinator may publish while the SDK recovers its parent turn. + const committed = await store + .get(input.scanId, input.threadId) + .catch(() => null); + if (committed?.status !== "succeeded") throw error; + } +} + +function selectedInput(run: DeepScanRunState): DeepScanFinalizationInput { + const selection = run.finalizationInput; + if (!selection) + throw new Error("Deep Scan has no selected finalization input."); + if ( + run.workflowVersion !== "deep-security-scan/v2" || + selection.version !== 1 + ) { + throw new Error("Unsupported Deep Scan finalization input version."); + } + return selection; +} diff --git a/plugins/codex-security/mcp-app/src/deep-scan/recovery-settings.ts b/plugins/codex-security/mcp-app/src/deep-scan/recovery-settings.ts new file mode 100644 index 0000000000..6bb5b02166 --- /dev/null +++ b/plugins/codex-security/mcp-app/src/deep-scan/recovery-settings.ts @@ -0,0 +1,470 @@ +import { promises as fs } from "node:fs"; +import { homedir } from "node:os"; +import { isAbsolute, join, win32 } from "node:path"; +import type { CodexOptions } from "@openai/codex-sdk"; +import { parse as parseToml } from "smol-toml"; +import { scanPreflightCodexConfig } from "../../../../../sdk/typescript/src/preflight-config.js"; +import { + resolveCodexProfile, + type JsonObject, +} from "../../../../../sdk/typescript/src/config.js"; +import { readScanLogs } from "../../../../../sdk/typescript/src/scan-logs.js"; +import { + resolveCodexPath, + workerRuntimeSettingsFromConfig, + type CodexSdkWorkerRuntimeSettings, +} from "./executor.js"; +import type { DeepWorkerParentSandbox } from "./parent-sandbox.js"; +import type { DeepScanRunState } from "./types.js"; + +/** Credentials and arbitrary environment/configuration stay with Codex. */ +export interface DeepScanExecutionSettings { + codexPath: string; + codexHome: string; + model?: string; + modelProvider?: string; + reasoningEffort?: string; + reasoningSummary?: string; + serviceTier?: string; + /** The native snapshot recorded no request tier; serviceTier preserves its wire behavior. */ + nativeServiceTierAbsent?: true; + providerConfig?: JsonObject; + parentSandbox?: DeepWorkerParentSandbox; + runtimeSettings?: CodexSdkWorkerRuntimeSettings; +} + +export interface DeepScanLegacySettingsContext { + config?: JsonObject; + usageOwner?: DeepScanRunState["usageOwner"]; +} + +export interface DeepScanExecutionSettingsSnapshot { + version: number; + settings: DeepScanExecutionSettings; +} + +export async function captureDeepScanExecutionSettings( + original: Pick, + parentSandbox: DeepWorkerParentSandbox, + environment: NodeJS.ProcessEnv = process.env, + parent?: { threadId: string; startedAt?: string }, +): Promise { + const codexHome = environment.CODEX_HOME || join(homedir(), ".codex"); + const configPath = + environment.CODEX_SECURITY_CONFIG_PATH ?? join(codexHome, "config.toml"); + let config: JsonObject; + try { + config = parseToml(await fs.readFile(configPath, "utf8")) as JsonObject; + } catch (error) { + if ( + environment.CODEX_SECURITY_CONFIG_PATH || + (error as NodeJS.ErrnoException).code !== "ENOENT" + ) + throw error; + config = {}; + } + // Reuse the SDK projection: custom provider credentials belong in the native home. + const selected = scanPreflightCodexConfig(resolveCodexProfile(config)); + // A recovered scan can have a different continuation. Only its recorded owner + // establishes original history; null means that historical binding is missing. + const owner = + original.usageOwner === undefined ? parent : original.usageOwner; + const native = !owner?.threadId + ? {} + : await originalParentSettings(codexHome, { + ...owner, + threadId: owner.threadId, + startedAt: parent?.startedAt ?? owner.startedAt, + }); + return executionSettings({ + codexPath: resolveCodexPath( + environment, + process.platform, + process.arch, + process.cwd(), + ), + codexHome: + !isAbsolute(codexHome) || + (process.platform === "win32" && + ["\\", "/"].includes(win32.parse(codexHome).root)) + ? await fs.realpath(codexHome) + : codexHome, + model: + original.model ?? (selected.model as string | undefined) ?? native.model, + reasoningEffort: + original.reasoningEffort ?? + (selected.model_reasoning_effort as string | undefined) ?? + native.reasoningEffort, + modelProvider: + (selected.model_provider as string | undefined) ?? native.modelProvider, + reasoningSummary: + (selected.model_reasoning_summary as string | undefined) ?? + native.reasoningSummary, + serviceTier: + (selected.service_tier as string | undefined) ?? native.serviceTier, + ...(selected.service_tier === undefined && native.nativeServiceTierAbsent + ? { nativeServiceTierAbsent: true as const } + : {}), + providerConfig: selected.model_providers as JsonObject | undefined, + runtimeSettings: workerRuntimeSettingsFromConfig(config), + parentSandbox, + }); +} + +async function originalParentSettings( + codexHome: string, + parent: { threadId: string; turnId?: string | null; startedAt?: string }, +): Promise> { + // Native config/read represents omitted selections as null. Recover recorded + // selections from the original parent; some native records omit the summary. + // History can be disabled or unavailable; configured selections still work. + try { + const log = await readScanLogs({ + scanId: parent.threadId, + threadId: parent.threadId, + executionThreadIds: [], + codexHome, + allowMissingRoot: true, + }); + const settings: Partial = {}; + let applied: Partial | undefined; + let summaryIsCompatibilityOnly = false; + const cutoff = + parent.startedAt === undefined ? Infinity : Date.parse(parent.startedAt); + for (const entry of log.events) { + const event = entry.event as Record; + const timestamp = + typeof event.timestamp === "string" + ? Date.parse(event.timestamp) + : undefined; + if (timestamp !== undefined && timestamp > cutoff) continue; + const payload = event.payload; + if (!payload || typeof payload !== "object" || Array.isArray(payload)) + continue; + const context = payload as Record; + if ( + event.type === "event_msg" && + context.type === "thread_settings_applied" + ) { + if ( + typeof context.thread_id === "string" && + context.thread_id !== parent.threadId + ) + continue; + const snapshot = context.thread_settings; + if ( + !snapshot || + typeof snapshot !== "object" || + Array.isArray(snapshot) + ) + continue; + const value = snapshot as Record; + applied = { + model: typeof value.model === "string" ? value.model : undefined, + modelProvider: + typeof value.model_provider_id === "string" + ? value.model_provider_id + : undefined, + reasoningEffort: + typeof value.reasoning_effort === "string" + ? value.reasoning_effort + : undefined, + reasoningSummary: + typeof value.reasoning_summary === "string" + ? value.reasoning_summary + : undefined, + // A persisted native absent tier and explicit standard both omit the + // request tier. This does not infer a tier from missing history. + serviceTier: + typeof value.service_tier === "string" + ? value.service_tier + : value.service_tier === undefined + ? "default" + : undefined, + ...(value.service_tier === undefined + ? { nativeServiceTierAbsent: true as const } + : {}), + }; + } + if (event.type === "session_meta") { + if (typeof context.model_provider === "string") + settings.modelProvider = context.model_provider; + // Codex 0.133 replaced turn_context.summary with a compatibility default. + // Fresh threads need not have a thread_settings_applied record to replace it. + const version = + typeof context.cli_version === "string" + ? /^(\d+)\.(\d+)\./u.exec(context.cli_version) + : null; + summaryIsCompatibilityOnly = + version !== null && + (Number(version[1]) > 0 || Number(version[2]) >= 133); + } + if (event.type === "turn_context") { + if (parent.turnId && context.turn_id !== parent.turnId) continue; + if (typeof context.model === "string") settings.model = context.model; + if (typeof context.effort === "string") + settings.reasoningEffort = context.effort; + if (!summaryIsCompatibilityOnly && typeof context.summary === "string") + settings.reasoningSummary = context.summary; + } + } + // Applied snapshots contain native selected values. Newer turn-context + // summaries are only a compatibility field, not the active selection. + return { ...settings, ...applied }; + } catch { + return {}; + } +} + +/** Read recorded execution settings or original legacy facts. */ +export async function loadDeepScanExecutionSettings( + _scanDir: string, + original?: Pick< + DeepScanRunState, + | "model" + | "reasoningEffort" + | "usageOwner" + | "createdAt" + | "workflowVersion" + | "executionSettings" + >, + readLegacyContext?: () => Promise, + environment: NodeJS.ProcessEnv = process.env, +): Promise> { + // Only the workbench creation transaction records the launch selection. Scan + // artifacts are model-writable and cannot select a preflight executable/home. + const saved = original?.executionSettings; + let settings: DeepScanExecutionSettings; + if (saved) { + if (saved.version !== 1) { + throw new Error( + "This Deep Scan uses an unsupported execution settings version.", + ); + } + settings = executionSettings(saved.settings); + } else { + if ( + original?.workflowVersion === "deep-security-scan/v1" || + original?.workflowVersion === "deep-scan-mcp/v1" + ) { + // Legacy runs predate the binding. Their saved recipe and recorded owner + // can recover selections, but cannot establish an original executable or + // home. Leave those unknown and retain the existing native launch behavior. + const context = await readLegacyContext?.(); + const selected = scanPreflightCodexConfig( + resolveCodexProfile(context?.config ?? {}), + ); + const owner = original.usageOwner ?? context?.usageOwner; + const home = environment.CODEX_HOME || join(homedir(), ".codex"); + const native = !owner?.threadId + ? {} + : await originalParentSettings(home, { + ...owner, + threadId: owner.threadId, + startedAt: original.createdAt ?? owner.startedAt, + }); + return { + model: + original.model ?? + (selected.model as string | undefined) ?? + native.model, + reasoningEffort: + original.reasoningEffort ?? + (selected.model_reasoning_effort as string | undefined) ?? + native.reasoningEffort, + modelProvider: + (selected.model_provider as string | undefined) ?? + native.modelProvider, + reasoningSummary: + (selected.model_reasoning_summary as string | undefined) ?? + native.reasoningSummary, + serviceTier: + (selected.service_tier as string | undefined) ?? native.serviceTier, + ...(selected.service_tier === undefined && + native.nativeServiceTierAbsent + ? { nativeServiceTierAbsent: true as const } + : {}), + providerConfig: + selected.model_provider === "amazon-bedrock" + ? (selected.model_providers as JsonObject | undefined) + : undefined, + }; + } + throw new Error( + "This Deep Scan has no recorded original execution settings; its executable and Codex home cannot be recovered.", + ); + } + if ( + !original || + (settings.model !== undefined && + settings.reasoningEffort !== undefined && + settings.modelProvider !== undefined && + settings.reasoningSummary !== undefined && + settings.serviceTier !== undefined) + ) + return settings; + // Earlier snapshots can omit native selections. Recover only from the saved + // home and recorded owner; the continuation's current config is not history. + const owner = original.usageOwner; + const native = !owner?.threadId + ? {} + : await originalParentSettings(settings.codexHome, { + ...owner, + threadId: owner.threadId, + startedAt: original.createdAt, + }); + // History reads can outlive this coordinator. Project missing selections for + // its workers without overwriting a snapshot owned by a newer coordinator. + return executionSettings({ + ...settings, + model: settings.model ?? original.model ?? native.model, + reasoningEffort: + settings.reasoningEffort ?? + original.reasoningEffort ?? + native.reasoningEffort, + modelProvider: settings.modelProvider ?? native.modelProvider, + reasoningSummary: settings.reasoningSummary ?? native.reasoningSummary, + serviceTier: settings.serviceTier ?? native.serviceTier, + ...(settings.serviceTier === undefined && native.nativeServiceTierAbsent + ? { nativeServiceTierAbsent: true as const } + : {}), + }); +} + +export function restoredDeepScanWorkerSettings( + settings: Partial, + currentParentSandbox: DeepWorkerParentSandbox, + environment: () => NodeJS.ProcessEnv = () => process.env, +): { + codexOptions: CodexOptions; + model?: string; + reasoningEffort?: string; + parentSandbox: DeepWorkerParentSandbox; + runtimeSettings?: CodexSdkWorkerRuntimeSettings; +} { + const originalSandbox = settings.parentSandbox; + const depths = [ + originalSandbox?.globScanMaxDepth, + currentParentSandbox.globScanMaxDepth, + ].filter((depth): depth is number => depth !== undefined); + // Native depth caps limit deny-glob expansion, not allowed traversal. Keep + // the larger finite cap, or no cap when either known policy has uncapped globs. + const uncapped = [originalSandbox, currentParentSandbox].some( + (sandbox) => + sandbox?.globScanMaxDepth === undefined && + sandbox?.filesystemDenies.some((path) => + ["*", "?", "[", "]"].some((character) => path.includes(character)), + ), + ); + return { + model: settings.model, + reasoningEffort: settings.reasoningEffort, + ...(settings.runtimeSettings === undefined + ? {} + : { runtimeSettings: settings.runtimeSettings }), + parentSandbox: { + filesystemDenies: [ + ...new Set([ + ...(originalSandbox?.filesystemDenies ?? []), + ...currentParentSandbox.filesystemDenies, + ]), + ], + ...(uncapped || depths.length === 0 + ? {} + : { globScanMaxDepth: Math.max(...depths) }), + }, + codexOptions: { + codexPathOverride: settings.codexPath, + // The executor reads this property for each launch. API keys can refresh; + // only the original account home and non-secret selections are bound. + get env() { + return Object.fromEntries( + Object.entries({ + ...environment(), + ...(settings.codexPath === undefined + ? {} + : { CODEX_CLI_PATH: settings.codexPath }), + ...(settings.codexHome === undefined + ? {} + : { CODEX_HOME: settings.codexHome }), + }).filter( + (entry): entry is [string, string] => entry[1] !== undefined, + ), + ); + }, + config: scanPreflightCodexConfig({ + ...(settings.model === undefined ? {} : { model: settings.model }), + ...(settings.reasoningEffort === undefined + ? {} + : { model_reasoning_effort: settings.reasoningEffort }), + ...(settings.modelProvider === undefined + ? {} + : { model_provider: settings.modelProvider }), + ...(settings.reasoningSummary === undefined + ? {} + : { model_reasoning_summary: settings.reasoningSummary }), + ...(settings.serviceTier === undefined + ? {} + : { service_tier: settings.serviceTier }), + ...(settings.providerConfig === undefined + ? {} + : { model_providers: settings.providerConfig }), + }) as NonNullable, + }, + }; +} + +function executionSettings( + value: DeepScanExecutionSettings, +): DeepScanExecutionSettings { + // Catalog provider definitions are reconstructed by the existing launch + // projection. Only Bedrock's per-scan AWS selectors need persistence. + const provider = + value.modelProvider === "amazon-bedrock" + ? (scanPreflightCodexConfig({ + ...(value.modelProvider === undefined + ? {} + : { model_provider: value.modelProvider }), + ...(value.providerConfig === undefined + ? {} + : { model_providers: value.providerConfig }), + }).model_providers as JsonObject | undefined) + : undefined; + const settings: DeepScanExecutionSettings = { + codexPath: value.codexPath, + codexHome: value.codexHome, + model: value.model, + modelProvider: value.modelProvider, + reasoningEffort: value.reasoningEffort, + reasoningSummary: value.reasoningSummary, + serviceTier: value.serviceTier, + ...(value.nativeServiceTierAbsent === true + ? { nativeServiceTierAbsent: true } + : {}), + ...(provider === undefined ? {} : { providerConfig: provider }), + ...(value.runtimeSettings === undefined + ? {} + : { runtimeSettings: structuredClone(value.runtimeSettings) }), + ...(value.parentSandbox === undefined + ? {} + : { + parentSandbox: { + filesystemDenies: [...value.parentSandbox.filesystemDenies], + ...(value.parentSandbox.globScanMaxDepth === undefined + ? {} + : { + globScanMaxDepth: value.parentSandbox.globScanMaxDepth, + }), + }, + }), + }; + if ( + typeof settings.codexPath !== "string" || + typeof settings.codexHome !== "string" + ) { + throw new Error( + "Deep Scan execution settings are missing the recorded executable or Codex home.", + ); + } + return settings; +} diff --git a/plugins/codex-security/mcp-app/src/deep-scan/registry.ts b/plugins/codex-security/mcp-app/src/deep-scan/registry.ts index 6b957cef14..3c1cbcb72c 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/registry.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/registry.ts @@ -2,7 +2,11 @@ import { setTimeout as delay } from "node:timers/promises"; import { DeepScanCoordinator } from "./coordinator.js"; import type { CoordinatorOptions } from "./coordinator.js"; import { isTransientPersistenceError } from "./store.js"; -import type { DeepScanCoordinatorClaim, DeepScanRunState } from "./types.js"; +import type { + CodexWorkerExecutor, + DeepScanCoordinatorClaim, + DeepScanRunState, +} from "./types.js"; const COORDINATOR_LEASE_MS = 30_000; const COORDINATOR_POLL_MS = 1_000; @@ -12,6 +16,11 @@ export { DeepScanNonRetryableError, } from "./coordinator.js"; +export interface DeepScanCoordinatorStartOptions extends CoordinatorOptions { + /** Resolve persisted execution settings only after acquiring this run. */ + prepareExecutor?: (run: DeepScanRunState) => Promise; +} + /** Owns the live coordinators in this MCP server process. */ export class DeepScanCoordinatorRegistry { private readonly coordinators = new Map(); @@ -20,7 +29,8 @@ export class DeepScanCoordinatorRegistry { return this.coordinators.get(scanId); } - start(options: CoordinatorOptions): DeepScanCoordinator { + start(options: DeepScanCoordinatorStartOptions): DeepScanCoordinator { + requireSupportedDeepScan(options.run); const existing = this.coordinators.get(options.run.scanId); if (existing) return existing; let coordinator!: DeepScanCoordinator; @@ -92,7 +102,10 @@ export class DeepScanRemoteCoordinator { private readonly input: { run: DeepScanRunState; registry: Pick; - options: Omit; + options: Omit< + DeepScanCoordinatorStartOptions, + "run" | "observeReplacement" + >; }, ) {} @@ -124,6 +137,7 @@ export class DeepScanRemoteCoordinator { } if (run !== undefined) { if (run.status !== "running") return run; + requireSupportedDeepScan(run); const heartbeat = run.updatedAt ? Date.parse(run.updatedAt) @@ -158,7 +172,11 @@ export class DeepScanRemoteCoordinator { } } if (claim?.acquired) { - const coordinator = registry.start({ ...options, run: claim.run }); + const coordinator = await startClaimedCoordinator( + registry, + options, + claim.run, + ); return deadline === undefined ? await coordinator.wait(signal) : await coordinator.wait( @@ -183,11 +201,12 @@ export class DeepScanRemoteCoordinator { export async function startOrJoinDeepScanCoordinator(input: { run: DeepScanRunState; registry: Pick; - options: Omit; + options: Omit; }): Promise<{ coordinator: DeepScanCoordinator | DeepScanRemoteCoordinator; joined: boolean; }> { + requireSupportedDeepScan(input.run); const existing = input.registry.get(input.run.scanId); if (existing) return { coordinator: existing, joined: true }; const threadId = input.options.threadId; @@ -212,11 +231,53 @@ export async function startOrJoinDeepScanCoordinator(input: { }; } return { - coordinator: input.registry.start({ ...input.options, run: claim.run }), + coordinator: await startClaimedCoordinator( + input.registry, + input.options, + claim.run, + ), joined: false, }; } +async function startClaimedCoordinator( + registry: Pick, + options: Omit, + run: DeepScanRunState, +): Promise { + requireSupportedDeepScan(run); + const executor = + options.prepareExecutor && !run.finalizationInput + ? await options.prepareExecutor(run) + : options.executor; + return registry.start({ ...options, executor, run }); +} + +function requireSupportedDeepScan(run: DeepScanRunState): void { + if ( + run.finalizationInput !== undefined && + (run.workflowVersion !== "deep-security-scan/v2" || + run.finalizationInput.version !== 1) + ) { + throw new Error( + "This executor does not support this Deep Scan finalization input version.", + ); + } + // Missing versions are supported for older adapters that did not project them. + if ( + (run.schemaVersion !== undefined && run.schemaVersion !== 1) || + (run.workflowVersion !== undefined && + run.workflowVersion !== "deep-security-scan/v1" && + run.workflowVersion !== "deep-scan-mcp/v1" && + run.workflowVersion !== "deep-security-scan/v2") + ) { + throw new Error( + "This Deep Scan uses an unsupported workflow or schema version. " + + "Resume it with a compatible Codex Security release.", + ); + } +} + function remoteAbortError(reason: unknown): Error { const error = new Error("Deep Scan observation was aborted.", { cause: reason, diff --git a/plugins/codex-security/mcp-app/src/deep-scan/store.ts b/plugins/codex-security/mcp-app/src/deep-scan/store.ts index 7e4487cbb9..6fd18bb73a 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/store.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/store.ts @@ -2,6 +2,7 @@ import { availableParallelism } from "node:os"; import { join } from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { writeJsonAtomic } from "./artifacts.js"; +import type { DeepScanExecutionSettings } from "./recovery-settings.js"; import { boundedDeepScanErrorMessage, DeepScanNonRetryableError, @@ -27,9 +28,11 @@ type JsonObject = Record; export type WorkbenchRunner = ( args: string[], input?: string, + selectFinalization?: boolean, + withExecutionSettings?: boolean, ) => Promise; -const WORKFLOW_VERSION = "deep-scan-mcp/v1"; +const WORKFLOW_VERSION = "deep-security-scan/v2"; const MAX_IDEMPOTENT_PERSISTENCE_ATTEMPTS = 3; const PERSISTENCE_RETRY_BASE_DELAY_MS = 100; @@ -116,6 +119,7 @@ export class WorkbenchDeepScanStore { { input: DeepScanCoordinatorLeaseInput; run: DeepScanRunState; + heartbeatWrite?: Promise; } >(); @@ -131,6 +135,7 @@ export class WorkbenchDeepScanStore { reasoningEffort?: string; threadId: string; scanRoot: string; + executionSettings?: DeepScanExecutionSettings | null; }): Promise { const userContext = input.userContext; const result = await this.enqueueWrite( @@ -141,7 +146,9 @@ export class WorkbenchDeepScanStore { ...(input.scanId ? ["--scan-id", input.scanId] : []), ...(input.targetPath ? ["--target-path", input.targetPath] : []), ...(input.scope ? ["--scope", input.scope] : []), - ...(userContext ? ["--user-context-stdin"] : []), + ...(userContext && input.executionSettings === undefined + ? ["--user-context-stdin"] + : []), ...(input.handoffClaimToken ? ["--claim-token", input.handoffClaimToken] : []), @@ -157,7 +164,14 @@ export class WorkbenchDeepScanStore { WORKFLOW_VERSION, ], false, - userContext, + input.executionSettings === undefined + ? userContext + : JSON.stringify({ + executionSettings: input.executionSettings, + userContext, + }), + false, + input.executionSettings !== undefined, ); const run = parseDeepScan(result); const startDisposition = result.startDisposition; @@ -166,23 +180,23 @@ export class WorkbenchDeepScanStore { "Codex Security workbench returned an invalid Deep Scan start disposition.", ); } - return run; + return { ...run, startDisposition }; } async get(scanId: string, threadId: string): Promise { const run = parseDeepScan( - await this.runWorkbench([ - "get-deep-scan", - "--scan-id", - scanId, - "--thread-id", - threadId, - ]), + await this.runWorkbench( + ["get-deep-scan", "--scan-id", scanId, "--thread-id", threadId], + undefined, + false, + true, + ), ); const generation = this.coordinatorLeases.get(scanId)?.run.coordinatorGeneration; if ( - run.status !== "running" || + (run.status !== "running" && + !(run.status === "succeeded" && run.finalizationInput)) || (generation !== undefined && run.coordinatorGeneration !== undefined && run.coordinatorGeneration > generation) @@ -195,17 +209,23 @@ export class WorkbenchDeepScanStore { async claimCoordinator( input: DeepScanCoordinatorLeaseInput, ): Promise { - const result = await this.enqueueWrite([ - "claim-deep-scan-coordinator", - "--scan-id", - input.scanId, - "--thread-id", - input.threadId, - ...this.coordinatorLeaseArgs(input.scanId), - ...(input.handoffClaimToken - ? ["--claim-token", input.handoffClaimToken] - : []), - ]); + const result = await this.enqueueWrite( + [ + "claim-deep-scan-coordinator", + "--scan-id", + input.scanId, + "--thread-id", + input.threadId, + ...this.coordinatorLeaseArgs(input.scanId), + ...(input.handoffClaimToken + ? ["--claim-token", input.handoffClaimToken] + : []), + ], + false, + undefined, + false, + true, + ); const run = parseDeepScan(result); const disposition = result.coordinatorDisposition; if ( @@ -247,16 +267,54 @@ export class WorkbenchDeepScanStore { ); } const updatedAt = new Date().toISOString(); - await writeJsonAtomic( - join( - lease.run.scanDir, - "artifacts", - "deep_discovery", - `coordinator-heartbeat-${lease.run.coordinatorGeneration}.json`, + await this.writeCoordinatorHeartbeat(lease, updatedAt); + return { ...lease.run, updatedAt }; + } + + async releaseCoordinator(scanId: string): Promise { + const lease = this.coordinatorLeases.get(scanId); + if (!lease) return; + this.coordinatorLeases.delete(scanId); + await this.writeCoordinatorHeartbeat(lease, new Date().toISOString(), true); + } + + private writeCoordinatorHeartbeat( + lease: { run: DeepScanRunState; heartbeatWrite?: Promise }, + updatedAt: string, + released = false, + ): Promise { + // Heartbeats bypass SQLite writes; only writes to this lease file must settle in order. + const { run } = lease; + const operation = (lease.heartbeatWrite ?? Promise.resolve()).then(() => + writeJsonAtomic( + join( + run.scanDir, + "artifacts", + "deep_discovery", + `coordinator-heartbeat-${run.coordinatorGeneration}.json`, + ), + { + coordinatorGeneration: run.coordinatorGeneration, + updatedAt, + ...(released ? { released: true } : {}), + }, ), - { coordinatorGeneration: lease.run.coordinatorGeneration, updatedAt }, ); - return { ...lease.run, updatedAt }; + lease.heartbeatWrite = operation.then( + () => undefined, + () => undefined, + ); + return operation; + } + + async cancel(scanId: string, threadId: string): Promise { + return this.enqueueWrite([ + "cancel-scan", + "--scan-id", + scanId, + "--thread-id", + threadId, + ]); } async updateWorker( @@ -291,8 +349,13 @@ export class WorkbenchDeepScanStore { ], true, ); - const worker = parseWorker(result, update.id); const state = objectValue(result.deepScan, "deepScan"); + const worker = parseWorker( + state.workerReceipt + ? { deepScan: { ...state, workers: [state.workerReceipt] } } + : result, + update.id, + ); return state.consecutiveErrors === undefined ? worker : { @@ -310,25 +373,27 @@ export class WorkbenchDeepScanStore { workerIds: string[]; promptPath: string; artifactDir: string; - }): Promise { - await this.enqueueWrite( - [ - "claim-deep-scan-dedup", - "--scan-id", - input.scanId, - "--worker-id", - input.id, - "--prompt-path", - input.promptPath, - "--artifact-dir", - input.artifactDir, - ...this.coordinatorLeaseArgs(input.scanId), - ...input.workerIds.flatMap((workerId) => [ - "--input-worker-id", - workerId, - ]), - ], - true, + }): Promise { + return parseDeepScan( + await this.enqueueWrite( + [ + "claim-deep-scan-dedup", + "--scan-id", + input.scanId, + "--worker-id", + input.id, + "--prompt-path", + input.promptPath, + "--artifact-dir", + input.artifactDir, + ...this.coordinatorLeaseArgs(input.scanId), + ...input.workerIds.flatMap((workerId) => [ + "--input-worker-id", + workerId, + ]), + ], + true, + ), ); } @@ -355,8 +420,45 @@ export class WorkbenchDeepScanStore { ); } + async selectFinalization(input: { + scanId: string; + coordinatorGeneration?: number; + reason: DeepScanTerminalReason; + manifestPath: string; + resultPath?: string; + omittedWorkerIds: string[]; + }): Promise { + return parseDeepScan( + await this.enqueueWrite( + [ + "finish-deep-scan", + "--scan-id", + input.scanId, + ...(input.coordinatorGeneration === undefined + ? this.coordinatorLeaseArgs(input.scanId) + : [ + "--coordinator-generation", + String(input.coordinatorGeneration), + ]), + "--terminal-reason", + input.reason, + "--manifest-path", + input.manifestPath, + ...input.omittedWorkerIds.flatMap((workerId) => [ + "--omitted-worker-id", + workerId, + ]), + ], + true, + JSON.stringify({ resultPath: input.resultPath ?? null }), + true, + ), + ); + } + async finish(input: { scanId: string; + coordinatorGeneration?: number; reason: DeepScanTerminalReason; manifestPath: string; stagedManifestPath?: string; @@ -368,7 +470,12 @@ export class WorkbenchDeepScanStore { "finish-deep-scan", "--scan-id", input.scanId, - ...this.coordinatorLeaseArgs(input.scanId), + ...(input.coordinatorGeneration === undefined + ? this.coordinatorLeaseArgs(input.scanId) + : [ + "--coordinator-generation", + String(input.coordinatorGeneration), + ]), "--terminal-reason", input.reason, "--manifest-path", @@ -482,12 +589,19 @@ export class WorkbenchDeepScanStore { args: string[], retryTransientFailure = false, input?: string, + selectFinalization = false, + withExecutionSettings = false, ): Promise { const operation = this.writeTail.then(async () => { try { return retryTransientFailure - ? await this.runIdempotentPersistence(args) - : await this.runWorkbench(args, input); + ? await this.runIdempotentPersistence(args, input, selectFinalization) + : await this.runWorkbench( + args, + input, + selectFinalization, + withExecutionSettings, + ); } catch (error) { const scanId = argumentValue(args, "--scan-id"); if (scanId && isStaleCoordinatorGenerationError(error)) { @@ -504,11 +618,15 @@ export class WorkbenchDeepScanStore { } /** Replay only existing, same-identity workbench mutations after transient failures. */ - private async runIdempotentPersistence(args: string[]): Promise { + private async runIdempotentPersistence( + args: string[], + input?: string, + selectFinalization = false, + ): Promise { const startedAt = Date.now(); for (let attempt = 1; ; attempt += 1) { try { - return await this.runWorkbench(args); + return await this.runWorkbench(args, input, selectFinalization); } catch (error) { if (!isTransientPersistenceError(error)) { throw error; @@ -689,6 +807,18 @@ export function parseDeepScan(result: JsonObject): DeepScanRunState { }; return { scanId: requiredString(value.scanId, "deepScan.scanId"), + ownerThreadId: optionalString(value.ownerThreadId), + schemaVersion: optionalPositiveInteger(value.schemaVersion), + workflowVersion: optionalString(value.workflowVersion), + finalizationInput: parseFinalizationInput(value.finalizationInput), + usageOwner: parseUsageOwner(value.usageOwner), + executionSettings: + value.executionSettings == null + ? undefined + : (objectValue( + value.executionSettings, + "deepScan.executionSettings", + ) as unknown as DeepScanRunState["executionSettings"]), status, phase: deepScanPhase(value.phase), coordinatorGeneration: optionalPositiveInteger(value.coordinatorGeneration), @@ -697,6 +827,8 @@ export function parseDeepScan(result: JsonObject): DeepScanRunState { targetPath: requiredString(value.targetPath, "deepScan.targetPath"), scope: requiredString(value.scope, "deepScan.scope"), userContext: optionalString(value.userContext), + model: optionalString(value.model), + reasoningEffort: optionalString(value.reasoningEffort), scanDir: requiredString(value.scanDir, "deepScan.scanDir"), config, dispatchedCount: nonNegativeInteger( @@ -717,6 +849,84 @@ export function parseDeepScan(result: JsonObject): DeepScanRunState { error: optionalString(value.error), persistedWorkers: parsePersistedWorkers(value.workers), persistedDedupInputs: parsePersistedDedupInputs(value.dedupInputs), + persistedMergeClaims: Array.isArray(value.mergeClaims) + ? value.mergeClaims.map((candidate) => { + const claim = objectValue(candidate, "deepScan.mergeClaim"); + return { + workerId: requiredString( + claim.workerId, + "deepScan.mergeClaim.workerId", + ), + previousWorkerId: optionalString(claim.previousWorkerId), + previousResultPath: optionalString(claim.previousResultPath), + previousResultSha256: optionalString(claim.previousResultSha256), + }; + }) + : [], + ...(value.committedMerge + ? { committedMerge: parseCommittedMerge(value.committedMerge) } + : {}), + }; +} + +function parseUsageOwner(value: unknown): DeepScanRunState["usageOwner"] { + if (value === undefined || value === null) return null; + const owner = objectValue(value, "deepScan.usageOwner"); + return { + threadId: optionalString(owner.threadId) ?? null, + turnId: optionalString(owner.turnId) ?? null, + startedAt: requiredString(owner.startedAt, "deepScan.usageOwner.startedAt"), + ...(typeof owner.dedicated === "boolean" + ? { dedicated: owner.dedicated } + : {}), + }; +} + +function parseFinalizationInput( + value: unknown, +): DeepScanRunState["finalizationInput"] { + if (value === undefined || value === null) return undefined; + const input = objectValue(value, "deepScan.finalizationInput"); + if ( + input.terminalReason !== "saturated" && + input.terminalReason !== "capped" + ) { + throw new Error( + "Codex Security workbench returned invalid finalization terminal reason.", + ); + } + if (!Array.isArray(input.omittedWorkerIds)) { + throw new Error( + "Codex Security workbench returned invalid finalization omissions.", + ); + } + return { + version: positiveInteger( + input.version, + "deepScan.finalizationInput.version", + ), + resultPath: + input.resultPath === null + ? null + : requiredString( + input.resultPath, + "deepScan.finalizationInput.resultPath", + ), + resultSha256: + input.resultSha256 === null + ? null + : requiredString( + input.resultSha256, + "deepScan.finalizationInput.resultSha256", + ), + terminalReason: input.terminalReason, + omittedWorkerIds: input.omittedWorkerIds.map((id) => + requiredString(id, "omittedWorkerId"), + ), + selectedAt: requiredString( + input.selectedAt, + "deepScan.finalizationInput.selectedAt", + ), }; } @@ -744,10 +954,37 @@ function parsePersistedDedupInputs( input.inputOrder, "deepScan.dedupInput.inputOrder", ), + resultManifestPath: optionalString(input.resultManifestPath), + resultManifestSha256: optionalString(input.resultManifestSha256), + attempt: optionalPositiveInteger(input.attempt), }; }); } +function parseCommittedMerge( + value: unknown, +): NonNullable { + const commit = objectValue(value, "deepScan.committedMerge"); + return { + workerId: requiredString( + commit.workerId, + "deepScan.committedMerge.workerId", + ), + resultManifestPath: requiredString( + commit.resultManifestPath, + "deepScan.committedMerge.resultManifestPath", + ), + resultManifestSha256: requiredString( + commit.resultManifestSha256, + "deepScan.committedMerge.resultManifestSha256", + ), + newFindings: nonNegativeInteger( + commit.newFindings, + "deepScan.committedMerge.newFindings", + ), + }; +} + function deepScanPhase(value: unknown): DeepScanRunState["phase"] { if (value === undefined || value === null) return undefined; if ( @@ -829,6 +1066,7 @@ function parsePersistedWorker(value: JsonObject): PersistedDeepScanWorker { attempt: nonNegativeInteger(value.attempt, "deepScan.worker.attempt"), threadId: optionalString(value.sdkThreadId), resultManifestPath: optionalString(value.resultManifestPath), + acceptedResultPath: optionalString(value.acceptedResultPath), completionSequence: optionalPositiveInteger(value.completionSequence), error: optionalString(value.error), }; diff --git a/plugins/codex-security/mcp-app/src/deep-scan/types.ts b/plugins/codex-security/mcp-app/src/deep-scan/types.ts index 33f63cbd25..512fba2231 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/types.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/types.ts @@ -1,5 +1,6 @@ +import type { DeepScanExecutionSettingsSnapshot } from "./recovery-settings.js"; import type { DeepReducerContext } from "../artifact-io.js"; -import type { WorkbenchDeepScanStore } from "./store.js"; +import type { ScanExecutionAttribution } from "../../../../../sdk/typescript/src/scan-sessions.js"; export type DeepScanTerminalReason = "saturated" | "capped"; @@ -28,8 +29,26 @@ export interface DeepScanCanonicalArtifacts { candidateLedgerPath: string; } +export type DeepScanReducerArtifacts = DeepScanCanonicalArtifacts; + +export interface DeepScanFinalizationInput { + version: number; + resultPath: string | null; + resultSha256: string | null; + terminalReason: DeepScanTerminalReason; + omittedWorkerIds: string[]; + selectedAt: string; +} + export interface DeepScanRunState { scanId: string; + ownerThreadId?: string; + startDisposition?: "created" | "joined"; + schemaVersion?: number; + workflowVersion?: string; + finalizationInput?: DeepScanFinalizationInput; + usageOwner?: ScanExecutionAttribution["owner"] | null; + executionSettings?: DeepScanExecutionSettingsSnapshot | null; status: DeepScanRunStatus; phase?: "setup" | "discovery" | "reducing" | "terminal"; coordinatorGeneration?: number; @@ -38,6 +57,8 @@ export interface DeepScanRunState { targetPath: string; scope: string; userContext?: string; + model?: string; + reasoningEffort?: string; scanDir: string; config: DeepScanConfig; dispatchedCount: number; @@ -49,12 +70,29 @@ export interface DeepScanRunState { error?: string; persistedWorkers?: PersistedDeepScanWorker[]; persistedDedupInputs?: PersistedDeepScanDedupInput[]; + persistedMergeClaims?: PersistedDeepScanMergeClaim[]; + committedMerge?: { + workerId: string; + resultManifestPath: string; + resultManifestSha256: string; + newFindings: number; + }; +} + +export interface PersistedDeepScanMergeClaim { + workerId: string; + previousWorkerId?: string; + previousResultPath?: string; + previousResultSha256?: string; } export interface PersistedDeepScanDedupInput { dedupWorkerId: string; discoveryWorkerId: string; inputOrder: number; + resultManifestPath?: string; + resultManifestSha256?: string; + attempt?: number; } export interface DeepScanCoordinatorClaim { @@ -91,6 +129,7 @@ export interface PersistedDeepScanWorker extends Omit< DeepScanWorkerMutation, "scanId" | "replaceableFailureKind" > { + acceptedResultPath?: string; completionSequence?: number; consecutiveErrors?: number; mergeState: DeepScanMergeState; @@ -106,10 +145,74 @@ export interface DedupCommit { } /** Durable operations implemented by the Python workbench. */ -export type DeepScanStore = Omit< - WorkbenchDeepScanStore, - "begin" | "coordinatorLeaseArgs" ->; +export interface DeepScanStore { + begin(input: { + scanId?: string; + targetPath?: string; + scope?: string; + userContext?: string; + handoffClaimToken?: string; + model?: string; + reasoningEffort?: string; + threadId: string; + scanRoot: string; + }): Promise; + get(scanId: string, threadId: string): Promise; + claimCoordinator( + input: DeepScanCoordinatorLeaseInput, + ): Promise; + heartbeatCoordinator( + input: DeepScanCoordinatorLeaseInput, + ): Promise; + cancel(scanId: string, threadId: string): Promise>; + updateWorker( + update: DeepScanWorkerMutation, + ): Promise; + claimDedup(input: { + id: string; + scanId: string; + workerIds: string[]; + promptPath: string; + artifactDir: string; + }): Promise; + commitDedup(commit: DedupCommit): Promise; + selectFinalization?(input: { + scanId: string; + coordinatorGeneration?: number; + reason: DeepScanTerminalReason; + manifestPath: string; + resultPath?: string; + omittedWorkerIds: string[]; + }): Promise; + finish(input: { + scanId: string; + coordinatorGeneration?: number; + reason: DeepScanTerminalReason; + manifestPath: string; + stagedManifestPath?: string; + omittedWorkerIds: string[]; + }): Promise; + fail( + scanId: string, + message: string, + status?: "failed" | "interrupted", + manifestPath?: string, + stagedManifestPath?: string, + ): Promise; + recordStoppedPublicationFailure( + scanId: string, + message: string, + coordinatorGeneration?: number, + ): Promise; + updateProgress(input: { + scanId: string; + handoffClaimToken?: string; + phase?: "preflight" | "discovery"; + deepReviewPass?: number; + reviewItemsTotal?: number; + reviewItemsCompleted?: number; + }): Promise; +} /** Host-bound worker artifact state; never populate this from model input. */ export interface CodexWorkerArtifactContext { diff --git a/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts b/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts index b8822f1666..583e59b46e 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts @@ -1,8 +1,13 @@ +import type { ScanDraftInput } from "../artifact-scan-draft.js"; +import { + auditEvidence, + runAcceptedAudit, +} from "../../../../../sdk/typescript/src/accepted-audit.js"; import { promises as fs } from "node:fs"; import { join } from "node:path"; -import { getCodexSecurityDeepReducerInputs } from "../artifact-deep-reducer.js"; +import { readDeepReductionSources } from "../artifact-deep-reducer.js"; import { - validateDiscoveryArtifacts, + readDiscoveryAuditDraft, validateReducerArtifacts, } from "./artifact-validation.js"; import type { @@ -35,8 +40,12 @@ import type { export interface AcceptedDiscovery { id: string; + label: string; + artifactDir: string; resultPath: string; completionSequence: number; + attempt: number; + threadId?: string; } export type DiscoveryOutcome = @@ -47,14 +56,21 @@ export type DiscoveryOutcome = status: "failed"; workerId: string; error: Error; + attempt?: number; + threadId?: string; replaceableFailureKind?: DeepScanReplaceableFailureKind; consecutiveErrors?: number; }; export interface SuccessfulDedupOutcome { type: "dedup"; + id: string; + consumed: AcceptedDiscovery[]; resultPath: string; result: DeepReductionInput; + newFindings: number; + attempt: number; + threadId?: string; run: DeepScanRunState; } @@ -73,6 +89,7 @@ export interface ReducerRequest { label: string; consumed: AcceptedDiscovery[]; previousReducerResultPath?: string; + previousSourceCoverage?: DeepReductionInput["sourceCoverage"]; } export interface DeepScanWorkerRunnerOptions { @@ -88,15 +105,22 @@ export interface DeepScanWorkerRunnerOptions { signal: AbortSignal; } +interface WorkerAttemptEvidence { + attempt: number; + threadId?: string; +} + type WorkerAttemptOutcome = | { status: "succeeded"; attempt: number; threadId?: string } | { status: "failed"; error: Error; + attempt?: number; + threadId?: string; replaceableFailureKind?: DeepScanReplaceableFailureKind; consecutiveErrors?: number; } - | { status: "canceled" }; + | { status: "canceled"; attempt?: number; threadId?: string }; /** Owns prompt rendering, retries, validation, and persistence for each worker. */ export class DeepScanWorkerRunner { @@ -155,8 +179,13 @@ export class DeepScanWorkerRunner { artifactContext: { root: artifactDir, layout: "worker" }, subagents: run.config.subagents, validate: async () => { - await validateDiscoveryArtifacts(artifacts, resultPath, run.scanId); - discoveryValidated = true; + const draft = await readDiscoveryAuditDraft( + artifacts, + resultPath, + run.scanId, + ); + discoveryValidated = draft.complete !== false; + return draft; }, beforeRetry: async (attempt) => { await archiveDirectory( @@ -243,8 +272,12 @@ export class DeepScanWorkerRunner { status: "succeeded", worker: { id: workerId, - resultPath, + label: workerLabel, + artifactDir, + resultPath: persisted.acceptedResultPath ?? resultPath, completionSequence: persisted.completionSequence, + attempt: outcome.attempt, + threadId: outcome.threadId, }, }; } @@ -253,9 +286,9 @@ export class DeepScanWorkerRunner { const { id: reducerId, label: reducerLabel, - consumed, - previousReducerResultPath, + previousSourceCoverage, } = request; + let { consumed, previousReducerResultPath } = request; const { artifacts, run } = this.options; const reducerRoot = join(artifacts.dedupRoot, reducerLabel); const artifactDir = join(reducerRoot, "output"); @@ -266,13 +299,37 @@ export class DeepScanWorkerRunner { const workerIds = consumed.map((worker) => worker.id); const basePrompt = renderDedupPrompt(reducerLabel, workerIds); await writePrivateFile(promptPath, basePrompt); - await this.options.store.claimDedup({ + const claimed = await this.options.store.claimDedup({ id: reducerId, scanId: run.scanId, workerIds, promptPath, artifactDir, }); + const claim = claimed?.persistedMergeClaims?.find( + (item) => item.workerId === reducerId, + ); + if (claim) { + previousReducerResultPath = claim.previousResultPath; + const inputs = + claimed?.persistedDedupInputs?.filter( + (item) => item.dedupWorkerId === reducerId, + ) ?? []; + consumed = inputs + .sort((a, b) => a.inputOrder - b.inputOrder) + .map((item) => { + const discovery = consumed.find( + (worker) => worker.id === item.discoveryWorkerId, + ); + if (!discovery || !item.resultManifestPath) + throw new Error("The reducer claim is missing an accepted input."); + return { + ...discovery, + resultPath: item.resultManifestPath, + attempt: item.attempt ?? discovery.attempt, + }; + }); + } this.options.log({ event: "dedup_claimed", scanId: run.scanId, @@ -280,6 +337,9 @@ export class DeepScanWorkerRunner { count: consumed.length, }); + const persistSourceCoverage = + "workflowVersion" in run && + run.workflowVersion === "deep-security-scan/v2"; const artifactContext = { root: artifactDir, repoRoot: run.targetPath, @@ -287,17 +347,23 @@ export class DeepScanWorkerRunner { layout: "reducer" as const, deepReducer: { scanRoot: artifacts.scanDir, + persistSourceCoverage, claimedWorkers: consumed.map((worker) => ({ id: worker.id, resultPath: worker.resultPath, + artifactDir: worker.artifactDir, + attempt: worker.attempt, })), previousReducerResultPath, }, }; // Snapshot inputs before execution: direct file output has the same // conservation checks as the MCP writer without rereading consumed sources. - const sources = await getCodexSecurityDeepReducerInputs(artifactContext); - let reducerValidation!: ReducerArtifactValidation; + const sources = await readDeepReductionSources(artifactContext); + if (sources.previous && previousSourceCoverage !== undefined) { + sources.previous.sourceCoverage = structuredClone(previousSourceCoverage); + } + let reducerValidation: ReducerArtifactValidation | undefined; let outcome = await this.runWorkerWithRetries({ workerId: reducerId, kind: "dedup", @@ -315,6 +381,7 @@ export class DeepScanWorkerRunner { reducerId, previousReducerResultPath, sources, + persistSourceCoverage, }, run.scanId, ); @@ -353,7 +420,11 @@ export class DeepScanWorkerRunner { }; } if (outcome.status === "canceled") throw abortError(); - + if (!reducerValidation) { + throw new Error( + `${reducerId} completed without validated reducer artifacts.`, + ); + } const commit = { id: reducerId, scanId: run.scanId, @@ -365,17 +436,27 @@ export class DeepScanWorkerRunner { reducerId, async () => await this.options.store.commitDedup(commit), ); + const accepted = committed.committedMerge; + const acceptedPath = accepted?.resultManifestPath ?? resultPath; + // V1 checkpoints omit host-only coverage; retain the validated projection. + const acceptedResult = reducerValidation.result; + const newFindings = accepted?.newFindings ?? reducerValidation.newFindings; this.options.log({ event: "dedup_committed", scanId: run.scanId, workerId: reducerId, count: consumed.length, - newFindings: reducerValidation.newFindings, + newFindings, }); return { type: "dedup", - resultPath, - result: reducerValidation.result, + id: reducerId, + consumed, + resultPath: acceptedPath, + result: acceptedResult, + newFindings, + attempt: outcome.attempt, + threadId: outcome.threadId, run: committed, }; } @@ -388,7 +469,7 @@ export class DeepScanWorkerRunner { artifactDir: string; artifactContext?: CodexWorkerArtifactContext; subagents: number; - validate: () => Promise; + validate: () => Promise; beforeRetry: (attempt: number) => Promise; }): Promise { const { run, signal } = this.options; @@ -423,53 +504,76 @@ export class DeepScanWorkerRunner { attempt, }); try { - const result = await this.options.executor.run({ - kind: input.kind, - promptPath: executionPromptPath, - // Discovery workers write only to their isolated directory. Reducers - // own shared scan artifacts; the target remains read-only. - workingDirectory: - input.kind === "discovery" - ? input.artifactDir - : join(run.scanDir, "artifacts"), - subagents: input.subagents, - signal, - resumeThreadId: resumableThreadId, - continuationPrompt: resumableThreadId - ? (continuationPrompt ?? - transientExecutionContinuation(input.kind, attempt)) - : undefined, - artifactContext: input.artifactContext, - onThreadStarted: async (threadId) => { - activeThreadId = threadId; - lastThreadId = threadId; - await this.options.store.updateWorker({ - ...baseMutation, - threadId, - }); - this.options.log({ - event: "worker_thread_started", - scanId: run.scanId, - workerId: input.workerId, - kind: input.kind, - attempt, - threadId, - }); - }, - }); - if (signal.aborted) { - return await this.cancelAttempt(input, attempt, activeThreadId); - } - validationStarted = true; - try { - await input.validate(); - } catch (validationError) { - throw withWorkerDiagnostics(validationError, result.diagnostics); + const execute = () => + this.options.executor.run({ + kind: input.kind, + promptPath: executionPromptPath, + // Discovery workers write only to their isolated directory. Reducers + // own shared scan artifacts; the target remains read-only. + workingDirectory: + input.kind === "discovery" + ? input.artifactDir + : join(run.scanDir, "artifacts"), + subagents: input.subagents, + signal, + resumeThreadId: resumableThreadId, + continuationPrompt: resumableThreadId + ? (continuationPrompt ?? + transientExecutionContinuation(input.kind, attempt)) + : undefined, + artifactContext: input.artifactContext, + onThreadStarted: async (threadId) => { + activeThreadId = threadId; + lastThreadId = threadId; + await this.options.store.updateWorker({ + ...baseMutation, + threadId, + }); + this.options.log({ + event: "worker_thread_started", + scanId: run.scanId, + workerId: input.workerId, + kind: input.kind, + attempt, + threadId, + }); + }, + }); + const accept = async (result: Awaited>) => { + validationStarted = true; + let accepted: ScanDraftInput | void; + try { + accepted = await input.validate(); + } catch (validationError) { + throw withWorkerDiagnostics(validationError, result.diagnostics); + } + validationCompleted = accepted?.complete !== false; + return accepted === undefined ? {} : auditEvidence(accepted); + }; + // Reducers keep their aggregate contract; discovery uses the shared audit. + const audit = + input.kind === "discovery" + ? await runAcceptedAudit({ signal, execute, accept }) + : undefined; + let result: Awaited>; + if (audit) { + if (audit.status === "checkpoint") { + throw withWorkerDiagnostics( + new Error( + "Standard scan worker wrote only a checkpoint; its audit is not complete.", + ), + audit.execution.diagnostics, + ); + } + result = audit.execution; + } else { + result = await execute(); + if (signal.aborted) + return await this.cancelAttempt(input, attempt, activeThreadId); + await accept(result); } - validationCompleted = true; - if (signal.aborted) { + if (signal.aborted) return await this.cancelAttempt(input, attempt, activeThreadId); - } this.options.log({ event: "worker_succeeded", scanId: run.scanId, @@ -521,6 +625,8 @@ export class DeepScanWorkerRunner { ...(persistedFailure.consecutiveErrors === undefined ? {} : { consecutiveErrors: persistedFailure.consecutiveErrors }), + attempt, + threadId: activeThreadId, }; } await this.options.store.updateWorker({ @@ -620,6 +726,12 @@ export class DeepScanWorkerRunner { try { return await operation(); } catch (firstError) { + // The workbench store already replays these writes before reporting exhaustion. + if ( + firstError instanceof Error && + firstError.name === "DeepScanPersistenceError" + ) + throw firstError; this.options.log({ event, scanId: this.options.run.scanId, @@ -648,7 +760,11 @@ export class DeepScanWorkerRunner { threadId: string | undefined, ): Promise { await this.persistWorkerCancellation(input, attempt, threadId); - return { status: "canceled" }; + return { + status: "canceled", + attempt, + threadId, + }; } } diff --git a/plugins/codex-security/mcp-app/src/server/compact-artifact-tools.ts b/plugins/codex-security/mcp-app/src/server/compact-artifact-tools.ts index 796aa9a3a4..1049437033 100644 --- a/plugins/codex-security/mcp-app/src/server/compact-artifact-tools.ts +++ b/plugins/codex-security/mcp-app/src/server/compact-artifact-tools.ts @@ -189,7 +189,7 @@ export function registerScanDraftTools( name: "record_codex_security_scan_draft", title: "Record Codex Security Scan Draft", description: - "Save the canonical threat model, findings and coverage as an unsealed draft. Use complete:false as soon as a model is available, even with empty findings and partial coverage, then for progress checkpoints; use complete:true for the final result. The host derives threatmodel.md. Keep unvalidated candidates in coverage.deferred. On terminal Standard or diff drafts, close generic review tasks with coverage.resolvedDeferred:[{id,reason}], copying IDs from the returned coverage. Update linked surfaces by their saved IDs.", + "Save the canonical threat model, findings and coverage as an unsealed draft. Use complete:false as soon as a model is available, even with empty findings and partial coverage, then for progress checkpoints; use complete:true for the final result. The host derives threatmodel.md. Keep unvalidated candidates in coverage.deferred. On terminal Standard or diff drafts, close generic review tasks with coverage.resolvedDeferred:[{id,reason}], copying IDs from the returned coverage. Update linked surfaces by their saved IDs. Provide the required top-level scanId. The workbench supplies findingId, occurrenceId and fingerprints within each finding; includePaths and excludePaths within scope; and documentType, schemaVersion, scanId, mode, includePaths, excludePaths, receiptRefs and inventoryStrategy within coverage; omit these host-owned fields from your draft.", inputSchema: scanDraftInputSchema, readOnly: false, handler: async (input, requestContext) => { @@ -306,7 +306,7 @@ export function registerCompactWorkerArtifactTools( name: "record_codex_security_scan_draft", title: "Record Codex Security Scan Draft", description: - "Save this Standard worker's semantic findings and coverage. Use complete:false for progress checkpoints, then complete:true for its final result; keep unvalidated candidates in coverage.deferred. On terminal drafts, close generic review tasks with coverage.resolvedDeferred:[{id,reason}], copying IDs from the returned coverage. Update linked surfaces by their saved IDs.", + "Save this Standard worker's semantic findings and coverage. Use complete:false for progress checkpoints, then complete:true for its final result; keep unvalidated candidates in coverage.deferred. On terminal drafts, close generic review tasks with coverage.resolvedDeferred:[{id,reason}], copying IDs from the returned coverage. Update linked surfaces by their saved IDs. Provide the required top-level scanId. The workbench supplies findingId, occurrenceId and fingerprints within each finding; includePaths and excludePaths within scope; and documentType, schemaVersion, scanId, mode, includePaths, excludePaths, receiptRefs and inventoryStrategy within coverage; omit these host-owned fields from your draft.", inputSchema: scanDraftInputSchema, readOnly: false, handler: async (value) => diff --git a/plugins/codex-security/mcp-app/tests/deep_scan_coordinator_fixture.ts b/plugins/codex-security/mcp-app/tests/deep_scan_coordinator_fixture.ts index e55fadcd82..c428b0170e 100644 --- a/plugins/codex-security/mcp-app/tests/deep_scan_coordinator_fixture.ts +++ b/plugins/codex-security/mcp-app/tests/deep_scan_coordinator_fixture.ts @@ -88,10 +88,11 @@ export async function fixtureRun( export type TestWorker = PersistedDeepScanWorker & Pick; export type StoreInput = Parameters< - DeepScanStore[Method] + NonNullable >[0]; export class FakeStore { + declare selectFinalization: DeepScanStore["selectFinalization"]; run: DeepScanRunState; declare failProgressAt: number | undefined; constructor(run: DeepScanRunState) { diff --git a/plugins/codex-security/mcp-app/tests/deep_scan_coverage_fixture.mjs b/plugins/codex-security/mcp-app/tests/deep_scan_coverage_fixture.mjs new file mode 100644 index 0000000000..4de988117c --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/deep_scan_coverage_fixture.mjs @@ -0,0 +1,606 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { build } from "esbuild"; + +const pluginRoot = fileURLToPath(new URL("../../", import.meta.url)); +const exec = promisify(execFile); +const bundled = await build({ + bundle: true, + stdin: { + contents: [ + 'export { DeepScanCoordinator } from "./src/deep-scan/coordinator.ts";', + 'export { createDeepScanArtifacts } from "./src/deep-scan/artifacts.ts";', + 'export { validateDiscoveryArtifacts } from "./src/deep-scan/artifact-validation.ts";', + 'export { WorkbenchDeepScanStore } from "./src/deep-scan/store.ts";', + 'export { createScanArtifactContext } from "./src/artifact-context.ts";', + 'export { recordCodexSecurityScanDraftViaWorkbench, saveScanDraftCheckpoint } from "./src/artifact-scan-draft.ts";', + 'export { recordCodexSecurityDeepReduction, getCodexSecurityDeepReducerInputs } from "./src/artifact-deep-reducer.ts";', + ].join("\n"), + resolveDir: path.join(pluginRoot, "mcp-app"), + }, + format: "esm", + platform: "node", + loader: { ".md": "text" }, + write: false, +}); +export async function publishCoverageFixture( + root, + completeness, + { + resume = false, + continueAfterResume = false, + immutableInputs = false, + materialFindings = false, + discardMutableResults = false, + legacyAttempts = false, + splitSeededReducers = false, + selectedRecovery = false, + sourceProvenance, + } = {}, +) { + const runtimePath = path.join(root, "fixture-runtime.mjs"); + await writeFile(runtimePath, bundled.outputFiles[0].contents); + const { + DeepScanCoordinator, + createDeepScanArtifacts, + validateDiscoveryArtifacts, + WorkbenchDeepScanStore, + createScanArtifactContext, + recordCodexSecurityScanDraftViaWorkbench, + recordCodexSecurityDeepReduction, + getCodexSecurityDeepReducerInputs, + saveScanDraftCheckpoint, + } = await import(pathToFileURL(runtimePath).href); + const targetPath = path.join(root, "target"); + const codexHome = path.join(root, "codex-home"); + const scanRoot = path.join(root, "scans"); + const threadId = "coverage-fixture-owner"; + const statuses = + completeness === "partial" + ? ["partial", "complete", "unknown"] + : completeness === "unknown" + ? ["unknown", "complete"] + : ["complete"]; + await mkdir(scanRoot, { mode: 0o700 }); + await mkdir(targetPath, { recursive: true }); + await mkdir(path.join(codexHome, "codex-security"), { recursive: true }); + await writeFile(path.join(targetPath, "source.py"), "# Synthetic source\n"); + await writeFile( + path.join(codexHome, "codex-security", "config.toml"), + `[deep_scan]\nworkers = 1\nsubagents = 0\nstop_after_no_new = ${statuses.length}\nmax_discovery_runs = ${statuses.length}\n`, + ); + const runWorkbench = async (args, input, selectFinalization = false) => { + const script = path.join(pluginRoot, "scripts", "workbench_db.py"); + const pythonArgs = selectFinalization + ? [ + "-c", + "import runpy, sys; script = sys.argv.pop(1); runpy.run_path(script)['main'](select_finalization=True)", + script, + ...args, + ] + : [script, ...args]; + const execution = exec(process.env.PYTHON || "python3", pythonArgs, { + env: { + ...process.env, + CODEX_HOME: codexHome, + CODEX_SECURITY_STATE_DIR: path.join(root, "state"), + }, + }); + if (input !== undefined) execution.child.stdin.end(input); + const { stdout } = await execution; + return JSON.parse(stdout); + }; + const store = new WorkbenchDeepScanStore(runWorkbench); + let run = await store.begin({ targetPath, scope: ".", threadId, scanRoot }); + assert.equal( + run.workflowVersion, + "deep-security-scan/v2", + "new scans use persisted finalization", + ); + if (selectedRecovery) { + ({ run } = await store.claimCoordinator({ scanId: run.scanId, threadId })); + } else { + // Seed an existing v1 run for legacy direct publication and in-memory coverage recovery. + await exec(process.env.PYTHON || "python3", [ + "-c", + [ + "import sqlite3, sys", + "with sqlite3.connect(sys.argv[1]) as db:", + " db.execute(\"UPDATE deep_scan_runs SET workflow_version = 'deep-scan-mcp/v1' WHERE scan_id = ?\", (sys.argv[2],))", + ].join("\n"), + path.join(root, "state", "workbench.sqlite3"), + run.scanId, + ]); + run = await store.get(run.scanId, threadId); + assert.equal(run.workflowVersion, "deep-scan-mcp/v1"); + } + const context = await createScanArtifactContext(run.scanId, runWorkbench, { + requireRunning: true, + }); + const rawSources = new Map(); + const writeReduction = async (context) => { + const inputs = await getCodexSecurityDeepReducerInputs(context); + const sources = [ + ...(inputs.previous?.findings ?? []), + ...inputs.discoveries.flatMap((source) => source.result.findings), + ]; + const findings = []; + if (sources.length) { + const finding = structuredClone(sources[0]); + finding.provenance.sourceFindingIds = [ + ...new Set( + sources.flatMap((source) => source.provenance.sourceFindingIds), + ), + ]; + delete finding.provenance.sourceFindings; + findings.push(finding); + } + await recordCodexSecurityDeepReduction(context, { + scanId: run.scanId, + findings, + }); + }; + const writeDiscovery = async (artifactDir, index) => { + const status = statuses[index]; + const pending = completeness === "partial" && status !== "complete"; + const coverage = { + completeness: status, + surfaces: [ + { + id: "shared-surface", + label: "Archive route", + disposition: pending ? "needs_follow_up" : "no_issue_found", + receiptRefs: ["artifacts/review.md"], + }, + ], + explicitExclusions: [ + { pattern: "vendor/", reason: "External dependency." }, + ], + deferred: pending + ? [ + { + id: "same-id", + candidateId: "candidate-1", + reason: + index === 0 + ? "Verify entry boundaries." + : "Verify symbolic links.", + paths: ["source.py"], + surfaceIds: ["shared-surface"], + }, + ] + : [], + openQuestions: pending + ? [{ question: `Deployment question ${index + 1}.` }] + : [], + }; + if (sourceProvenance !== undefined) { + for (const field of [ + "surfaces", + "explicitExclusions", + "deferred", + "openQuestions", + ]) { + for (const item of coverage[field]) + item.provenance = structuredClone(sourceProvenance); + } + } + await mkdir(path.join(artifactDir, "artifacts"), { recursive: true }); + await writeFile( + path.join(artifactDir, "artifacts", "review.md"), + "Synthetic review evidence.\n", + ); + const resultPath = path.join(artifactDir, "result.json"); + const findings = + materialFindings && index < 2 + ? [ + { + ruleId: "archive-extraction", + identity: { anchor: "archive-destination" }, + title: "Archive entries can escape the destination", + summary: + "Archive extraction requires both entry containment and symbolic-link handling.", + severity: { level: "high" }, + confidence: { + level: "high", + rationale: "Synthetic accepted source evidence.", + }, + taxonomy: { category: "path-traversal", cwe: ["CWE-22"] }, + locations: [{ path: "source.py", startLine: 1, endLine: 1 }], + remediation: materialRemediations[index], + remediationTests: [materialRemediationTests[index]], + provenance: { source: "local_plugin" }, + }, + ] + : []; + const bytes = JSON.stringify({ + scanId: run.scanId, + complete: true, + findings, + coverage, + }); + await writeFile(resultPath, bytes); + rawSources.set(resultPath, bytes); + if (immutableInputs) { + await saveScanDraftCheckpoint( + { root: artifactDir, repoRoot: targetPath, layout: "worker" }, + JSON.parse(bytes), + ); + const head = JSON.parse( + await readFile(path.join(artifactDir, "checkpoint-head.json"), "utf8"), + ); + const acceptedPath = path.join( + artifactDir, + "checkpoints", + head.checkpoint, + ); + rawSources.set(acceptedPath, await readFile(acceptedPath, "utf8")); + return acceptedPath; + } + return resultPath; + }; + if (resume) { + const workers = []; + const seeded = continueAfterResume ? statuses.slice(0, -1) : statuses; + for (const index of seeded.keys()) { + const workerRoot = path.join( + run.scanDir, + "artifacts", + "deep_discovery", + "workers", + `discovery-${String(index + 1).padStart(4, "0")}`, + ); + const artifactDir = path.join(workerRoot, "output"); + const worker = { + id: randomUUID(), + scanId: run.scanId, + kind: "discovery", + promptPath: path.join(workerRoot, "prompt.md"), + artifactDir, + attempt: index === 0 ? 2 : 1, + }; + const writtenPath = await writeDiscovery(artifactDir, index); + const resultManifestPath = discardMutableResults + ? path.join(artifactDir, "result.json") + : writtenPath; + await writeFile(worker.promptPath, "Synthetic discovery prompt.\n"); + for (const status of ["queued", "running", "succeeded"]) { + await store.updateWorker({ + ...worker, + status, + ...(status === "succeeded" ? { resultManifestPath } : {}), + }); + } + workers.push({ ...worker, resultPath: resultManifestPath }); + } + const batches = splitSeededReducers + ? [workers.slice(0, 2), workers.slice(2)] + : [workers]; + let lastReducerId; + let lastReducerReference; + for (const [index, batch] of batches.entries()) { + const label = `dedup-${String(index + 1).padStart(4, "0")}`; + const artifactDir = path.join( + run.scanDir, + "artifacts", + "deep_discovery", + "dedup", + label, + "output", + ); + const promptPath = path.join(path.dirname(artifactDir), "prompt.md"); + await mkdir(artifactDir, { recursive: true }); + await writeFile(promptPath, "Synthetic reducer prompt.\n"); + const id = randomUUID(); + const claimed = await store.claimDedup({ + id, + scanId: run.scanId, + workerIds: batch.map((worker) => worker.id), + artifactDir, + promptPath, + }); + if (discardMutableResults) { + await store.updateWorker({ + id, + scanId: run.scanId, + kind: "dedup", + status: "running", + artifactDir, + promptPath, + attempt: 1, + }); + } + const resultManifestPath = path.join(artifactDir, "result.json"); + // Legacy accepted reducers omitted coverage entirely. + if (materialFindings) { + await writeReduction({ + root: artifactDir, + repoRoot: targetPath, + scanId: run.scanId, + layout: "reducer", + deepReducer: { + scanRoot: run.scanDir, + claimedWorkers: batch.map((worker) => { + const input = claimed.persistedDedupInputs.find( + (input) => + input.dedupWorkerId === id && + input.discoveryWorkerId === worker.id, + ); + return { + ...worker, + resultPath: input.resultManifestPath ?? worker.resultPath, + attempt: input.attempt ?? worker.attempt, + }; + }), + persistSourceCoverage: selectedRecovery, + previousReducerResultPath: claimed.persistedMergeClaims?.find( + (claim) => claim.workerId === id, + )?.previousResultPath, + }, + }); + } else { + await writeFile( + resultManifestPath, + JSON.stringify({ scanId: run.scanId, findings: [] }), + ); + } + rawSources.set( + resultManifestPath, + await readFile(resultManifestPath, "utf8"), + ); + const committed = await store.commitDedup({ + id, + scanId: run.scanId, + newFindings: materialFindings && index === 0 ? 1 : 0, + resultManifestPath, + }); + lastReducerReference = committed.committedMerge.resultManifestPath; + lastReducerId = id; + } + if (legacyAttempts) { + // Migrated discoveries and prior reducers can have frozen claims without attempt rows. + await exec(process.env.PYTHON || "python3", [ + "-c", + [ + "import sqlite3, sys", + "with sqlite3.connect(sys.argv[1]) as db:", + ' db.execute("DELETE FROM deep_scan_attempts WHERE worker_id != ?", (sys.argv[2],))', + ].join("\n"), + path.join(root, "state", "workbench.sqlite3"), + lastReducerId, + ]); + } + run = await store.get(run.scanId, threadId); + if (discardMutableResults) { + for (const worker of run.persistedWorkers) { + const acceptedPath = + worker.acceptedResultPath ?? + run.persistedDedupInputs.find( + (input) => input.discoveryWorkerId === worker.id, + )?.resultManifestPath ?? + run.persistedMergeClaims.find( + (claim) => claim.previousWorkerId === worker.id, + )?.previousResultPath; + assert.ok(acceptedPath, "the real store retains an accepted reference"); + assert.notEqual(acceptedPath, worker.resultManifestPath); + rawSources.set(acceptedPath, await readFile(acceptedPath, "utf8")); + rawSources.delete(worker.resultManifestPath); + await rm(worker.resultManifestPath); + } + } + if (selectedRecovery) { + run = await store.selectFinalization({ + scanId: run.scanId, + reason: "capped", + manifestPath: path.join(run.scanDir, "scan-manifest.json"), + resultPath: lastReducerReference, + omittedWorkerIds: [], + }); + } + } + let discoveryCalls = 0; + const executor = { + async run(request) { + assert.equal( + resume && !continueAfterResume, + false, + "accepted legacy sources should resume without new model work", + ); + const thread = request.resumeThreadId ?? randomUUID(); + await request.onThreadStarted?.(thread); + if (request.kind === "discovery") { + discoveryCalls++; + const index = + Number( + path.basename(path.dirname(request.promptPath)).split("-").at(-1), + ) - 1; + if (index === 0 && !request.resumeThreadId) + return { + threadId: thread, + finalResponse: "Continue the unfinished audit.", + }; + await writeDiscovery(request.artifactContext.root, index); + } else { + if (immutableInputs) { + const current = await store.get(run.scanId, threadId); + for (const claimed of request.artifactContext.deepReducer + .claimedWorkers) { + const accepted = current.persistedWorkers.find( + (worker) => worker.id === claimed.id, + ); + assert.equal( + claimed.resultPath, + accepted.acceptedResultPath ?? accepted.resultManifestPath, + "the reducer uses the exact accepted input", + ); + assert.equal( + claimed.artifactDir, + accepted.artifactDir, + "receipts retain their original output owner", + ); + } + } + await writeReduction({ + ...request.artifactContext, + repoRoot: targetPath, + scanId: run.scanId, + }); + } + return { threadId: thread, finalResponse: "Audit finished." }; + }, + }; + let publicationCalls = 0; + const options = { + run, + store, + executor, + pluginRoot, + retryDelaysMs: [1], + onComplete: async (draft, signal, publication) => { + publicationCalls++; + if (selectedRecovery && publicationCalls === 1) + throw new Error("Synthetic selected publication failure"); + await recordCodexSecurityScanDraftViaWorkbench( + context, + draft, + runWorkbench, + signal, + selectedRecovery ? publication : undefined, + ); + }, + }; + const coordinator = new DeepScanCoordinator(options); + coordinator.start(); + let terminal; + if (selectedRecovery) { + await assert.rejects( + coordinator.wait(undefined, 30_000), + /Synthetic selected publication failure/, + ); + const pending = await store.get(run.scanId, threadId); + assert.equal(pending.status, "running"); + assert.deepEqual(pending.finalizationInput, run.finalizationInput); + const worker = pending.persistedWorkers.find( + (worker) => worker.kind === "discovery", + ); + const rejected = { + scanId: run.scanId, + complete: false, + findings: [], + coverage: { + completeness: "complete", + surfaces: [], + explicitExclusions: [], + deferred: [], + }, + }; + await saveScanDraftCheckpoint( + { root: worker.artifactDir, repoRoot: targetPath, layout: "worker" }, + rejected, + ); + const replacement = path.join(worker.artifactDir, "result.json"); + await writeFile(replacement, JSON.stringify(rejected)); + await assert.rejects( + validateDiscoveryArtifacts( + createDeepScanArtifacts(run.scanDir), + replacement, + run.scanId, + ), + /only a checkpoint/, + ); + const headPath = path.join(worker.artifactDir, "checkpoint-head.json"); + const head = JSON.parse(await readFile(headPath, "utf8")); + for (const file of [ + replacement, + headPath, + path.join(worker.artifactDir, "checkpoints", head.checkpoint), + ]) { + rawSources.set(file, await readFile(file, "utf8")); + } + const restarted = new DeepScanCoordinator({ ...options, run: pending }); + restarted.start(); + terminal = await restarted.wait(undefined, 30_000); + assert.deepEqual(terminal.finalizationInput, run.finalizationInput); + assert.equal(publicationCalls, 2); + } else { + terminal = await coordinator.wait(undefined, 30_000); + } + assert.equal(terminal?.status, "succeeded", terminal?.error); + assert.equal( + terminal.noNewStreak, + materialFindings + ? resume && !continueAfterResume && !splitSeededReducers + ? 0 + : 1 + : statuses.length, + "source coverage must not change stopping policy", + ); + assert.equal( + discoveryCalls, + resume ? (continueAfterResume ? 1 : 0) : statuses.length + 1, + ); + const accepted = await store.get(run.scanId, threadId); + for (const worker of accepted.persistedWorkers.filter( + (worker) => worker.kind === "dedup", + )) { + const resultPath = + worker.acceptedResultPath ?? + accepted.persistedMergeClaims.find( + (claim) => claim.previousWorkerId === worker.id, + )?.previousResultPath ?? + worker.resultManifestPath; + const result = JSON.parse(await readFile(resultPath, "utf8")); + assert.equal( + Object.hasOwn(result, "sourceCoverage"), + selectedRecovery, + "coverage persistence follows the accepted workflow version", + ); + if (!rawSources.has(worker.resultManifestPath)) { + for (const name of await readdir( + path.join(worker.artifactDir, "checkpoints"), + )) { + const checkpoint = JSON.parse( + await readFile( + path.join(worker.artifactDir, "checkpoints", name), + "utf8", + ), + ); + assert.equal( + Object.hasOwn(checkpoint, "sourceCoverage"), + selectedRecovery, + "checkpoint coverage follows the accepted workflow version", + ); + } + } + } + await runWorkbench(["complete-scan", "--scan-id", run.scanId]); + for (const [file, bytes] of rawSources) + assert.equal(await readFile(file, "utf8"), bytes); + return { scanDir: run.scanDir, threadId, terminal }; +} + +export const materialRemediations = [ + "Check the destination before writing the archive entry.", + "Reject symbolic links before opening the destination.", +]; +export const materialRemediationTests = [ + "Reject an archive entry outside the destination.", + "Reject a symbolic link inside the destination.", +]; + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const result = await publishCoverageFixture( + process.argv[2], + process.argv[3], + { + resume: process.argv[4] === "true", + continueAfterResume: process.argv[5] === "true", + }, + ); + process.stdout.write(JSON.stringify(result)); +} diff --git a/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_bank.mjs b/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_bank.mjs new file mode 100644 index 0000000000..0fed15b231 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_bank.mjs @@ -0,0 +1,180 @@ +// Synthetic accepted artifacts. Expected fixes are independent of display identities. +export const scanId = "7fc17317-9594-49e0-b06a-d72fd7e14bba"; +export const bankVersion = "accepted-source-bank/v1"; + +export const fixes = { + owner: "Check document ownership before returning document contents.", + path: "Resolve archive paths and reject entries outside the extraction directory.", + sql: "Bind the search term as a SQL parameter.", + html: "HTML-encode the search term before rendering the response.", +}; + +function finding(id, remediation, extra = {}) { + return { + ruleId: "synthetic." + id, + identity: { anchor: id }, + title: "Request boundary " + id, + summary: "A request-controlled value crosses an unchecked boundary.", + severity: { level: "high" }, + confidence: { + level: "high", + rationale: "A synthetic local test reaches the sink.", + }, + taxonomy: { category: "input-validation", cwe: ["CWE-20"] }, + locations: [{ path: "src/routes.py", startLine: 12, endLine: 14 }], + remediation, + provenance: { source: "local_plugin" }, + ...extra, + }; +} + +function coverage(completeness = "complete", deferred = []) { + return { completeness, surfaces: [], explicitExclusions: [], deferred }; +} + +function worker(id, findings, scanCoverage = coverage()) { + return { + id, + result: { scanId, complete: true, findings, coverage: scanCoverage }, + }; +} + +export const workers = [ + worker( + "worker-owner", + [ + finding("owner", fixes.owner, { + summary: + "An authenticated user can read another user's document by changing its ID.", + validation: { + summary: "The ownership check is absent; authentication is required.", + }, + }), + ], + coverage("partial", [ + { + candidateId: "candidate-1", + reason: "Check the alternate document handler.", + paths: ["src/alternate.py"], + }, + ]), + ), + worker( + "worker-path", + [finding("path", fixes.path)], + coverage("partial", [ + { + candidateId: "candidate-1", + reason: "Check symlink extraction separately.", + paths: ["src/archive.py"], + }, + ]), + ), + worker("worker-owner-duplicate", [ + finding("owner-copy", fixes.owner, { + summary: "Document contents may be reachable without authentication.", + severity: { level: "critical" }, + confidence: { + level: "low", + rationale: "Authentication middleware was not examined.", + }, + validation: { + summary: + "The unauthenticated claim is untested; ownership check is absent.", + }, + }), + ]), + worker("worker-bundled", [ + finding("search-bundle", fixes.sql + " " + fixes.html, { + summary: + "The search route interpolates the query into SQL and separately into HTML.", + locations: [{ path: "src/search.py", startLine: 5, endLine: 9 }], + }), + ]), +]; + +export const sourceGroups = { + "worker-owner:0": "owner", + "worker-path:0": "path", + "worker-owner-duplicate:0": "owner", + "worker-bundled:0": "search-bundle", +}; + +export const sourceFixes = { + "worker-owner:0": ["owner"], + "worker-path:0": ["path"], + "worker-owner-duplicate:0": ["owner"], + "worker-bundled:0": ["sql", "html"], +}; + +// This history is distinct from the immutable accepted terminal bank above. +// A newer rejection is authoritative for this logical worker's final result. +export const rejectionHistory = { + workerId: "worker-rejected", + earlier: worker("worker-rejected", [finding("safe-query", fixes.sql)]).result, + latest: worker("worker-rejected", [], { + ...coverage(), + surfaces: [ + { + label: "Search SQL", + disposition: "rejected", + notes: + "The driver binds parameters; the earlier interpolation claim was disproved.", + }, + ], + }).result, +}; + +export function permutations(items) { + if (!items.length) return [[]]; + return items.flatMap((item, index) => + permutations(items.filter((_, i) => i !== index)).map((rest) => [ + item, + ...rest, + ]), + ); +} + +export function partitions(items) { + if (!items.length) return [[]]; + return items.flatMap((_, index) => + partitions(items.slice(index + 1)).map((rest) => [ + items.slice(0, index + 1), + ...rest, + ]), + ); +} + +export function originals(inputs) { + const sources = new Map(); + for (const current of inputs.previous?.findings ?? []) { + for (const source of current.provenance.sourceFindings ?? []) + sources.set(source.id, source.finding); + } + for (const discovery of inputs.discoveries) { + discovery.result.findings.forEach((value, i) => + sources.set(`${discovery.workerId}:${i}`, value), + ); + } + return sources; +} + +// Scripted proposals isolate host reconciliation from model variability. +export function proposal(inputs, groupFor = (id) => sourceGroups[id]) { + const grouped = new Map(); + for (const [id, value] of originals(inputs)) { + const group = groupFor(id); + if (!grouped.has(group)) grouped.set(group, { value, refs: [] }); + grouped.get(group).refs.push(id); + } + return { + scanId, + complete: true, + findings: [...grouped].map(([group, { value, refs }]) => ({ + ...structuredClone(value), + ruleId: "synthetic." + group, + identity: { anchor: group }, + provenance: { source: "local_plugin", sourceFindingIds: refs }, + })), + }; +} diff --git a/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_replay.mjs b/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_replay.mjs new file mode 100644 index 0000000000..26b6a61632 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_replay.mjs @@ -0,0 +1,105 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { build } from "esbuild"; +import { proposal, scanId, workers } from "./accepted_source_bank.mjs"; + +export async function loadReducer() { + const bundled = await build({ + bundle: true, + entryPoints: [ + fileURLToPath( + new URL("../../src/artifact-deep-reducer.ts", import.meta.url), + ), + ], + format: "esm", + platform: "node", + write: false, + }); + return import( + "data:text/javascript;base64," + + Buffer.from(bundled.outputFiles[0].contents).toString("base64") + ); +} + +export async function prepareReplay(root, bank = workers) { + const workerRoot = path.join(root, "artifacts", "deep_discovery", "workers"); + const records = new Map(); + for (const [i, worker] of bank.entries()) { + const output = path.join(workerRoot, worker.id, "output"); + await mkdir(output, { recursive: true }); + const resultPath = path.join(output, "result.json"); + await writeFile(resultPath, JSON.stringify(worker.result) + "\n"); + records.set(worker.id, { + id: worker.id, + resultPath, + completionSequence: i + 1, + }); + } + return records; +} + +export async function runBatch( + reducer, + root, + records, + ids, + index, + previous, + propose = proposal, +) { + const output = path.join( + root, + "artifacts", + "deep_discovery", + "dedup", + `dedup-${index}`, + "output", + ); + await mkdir(output, { recursive: true }); + const context = { + root: output, + repoRoot: root, + scanId, + layout: "reducer", + deepReducer: { + scanRoot: root, + claimedWorkers: ids.map((id) => records.get(id)), + ...(previous ? { previousReducerResultPath: previous } : {}), + }, + }; + const inputs = await reducer.getCodexSecurityDeepReducerInputs(context); + const submitted = propose(inputs); + const receipt = await reducer.recordCodexSecurityDeepReduction( + context, + submitted, + ); + const resultPath = path.join(output, "result.json"); + return { + inputs, + submitted, + receipt, + resultPath, + result: JSON.parse(await readFile(resultPath, "utf8")), + }; +} + +export async function replay(reducer, root, batches, propose = proposal) { + const records = await prepareReplay(root); + const steps = []; + let previous; + for (const [index, batch] of batches.entries()) { + const step = await runBatch( + reducer, + root, + records, + batch, + index, + previous, + propose, + ); + steps.push(step); + previous = step.resultPath; + } + return { steps, result: steps.at(-1).result }; +} diff --git a/plugins/codex-security/mcp-app/tests/test_accepted_source_replay.mjs b/plugins/codex-security/mcp-app/tests/test_accepted_source_replay.mjs new file mode 100644 index 0000000000..2b87097731 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_accepted_source_replay.mjs @@ -0,0 +1,69 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, realpath, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { + partitions, + permutations, + sourceFixes, + workers, +} from "./fixtures/accepted_source_bank.mjs"; +import { loadReducer, replay } from "./fixtures/accepted_source_replay.mjs"; + +test("accepted sources survive completion orders and eligible batch partitions", async () => { + const reducer = await loadReducer(); + const root = await realpath( + await mkdtemp(path.join(tmpdir(), "accepted-source-replay-")), + ); + let count = 0; + try { + for (const order of permutations(workers.map((worker) => worker.id))) { + for (const batches of partitions(order)) { + // Ordinary rolling runs require two successes for the first merge. + if (batches[0].length < 2) continue; + const runRoot = path.join(root, String(count++)); + const { result, steps } = await replay(reducer, runRoot, batches); + assert.equal(result.findings.length, 3); + const refs = result.findings.flatMap( + (finding) => finding.provenance.sourceFindingIds, + ); + assert.deepEqual(refs.toSorted(), Object.keys(sourceFixes).toSorted()); + for (const step of steps) { + assert.deepEqual( + step.receipt.consumedWorkerIds, + batches[steps.indexOf(step)], + ); + } + for (const finding of result.findings) { + for (const source of finding.provenance.sourceFindings) { + const [workerId, index] = source.id.split(":"); + const original = workers.find((worker) => worker.id === workerId) + .result.findings[Number(index)]; + assert.deepEqual(source.finding, original); + } + } + for (const worker of workers) { + const persisted = JSON.parse( + await readFile( + path.join( + runRoot, + "artifacts", + "deep_discovery", + "workers", + worker.id, + "output", + "result.json", + ), + "utf8", + ), + ); + assert.deepEqual(persisted, worker.result); + } + } + } + assert.equal(count, 96); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_deep_reducer.ts b/plugins/codex-security/mcp-app/tests/test_artifact_deep_reducer.ts index 9d6cf90ec3..6c43799b99 100644 --- a/plugins/codex-security/mcp-app/tests/test_artifact_deep_reducer.ts +++ b/plugins/codex-security/mcp-app/tests/test_artifact_deep_reducer.ts @@ -8,15 +8,35 @@ import path from "node:path"; import { importSource } from "./import-module.ts"; type Finding = ReturnType; +type SourceCoverage = + import("../src/artifact-scan-draft.js").ScanDraftInput["coverage"] & { + reviews: unknown[]; + deferred: { provenance: unknown }[]; + surfaces: { receiptRefs: unknown[] }[]; + }; const { deepReducerInputsInputSchema, deepReductionInputSchema, - getCodexSecurityDeepReducerInputs, + getCodexSecurityDeepReducerInputs: getModelInputs, + readDeepReductionSources, recordCodexSecurityDeepReduction, -} = await importSource( +} = (await importSource( new URL("../src/artifact-deep-reducer.ts", import.meta.url).pathname, -); +)) as typeof import("../src/artifact-deep-reducer.js"); + +async function getCodexSecurityDeepReducerInputs( + context: Parameters[0], +) { + return (await readDeepReductionSources(context)) as Omit< + Awaited>, + "discoveries" + > & { + discoveries: (Awaited< + ReturnType + >["discoveries"][number] & { coverage: SourceCoverage })[]; + }; +} const validReduction = reduction([]); @@ -119,9 +139,10 @@ try { root: outputRoot, repoRoot: root, scanId, - layout: "reducer", + layout: "reducer" as const, deepReducer: { scanRoot, + persistSourceCoverage: true, claimedWorkers: [first, second], }, }; @@ -166,13 +187,39 @@ try { /evidenceRefs must refer/, "live reducer submissions reject unknown evidence references instead of silently removing them", ); - assert.deepEqual(inputs, { - discoveries: [ - { workerId: first.id, result: withSourceRefs(first) }, - { workerId: second.id, result: withSourceRefs(second) }, - ], - previous: null, + assert.deepEqual( + { + ...inputs, + discoveries: inputs.discoveries.map(({ coverage, ...source }) => source), + }, + { + discoveries: [ + { workerId: first.id, result: withSourceRefs(first) }, + { workerId: second.id, result: withSourceRefs(second) }, + ], + previous: null, + }, + ); + assert.equal(inputs.discoveries[0]!.coverage!.completeness, "partial"); + assert.equal(inputs.discoveries[1]!.coverage!.completeness, "unknown"); + assert.deepEqual( + await getModelInputs(context), + { + discoveries: inputs.discoveries.map(({ workerId, result }) => ({ + workerId, + result, + })), + previous: null, + }, + "coverage accounting does not change reducer model inputs", + ); + assert.deepEqual(inputs.discoveries[0]!.coverage!.deferred[0].provenance, { + workerId: first.id, + candidateId: "candidate-upload", }); + assert.deepEqual(inputs.discoveries[0]!.coverage!.surfaces[0].receiptRefs, [ + "artifacts/deep_discovery/workers/discovery-0001/output/artifacts/missing-worker-receipt.md", + ]); assert.equal(JSON.stringify(inputs).includes(root), false); assert.equal(JSON.stringify(inputs).includes("result.json"), false); @@ -196,6 +243,13 @@ try { const outcome = await recordCodexSecurityDeepReduction(context, merged); const mergedWithSources = { ...merged, + sourceCoverage: { + ...inputs.discoveries[0]!.coverage!, + reviews: [ + ...inputs.discoveries[0]!.coverage!.reviews, + ...inputs.discoveries[1]!.coverage!.reviews, + ], + }, findings: [ retainedFinding(shared, [ { id: "worker-001:0", finding: shared }, @@ -219,7 +273,7 @@ try { assert.deepEqual( await readJson(outputRoot, "checkpoints", checkpointNames[0]), mergedWithSources, - "reducer checkpoints retain the accepted findings and scope without coverage", + "reducer checkpoints retain accepted findings, scope and source coverage", ); assert.deepEqual( @@ -256,8 +310,8 @@ try { await getCodexSecurityDeepReducerInputs(collisionContext); const sourceFindingIds = collisionInputs.discoveries[0].result.findings.flatMap( - (finding: { provenance: { sourceFindingIds: string[] } }) => - finding.provenance.sourceFindingIds, + (finding) => + (finding.provenance as { sourceFindingIds: string[] }).sourceFindingIds, ); assert.deepEqual(sourceFindingIds, [ "worker-collision:0", @@ -316,18 +370,33 @@ try { root: nextOutputRoot, repoRoot: root, scanId, - layout: "reducer", + layout: "reducer" as const, deepReducer: { scanRoot, + persistSourceCoverage: true, claimedWorkers: [third], previousReducerResultPath: path.join(outputRoot, "result.json"), }, }; const nextInputs = await getCodexSecurityDeepReducerInputs(nextContext); - assert.deepEqual(nextInputs, { - discoveries: [{ workerId: third.id, result: withSourceRefs(third) }], - previous: mergedWithSources, - }); + const { sourceCoverage, ...previousModelInput } = mergedWithSources; + assert.deepEqual( + (await getModelInputs(nextContext)).previous, + previousModelInput, + "host coverage metadata is excluded from the previous model input too", + ); + assert.deepEqual( + { + ...nextInputs, + discoveries: nextInputs.discoveries.map( + ({ coverage, ...source }) => source, + ), + }, + { + discoveries: [{ workerId: third.id, result: withSourceRefs(third) }], + previous: mergedWithSources, + }, + ); await assert.rejects( recordCodexSecurityDeepReduction(nextContext, reduction([shared])), (error: NodeJS.ErrnoException) => @@ -337,17 +406,29 @@ try { await recordCodexSecurityDeepReduction(nextContext, merged), { findingCount: 2, consumedWorkerIds: [third.id] }, ); - assert.deepEqual(await readJson(nextOutputRoot, "result.json"), { - ...mergedWithSources, - findings: [ - retainedFinding(shared, [ - { id: "worker-003:0", finding: shared }, - { id: "worker-001:0", finding: shared }, - { id: "worker-002:0", finding: shared }, - ]), - mergedWithSources.findings[1], - ], - }); + assert.deepEqual( + JSON.parse( + await readFile(path.join(nextOutputRoot, "result.json"), "utf8"), + ), + { + ...mergedWithSources, + sourceCoverage: { + ...mergedWithSources.sourceCoverage, + reviews: [ + ...(mergedWithSources.sourceCoverage as SourceCoverage).reviews, + ...nextInputs.discoveries[0]!.coverage!.reviews, + ], + }, + findings: [ + retainedFinding(shared, [ + { id: "worker-003:0", finding: shared }, + { id: "worker-001:0", finding: shared }, + { id: "worker-002:0", finding: shared }, + ]), + mergedWithSources.findings[1], + ], + }, + ); const enrichedPrevious = structuredClone(mergedWithSources); enrichedPrevious.findings[0].summary = diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_scan_draft.ts b/plugins/codex-security/mcp-app/tests/test_artifact_scan_draft.ts index 9e690dd422..a2ce53fac9 100644 --- a/plugins/codex-security/mcp-app/tests/test_artifact_scan_draft.ts +++ b/plugins/codex-security/mcp-app/tests/test_artifact_scan_draft.ts @@ -838,33 +838,39 @@ try { "obsolete.json", ); await writeFile(obsoleteCheckpointPath, "{malformed obsolete checkpoint\n"); - const deepWorkbenchWrites = mock.fn(async (arguments_: string[]) => { - assert.deepEqual(arguments_.slice(0, 3), [ - "write-scan-draft", - "--scan-id", - scanId, - ]); - assert.equal(arguments_.includes("--expected-draft-digest"), false); - assert.deepEqual(arguments_.slice(-2), ["--claim-token", claimToken]); - const draftPath = arguments_[arguments_.indexOf("--draft-path") + 1]; - const checkpointPath = - arguments_[arguments_.indexOf("--checkpoint-path") + 1]; - const staged = await readJson(draftPath); - const stagedCheckpoint = await readJson(checkpointPath); - assert.deepEqual(staged.findings, acceptedDeepFindings); - assert.deepEqual(staged.coverage, acceptedDeepCoverage); - assert.deepEqual(stagedCheckpoint.findings, acceptedDeepDraft.findings); - assert.equal(stagedCheckpoint.handoffClaimToken, undefined); - }); + let deepWorkbenchWrites = 0; + const deepPublication = { + coordinatorGeneration: 3, + resultPath: path.join(deepParentRoot, "workers", "reducer", "result.json"), + }; await recordCodexSecurityScanDraftViaWorkbench( deepParentContext, acceptedDeepDraft, - deepWorkbenchWrites, - ); - assert.equal( - deepWorkbenchWrites.mock.callCount(), - 1, - "terminal Deep drafts still publish through the workbench lock despite obsolete malformed checkpoints", + async (arguments_: string[]) => { + deepWorkbenchWrites += 1; + assert.deepEqual(arguments_.slice(0, 3), [ + "write-scan-draft", + "--scan-id", + scanId, + ]); + assert.equal(arguments_.includes("--expected-draft-digest"), false); + assert.deepEqual(arguments_.slice(-2), ["--claim-token", claimToken]); + const draftPath = arguments_[arguments_.indexOf("--draft-path") + 1]; + const checkpointPath = + arguments_[arguments_.indexOf("--checkpoint-path") + 1]; + const staged = JSON.parse(await readFile(draftPath, "utf8")); + const stagedCheckpoint = JSON.parse( + await readFile(checkpointPath, "utf8"), + ); + assert.deepEqual(staged.deepScanPublication, deepPublication); + assert.equal(stagedCheckpoint.deepScanPublication, undefined); + assert.deepEqual(staged.findings, acceptedDeepFindings); + assert.deepEqual(staged.coverage, acceptedDeepCoverage); + assert.deepEqual(stagedCheckpoint.findings, acceptedDeepDraft.findings); + assert.equal(stagedCheckpoint.handoffClaimToken, undefined); + }, + undefined, + deepPublication, ); assert.deepEqual(await readdir(path.join(deepParentRoot, "drafts")), []); diff --git a/plugins/codex-security/mcp-app/tests/test_audit_acceptance_contract.mjs b/plugins/codex-security/mcp-app/tests/test_audit_acceptance_contract.mjs new file mode 100644 index 0000000000..48a58df1be --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_audit_acceptance_contract.mjs @@ -0,0 +1,255 @@ +import assert from "node:assert/strict"; +import { + mkdtemp, + mkdir, + readFile, + readdir, + realpath, + rm, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { build } from "esbuild"; + +const bundle = await build({ + stdin: { + contents: `export * from "./src/artifact-scan-draft.ts"; + export * from "./src/deep-scan/artifact-validation.ts"; + export * from "./src/deep-scan/artifacts.ts"; + export * from "../../../sdk/typescript/src/accepted-audit.ts";`, + resolveDir: path.resolve(import.meta.dirname, ".."), + }, + bundle: true, + format: "esm", + platform: "node", + write: false, + footer: { js: "//# sourceURL=audit-acceptance-contract.js" }, +}); +const { + createDeepScanArtifacts, + recordCodexSecurityScanDraft, + recordCodexSecurityWorkerScanDraft, + validateDiscoveryArtifacts, + readDiscoveryAuditDraft, + auditEvidence, + runAcceptedAudit, +} = await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}` +); + +const scanId = "811aef98-3709-4c2d-8b7a-742977521865"; +const finding = { + ruleId: "path-traversal.archive-extraction", + title: "Unsafe archive extraction", + summary: "An archive entry reaches a filesystem write.", + severity: { level: "high" }, + confidence: { level: "high", rationale: "Source review." }, + taxonomy: { category: "path-traversal", cwe: ["CWE-22"] }, + locations: [{ path: "src/extract.py", startLine: 4 }], + remediation: "Validate the resolved output path before writing.", + provenance: { source: "local_plugin", candidateId: "archive-entry" }, +}; + +for (const completeness of ["complete", "partial", "unknown"]) { + test(`Standard and Deep retain accepted semantic evidence with ${completeness} coverage`, async () => { + const root = await realpath( + await mkdtemp(path.join(tmpdir(), "audit-contract-")), + ); + try { + const repository = path.join(root, "repository"); + const scanDir = path.join(root, "scan"); + const artifacts = createDeepScanArtifacts(scanDir); + const workerRoot = path.join( + artifacts.workersRoot, + "discovery-0001", + "output", + ); + await Promise.all([ + mkdir(repository), + mkdir(workerRoot, { recursive: true }), + ]); + const semantic = { + scanId, + complete: true, + threatModel: { + summary: "An untrusted caller supplies archive entries.", + }, + findings: [finding], + coverage: { + completeness, + surfaces: [ + { + id: "archive", + label: "Archive extraction", + disposition: "reported", + receiptRefs: [], + }, + ], + explicitExclusions: [], + deferred: + completeness === "complete" + ? [] + : [ + { + id: "deployment-controls", + reason: "Deployment controls remain unverified.", + }, + ], + }, + }; + const standard = { + root: scanDir, + repoRoot: repository, + layout: "scan", + scanId, + mode: "standard", + status: "running", + scope: ".", + targetContract: { + target: { + allowedKinds: ["directory_snapshot"], + targetId: "target_example", + displayName: "example", + requiredSnapshotDigest: `codex-security-snapshot/v1:sha256:${"a".repeat(64)}`, + }, + scope: { requiredIncludePaths: ["."], requiredExcludePaths: [] }, + diffTarget: null, + }, + }; + const worker = { + root: workerRoot, + repoRoot: repository, + layout: "worker", + scanId, + }; + const checkpoint = { ...semantic, complete: false }; + await recordCodexSecurityScanDraft(standard, checkpoint); + await recordCodexSecurityWorkerScanDraft(worker, checkpoint); + await assert.rejects( + validateDiscoveryArtifacts( + artifacts, + path.join(workerRoot, "result.json"), + scanId, + ), + /checkpoint/, + ); + assert.equal( + JSON.parse(await readFile(path.join(scanDir, "scan-manifest.json"))) + .scan.complete, + false, + ); + const controller = new AbortController(); + const execute = async () => ({ + threadId: "audit-conversation", + usage: null, + }); + const accept = async () => + auditEvidence( + await readDiscoveryAuditDraft( + artifacts, + path.join(workerRoot, "result.json"), + scanId, + ), + ); + const unfinished = await runAcceptedAudit({ + signal: controller.signal, + execute, + accept, + }); + assert.equal(unfinished.status, "checkpoint"); + assert.equal(unfinished.checkpoint.complete, false); + assert.equal(unfinished.accepted, undefined); + assert.equal(unfinished.execution.usage, null); + const standardWrite = await recordCodexSecurityScanDraft( + standard, + semantic, + ); + const deepWrite = await recordCodexSecurityWorkerScanDraft( + worker, + semantic, + ); + assert.equal(standardWrite.status, "draft_written"); + assert.equal(deepWrite.status, "draft_written"); + const accepted = await validateDiscoveryArtifacts( + artifacts, + path.join(workerRoot, "result.json"), + scanId, + ); + const manifest = JSON.parse( + await readFile(path.join(scanDir, "scan-manifest.json")), + ); + const findings = JSON.parse( + await readFile(path.join(scanDir, "findings.json")), + ); + const coverage = JSON.parse( + await readFile(path.join(scanDir, "coverage.json")), + ); + assert.deepEqual(accepted.findings, [finding]); + for (const [key, value] of Object.entries(finding)) { + assert.deepEqual(findings.findings[0][key], value); + } + assert.ok(findings.findings[0].identity.anchor); + assert.deepEqual(accepted.threatModel, manifest.scan.threatModel); + for (const field of [ + "completeness", + "surfaces", + "explicitExclusions", + "deferred", + ]) { + assert.deepEqual(coverage[field], accepted.coverage[field]); + } + assert.equal(accepted.scanId, scanId); + const audit = await runAcceptedAudit({ + signal: controller.signal, + execute, + accept, + }); + assert.equal(audit.status, "accepted"); + assert.deepEqual(audit.accepted, accepted); + assert.deepEqual(audit.checkpoint, accepted); + const failure = new Error("Synthetic execution failure"); + await assert.rejects( + runAcceptedAudit({ + signal: controller.signal, + execute: async () => { + throw failure; + }, + accept: async () => { + assert.fail("An execution failure cannot accept old output"); + }, + }), + (error) => error === failure, + ); + await assert.rejects( + runAcceptedAudit({ + signal: controller.signal, + execute, + accept: async () => { + const evidence = await accept(); + controller.abort("user canceled"); + return evidence; + }, + }), + (error) => error === controller.signal.reason, + ); + assert.equal(manifest.scan.sealedAt, undefined); + assert.equal(manifest.scan.artifacts, undefined); + assert.equal( + (await readdir(workerRoot)).includes("scan-manifest.json"), + false, + ); + assert.equal((await readdir(scanDir)).includes("report.md"), false); + await assert.rejects( + validateDiscoveryArtifacts( + artifacts, + path.join(workerRoot, "result.json"), + "b4c84677-5aaf-410c-88d2-3e97e6f8c4d8", + ), + /scan/, + ); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_compact_artifact_server.ts b/plugins/codex-security/mcp-app/tests/test_compact_artifact_server.ts index 564fda8302..c89ecb73ba 100644 --- a/plugins/codex-security/mcp-app/tests/test_compact_artifact_server.ts +++ b/plugins/codex-security/mcp-app/tests/test_compact_artifact_server.ts @@ -19,7 +19,12 @@ type CompletedResult = { fingerprints: { primary: string }; })[]; }; - coverage: Record & { surfaces: { disposition: string }[] }; + coverage: Record & { + completeness: string; + surfaces: { disposition: string }[]; + deferred: unknown[]; + explicitExclusions: unknown[]; + }; }; type ToolResponse = Awaited>; type WorkspaceResult = { @@ -28,6 +33,8 @@ type WorkspaceResult = { type ScanResult = { scan: { progress: Record; + scanDir: string; + usage: unknown; reportAvailable: boolean; continuationThreadId?: string; handoffClaimToken?: string; @@ -267,8 +274,12 @@ async function testCompactDiffScanCompletion( }), `${runtimeLabel}: record compact diff canonical semantics`, ); + const completion = await call("complete_codex_security_scan", { + scanId, + handoffClaimToken, + }); requireSuccessfulTool( - await call("complete_codex_security_scan", { scanId, handoffClaimToken }), + completion, `${runtimeLabel}: complete compact diff scan`, ); const completed = requireSuccessfulTool( @@ -286,6 +297,12 @@ async function testCompactDiffScanCompletion( ); assert.equal(completed.coverage.inventoryStrategy, "diff"); assert.equal(completed.findings.findings.length, 0); + await assertCompletionCoverageAndReplay( + call, + { scanId, handoffClaimToken }, + completion, + completed.coverage, + ); } finally { await client.close(); } @@ -728,8 +745,12 @@ async function testSemanticScanDraftCompletion( unit: null, }); + const completion = await call("complete_codex_security_scan", { + scanId, + handoffClaimToken, + }); const completed = requireSuccessfulTool( - await call("complete_codex_security_scan", { scanId, handoffClaimToken }), + completion, `${runtimeLabel}: finalize the accepted draft exactly once`, ); assert.equal(completed.scan.progress.status, "complete"); @@ -772,6 +793,12 @@ async function testSemanticScanDraftCompletion( assert.equal(results.manifest.scan.target.kind, "directory_snapshot"); assert.equal(results.coverage.inventoryStrategy, "directory"); assert.equal(results.coverage.completeness, "partial"); + await assertCompletionCoverageAndReplay( + call, + { scanId, handoffClaimToken }, + completion, + results.coverage, + ); assert.deepEqual(results.coverage.includePaths, ["."]); assert.deepEqual(results.coverage.excludePaths, []); assert.equal(results.coverage.surfaces[0].disposition, "reported"); @@ -1063,6 +1090,61 @@ async function testClaimedParentArtifactOperations( } } +async function assertCompletionCoverageAndReplay( + call: ReturnType, + identity: { scanId: string; handoffClaimToken?: string }, + completion: ToolResponse, + coverage: CompletedResult["coverage"], +) { + const expected = { + completeness: coverage.completeness, + surfaceCount: coverage.surfaces.length, + deferredCount: coverage.deferred.length, + explicitExclusionCount: coverage.explicitExclusions.length, + }; + type Completion = ScanResult & { coverageSummary: typeof expected }; + const scan = requireSuccessfulTool(completion).scan; + const snapshot = () => + Promise.all( + ["scan-manifest.json", "findings.json", "coverage.json", "report.md"].map( + (name) => readFile(path.join(scan.scanDir, name), "utf8"), + ), + ); + const sealed = await snapshot(); + const replay = await call("complete_codex_security_scan", identity); + for (const result of [completion, replay]) { + const structured = requireSuccessfulTool(result); + assert.equal(structured.scan.progress.status, "complete"); + const content = (result.content as TextContent[])[0]; + assert.equal(content.type, "text"); + assert.match( + content.text, + new RegExp(`Canonical coverage: ${expected.completeness}\\b`), + ); + assert.match( + content.text, + new RegExp(`\\b${expected.surfaceCount} surfaces\\b`), + ); + assert.match( + content.text, + new RegExp(`\\b${expected.deferredCount} deferred items\\b`), + ); + assert.match( + content.text, + new RegExp( + `\\b${expected.explicitExclusionCount} explicit exclusions\\b`, + ), + ); + assert.deepEqual(structured.coverageSummary, expected); + assert.deepEqual(structured.scan.usage, scan.usage); + } + assert.deepEqual( + await snapshot(), + sealed, + "completion replay preserves sealed artifacts", + ); +} + function requireSuccessfulTool>( result: ToolResponse, label?: string, @@ -1298,6 +1380,18 @@ async function testDiscoveryWorkerToolList(bundle: string) { }, }; + for (const arguments_ of [ + { ...input, scope: { includePaths: ["src"], excludePaths: [] } }, + { ...input, coverage: { ...input.coverage, scanId } }, + ]) { + requireToolError( + await client.callTool({ name: tool.name, arguments: arguments_ }), + /expected never/, + "The draft description must not change rejection of workbench-owned metadata.", + ); + await assert.rejects(readFile(resultPath), { code: "ENOENT" }); + } + requireToolError( await client.callTool({ name: tool.name, diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.ts index 9a7ac50c05..9c324d60a1 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.ts +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.ts @@ -213,9 +213,20 @@ async function testReducerValidation(root: string) { ], previous: null, }; - const validateSnapshot = (reducerId = "dedup-0001", snapshot = sources) => + const validateSnapshot = ( + reducerId = "dedup-0001", + snapshot = sources, + persistSourceCoverage = false, + ) => validateReducerArtifacts( - { artifacts, artifactDir, resultPath, reducerId, sources: snapshot }, + { + artifacts, + artifactDir, + resultPath, + reducerId, + sources: snapshot, + persistSourceCoverage, + }, scanId, ); await assert.rejects(validateSnapshot(), /unaccounted source findings/); @@ -224,10 +235,18 @@ async function testReducerValidation(root: string) { const validatedSnapshot = await validateSnapshot(); assert.equal(validatedSnapshot.newFindings, 2); const admitted = await readJson(resultPath); + const { sourceCoverage, ...legacySnapshot } = validatedSnapshot.result; + assert.equal(sourceCoverage.completeness, "unknown"); assert.deepEqual( - validatedSnapshot.result, + legacySnapshot, admitted, - "validation returns the same reconciled result that was accepted on disk", + "v1 preserves host coverage in memory while retaining the legacy persisted shape", + ); + const versionedSnapshot = await validateSnapshot("dedup-0001", sources, true); + assert.deepEqual( + versionedSnapshot.result, + JSON.parse(await readFile(resultPath, "utf8")), + "v2 persists the full host projection", ); assert.equal(Object.hasOwn(admitted, "coverage"), false); assert.deepEqual(admitted.findings[1].provenance.sourceFindingIds, [ diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_attempt_replay.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_attempt_replay.mjs new file mode 100644 index 0000000000..d89d889ce1 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_attempt_replay.mjs @@ -0,0 +1,241 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; +import { build } from "esbuild"; + +const app = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const plugin = path.resolve(app, ".."); +const bundle = await build({ + bundle: true, + format: "esm", + platform: "node", + write: false, + loader: { ".md": "text" }, + stdin: { + resolveDir: app, + contents: [ + 'export { WorkbenchDeepScanStore } from "./src/deep-scan/store.ts";', + 'export { DeepScanWorkerRunner } from "./src/deep-scan/worker-runner.ts";', + 'export { DeepScanCoordinator } from "./src/deep-scan/coordinator.ts";', + 'export { createDeepScanArtifacts, ensureDeepScanDirectories } from "./src/deep-scan/artifacts.ts";', + ].join("\n"), + }, +}); +const { + WorkbenchDeepScanStore, + DeepScanWorkerRunner, + DeepScanCoordinator, + createDeepScanArtifacts, + ensureDeepScanDirectories, +} = await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}` +); +const execute = promisify(execFile); +for (const responseLosses of [3, 1]) await testResponseLoss(responseLosses); + +async function testResponseLoss(responseLosses) { + const root = await mkdtemp(path.join(tmpdir(), "deep-attempt-receipt-")); + const target = path.join(root, "target"); + const environment = { + ...process.env, + CODEX_HOME: path.join(root, "home"), + CODEX_SECURITY_STATE_DIR: path.join(root, "state"), + }; + const counts = new Map(); + const receipts = new Map(); + const raw = async (args) => { + const { stdout } = await execute( + process.env.PYTHON || "python3", + [path.join(plugin, "scripts/workbench_db.py"), ...args], + { + env: environment, + timeout: 30_000, + maxBuffer: 4 * 1024 * 1024, + }, + ); + return JSON.parse(stdout); + }; + const store = new WorkbenchDeepScanStore(async (args) => { + const result = await raw(args); + const key = + args[0] === "commit-deep-scan-dedup" + ? "merge" + : args[0] === "upsert-deep-scan-worker" && + args[args.indexOf("--status") + 1] === "succeeded" + ? "acceptance" + : null; + const operation = + key === "acceptance" + ? `${key}:${args[args.indexOf("--worker-id") + 1]}` + : key; + if (key) { + counts.set(key, (counts.get(key) ?? 0) + 1); + const receipt = + key === "acceptance" + ? result.deepScan.workerReceipt + : result.deepScan.committedMerge; + if (!receipts.has(operation)) receipts.set(operation, receipt); + else + assert.deepEqual( + receipt, + receipts.get(operation), + "replay returns the original operation receipt", + ); + if (counts.get(key) <= responseLosses) { + const error = new Error("fixture response lost after committed write"); + error.code = "ETIMEDOUT"; + throw error; + } + } + return result; + }); + try { + await mkdir(target); + await writeFile(path.join(target, "fixture.py"), "print('fixture')\n"); + const run = await store.begin({ + targetPath: target, + scope: ".", + threadId: "fixture-owner", + scanRoot: path.join(root, "scans"), + }); + const artifacts = createDeepScanArtifacts(run.scanDir); + await ensureDeepScanDirectories(artifacts); + let executions = 0; + const runner = new DeepScanWorkerRunner({ + run, + store, + artifacts, + pluginRoot: plugin, + signal: new AbortController().signal, + random: () => 0.5, + log: () => {}, + retryDelaysMs: [], + clock: { now: () => Date.now(), sleep: async () => {} }, + executor: { + async run(request) { + executions++; + if (request.kind === "dedup") { + const workers = request.artifactContext.deepReducer.claimedWorkers; + const prompt = await readFile(request.promptPath, "utf8"); + const configuration = JSON.parse( + prompt.match(/```json\n([\s\S]*?)\n```/)[1], + ); + assert.deepEqual( + configuration.claimedWorkerIds, + workers.map((worker) => worker.id), + ); + assert.equal( + workers.every((worker) => + worker.resultPath.includes("checkpoints"), + ), + true, + ); + assert.deepEqual( + workers.map((worker) => worker.attempt), + [1, 1], + "execution uses the immutable claim attempts", + ); + } + await request.onThreadStarted?.(`fixture-session-${executions}`); + const draft = { + scanId: run.scanId, + findings: [], + threatModel: { summary: "Synthetic fixture." }, + }; + if (request.kind === "discovery") + draft.coverage = { + completeness: "complete", + surfaces: [{ label: "Fixture", disposition: "no_issue_found" }], + explicitExclusions: [], + deferred: [], + }; + await writeFile( + path.join(request.artifactContext.root, "result.json"), + JSON.stringify(draft), + ); + return { threadId: `fixture-session-${executions}` }; + }, + }, + }); + if (responseLosses === 3) { + const outcome = await runner + .runDiscoveryWorker(randomUUID(), "discovery-1") + .catch((error) => error); + assert.equal( + counts.get("acceptance"), + 3, + "the runner must not multiply the store's retry policy", + ); + assert.match(outcome.message, /response lost/); + assert.equal(executions, 1); + return; + } + const discovery = await runner.runDiscoveryWorker( + randomUUID(), + "discovery-1", + ); + assert.equal(discovery.status, "succeeded"); + assert.match(discovery.worker.resultPath, /checkpoints/); + const second = await runner.runDiscoveryWorker(randomUUID(), "discovery-2"); + // Acceptance receipts are operation-specific, so only the first discovery loses a response. + await rm(path.join(discovery.worker.artifactDir, "result.json")); + const merged = await runner.runReducer({ + id: randomUUID(), + label: "dedup-1", + consumed: [discovery.worker, second.worker].map((worker) => ({ + ...worker, + resultPath: path.join(worker.artifactDir, "result.json"), + attempt: 99, + })), + }); + assert.equal(merged.error, undefined, merged.error?.stack); + assert.match(merged.resultPath, /checkpoints/); + assert.equal(merged.newFindings, 0); + assert.equal( + merged.run.persistedDedupInputs.filter( + (input) => input.dedupWorkerId === merged.id, + ).length, + 2, + ); + assert.equal(counts.get("merge"), 2); + assert.equal(executions, 3); + assert.equal(run.workflowVersion, "deep-security-scan/v2"); + assert.deepEqual( + JSON.parse(await readFile(merged.resultPath, "utf8")), + merged.result, + ); + assert.equal(merged.result.sourceCoverage.completeness, "complete"); + const snapshot = await store.get(run.scanId, "fixture-owner"); + const resumed = new DeepScanCoordinator({ + run: snapshot, + store, + pluginRoot: plugin, + executor: { + run: async () => + assert.fail("accepted recovery must not execute another model"), + }, + }); + const beforeRecovery = await readFile(merged.resultPath, "utf8"); + const recovered = await resumed.recoverAcceptedDiscoveries(); + assert.deepEqual( + recovered.map((worker) => worker.resultPath), + [discovery.worker.resultPath, second.worker.resultPath], + ); + await rm(path.join(path.dirname(merged.resultPath), "..", "result.json")); + const reducers = await resumed.recoverCompletedReducers(recovered); + assert.equal(reducers.resultPath, merged.resultPath); + assert.deepEqual(reducers.result, merged.result); + assert.equal( + await readFile(merged.resultPath, "utf8"), + beforeRecovery, + "recovery cannot rewrite accepted bytes", + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_checkpoint_coverage.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_checkpoint_coverage.mjs new file mode 100644 index 0000000000..d28d432854 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_checkpoint_coverage.mjs @@ -0,0 +1,49 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { publishCoverageFixture } from "./deep_scan_coverage_fixture.mjs"; + +for (const continueAfterResume of [false, true]) { + test(`immutable discovery receipts survive resumed publication (continued: ${continueAfterResume})`, async () => { + const root = await mkdtemp( + path.join(tmpdir(), "deep-checkpoint-coverage-"), + ); + try { + const fixture = path.join(root, "fixture"); + await mkdir(fixture, { mode: 0o700 }); + const { scanDir } = await publishCoverageFixture(fixture, "partial", { + resume: true, + continueAfterResume, + immutableInputs: true, + }); + const coverage = JSON.parse( + await readFile(path.join(scanDir, "coverage.json"), "utf8"), + ); + assert.equal(coverage.completeness, "partial"); + assert.deepEqual( + coverage.reviews.map((review) => review.completeness), + ["partial", "complete", "unknown"], + ); + assert.equal(coverage.reviews[0].attempt, 2); + assert.equal( + new Set(coverage.deferred.map((item) => item.candidateId)).size, + 2, + ); + const report = await readFile(path.join(scanDir, "report.md"), "utf8"); + for (const item of coverage.deferred) + assert.ok(report.includes(item.reason)); + for (const surface of coverage.surfaces) { + for (const receipt of surface.receiptRefs) { + assert.equal( + await readFile(path.join(scanDir, receipt), "utf8"), + "Synthetic review evidence.\n", + ); + } + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_compatibility.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_compatibility.mjs new file mode 100644 index 0000000000..64e5935ed7 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_compatibility.mjs @@ -0,0 +1,204 @@ +import assert from "node:assert/strict"; +import { build } from "esbuild"; + +const bundle = await build({ + bundle: true, + entryPoints: [ + new URL("../src/deep-scan/registry.ts", import.meta.url).pathname, + ], + format: "esm", + loader: { ".md": "text" }, + platform: "node", + write: false, +}); +const { startOrJoinDeepScanCoordinator, DeepScanRemoteCoordinator } = + await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}` + ); + +await testUnsupportedWorkflowDoesNotAcquireOwnership(); +await testUnsupportedSelectionDoesNotAcquireOwnership(); + +async function testUnsupportedSelectionDoesNotAcquireOwnership() { + for (const selection of [ + { + workflowVersion: "deep-security-scan/v1", + finalizationInput: { version: 1 }, + }, + { + workflowVersion: "deep-security-scan/v2", + finalizationInput: { version: 99 }, + }, + ]) { + await assert.rejects( + startOrJoinDeepScanCoordinator({ + run: { scanId: "fixture", schemaVersion: 1, ...selection }, + registry: { + get: () => + assert.fail("unsupported selection inspected a live coordinator"), + start: () => + assert.fail("unsupported selection started a coordinator"), + }, + options: { + threadId: "fixture-thread", + prepareExecutor: async () => + assert.fail("unsupported selection resolved settings"), + store: { + claimCoordinator: async () => + assert.fail("unsupported selection acquired ownership"), + }, + }, + }), + /finalization input version/, + ); + } +} + +async function testUnsupportedWorkflowDoesNotAcquireOwnership() { + for (const version of [ + { schemaVersion: 99, workflowVersion: "deep-scan-mcp/v1" }, + { schemaVersion: 1, workflowVersion: "future/v99" }, + ]) { + let mutations = 0; + await assert.rejects( + startOrJoinDeepScanCoordinator({ + run: { scanId: "fixture", ...version }, + registry: { + get: () => undefined, + start: () => { + mutations += 1; + }, + }, + options: { + threadId: "fixture-thread", + store: { + claimCoordinator: async () => { + mutations += 1; + }, + }, + }, + }), + /unsupported workflow or schema version/, + ); + assert.equal(mutations, 0); + } +} + +// A joining client must not resolve or replace the live executor's settings. +for (const workflowVersion of [ + "deep-scan-mcp/v1", + "deep-security-scan/v1", + "deep-security-scan/v2", +]) { + for (const local of [true, false]) { + let preparations = 0; + const run = { scanId: "fixture", status: "running", workflowVersion }; + const options = { + threadId: "fixture-thread", + executor: { marker: "observer" }, + prepareExecutor: async () => { + preparations += 1; + return {}; + }, + store: { claimCoordinator: async () => ({ run, acquired: false }) }, + }; + await startOrJoinDeepScanCoordinator({ + run, + registry: { + get: () => (local ? {} : undefined), + start: () => assert.fail("observer started"), + }, + options, + }); + assert.equal(preparations, 0); + } +} + +// Selected publication has no worker launch and must not need current settings. +for (const selected of [false, true]) { + const run = { + scanId: "fixture", + status: "running", + workflowVersion: "deep-security-scan/v2", + ...(selected ? { finalizationInput: { version: 1 } } : {}), + }; + let preparations = 0; + const fallback = {}; + const restored = {}; + await startOrJoinDeepScanCoordinator({ + run, + registry: { + get: () => undefined, + start: (options) => { + assert.equal(options.executor, selected ? fallback : restored); + assert.equal(options.run, run); + return {}; + }, + }, + options: { + threadId: "fixture-thread", + executor: fallback, + prepareExecutor: async () => { + preparations += 1; + return restored; + }, + store: { claimCoordinator: async () => ({ run, acquired: true }) }, + }, + }); + assert.equal(preparations, selected ? 0 : 1); +} + +const originalNow = Date.now; +try { + let now = 0; + Date.now = () => now; + const run = { + scanId: "fixture", + status: "running", + updatedAt: "1970-01-01T00:00:00Z", + }; + const acquired = { + ...run, + model: "original-model", + coordinatorGeneration: 3, + }; + let preparations = 0; + const executor = { marker: "restored" }; + const registry = { + get: () => undefined, + start: (options) => { + assert.equal(options.run, acquired); + assert.equal(options.executor, executor); + return { wait: async () => ({ ...acquired, status: "succeeded" }) }; + }, + }; + const options = { + threadId: "fixture-thread", + executor: { marker: "observer" }, + prepareExecutor: async (state) => { + assert.equal(state, acquired); + preparations += 1; + return executor; + }, + store: { + get: async () => run, + claimCoordinator: async () => ({ run: acquired, acquired: true }), + }, + }; + await startOrJoinDeepScanCoordinator({ + run: { ...run, startDisposition: "created" }, + registry, + options, + }); + assert.equal(preparations, 1); + const remote = new DeepScanRemoteCoordinator({ run, registry, options }); + now = 60_000; + assert.equal((await remote.wait(undefined, 1_000)).status, "succeeded"); + assert.equal( + preparations, + 2, + "takeover resolves settings from the newly acquired run", + ); +} finally { + Date.now = originalNow; +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts index 3482d6c951..cf6ae30903 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts @@ -8,7 +8,7 @@ import type { } from "../src/deep-scan/types.js"; import { mock } from "node:test"; import assert from "node:assert/strict"; -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { mkdir, readFile, @@ -46,6 +46,144 @@ import { type StoreInput, } from "./deep_scan_coordinator_fixture.ts"; +async function testSelectedFinalizationOwner() { + for (const failure of [ + undefined, + "draft", + "finish", + "parent", + "replacement", + ] as const) { + const fixture = await fixtureRun({ + workers: 1, + subagents: 0, + stopAfterNoNew: 1, + maxDiscoveryRuns: 1, + }); + fixture.run.workflowVersion = "deep-security-scan/v2"; + fixture.run.coordinatorGeneration = 3; + fixture.run.finalizationInput = { + version: 1, + resultPath: null, + resultSha256: null, + terminalReason: "capped", + omittedWorkerIds: [], + selectedAt: "2026-01-01T00:00:00Z", + }; + const store = new FakeStore(fixture.run); + const finishStarted = Promise.withResolvers(); + const releaseFinish = Promise.withResolvers(); + const parentStarted = Promise.withResolvers(); + const releaseParent = Promise.withResolvers(); + const calls: string[] = []; + const events: DeepScanLogEvent[] = []; + store.finish = async () => { + calls.push("finish"); + finishStarted.resolve(); + await releaseFinish.promise; + if (failure === "finish") + throw new Error("injected selected finish failure"); + if (failure === "replacement") { + store.run = { + ...store.run, + status: "succeeded", + coordinatorGeneration: 4, + }; + throw new Error("Deep Scan coordinator generation is stale."); + } + store.run = { + ...store.run, + status: "succeeded", + terminalReason: "capped", + }; + return structuredClone(store.run); + }; + const coordinator = new DeepScanCoordinator({ + run: fixture.run, + store, + pluginRoot: fixture.pluginRoot, + executor: { + run: async () => + assert.fail("Selected finalization cannot start model work"), + }, + threadId: "selected-owner", + log: (event: DeepScanLogEvent) => events.push(event), + onComplete: async () => { + calls.push("draft"); + if (failure === "draft") + throw new Error("injected selected draft failure"); + }, + onFinalized: async (run: DeepScanRunState) => { + calls.push("parent"); + assert.equal(run.status, "succeeded"); + assert.equal(run.coordinatorGeneration, 3); + parentStarted.resolve(); + await releaseParent.promise; + if (failure === "parent") + throw new Error("injected selected parent failure"); + }, + }); + // Rejections stay observed even when the user detaches their only waiter. + const settled = coordinator.settled(); + settled.catch(() => {}); + const observer = new AbortController(); + coordinator.start(); + const detached = coordinator.wait(observer.signal); + observer.abort(); + await assert.rejects(detached, { name: "AbortError" }); + if (failure !== "draft") { + await finishStarted.promise; + assert.deepEqual(calls, ["draft", "finish"]); + assert.equal(await coordinator.wait(undefined, 10), undefined); + releaseFinish.resolve(); + } + if (failure === undefined || failure === "parent") { + await parentStarted.promise; + assert.equal( + await coordinator.wait(undefined, 10), + undefined, + "wait includes enclosing completion", + ); + releaseParent.resolve(); + } + if (failure && failure !== "replacement") { + await assert.rejects( + settled, + new RegExp(`injected selected ${failure} failure`), + ); + assert.equal( + store.failureMessages.length, + 0, + "publication errors retain the original selected stop", + ); + assert.equal( + events.some( + (event) => event.event === "coordinator_publication_pending", + ), + true, + ); + assert.equal( + store.run.status, + failure === "parent" ? "succeeded" : "running", + ); + } else { + assert.equal((await settled).status, "succeeded"); + } + assert.deepEqual( + calls, + failure === "draft" + ? ["draft"] + : failure === "finish" || failure === "replacement" + ? ["draft", "finish"] + : ["draft", "finish", "parent"], + ); + assert.deepEqual( + store.run.finalizationInput, + fixture.run.finalizationInput, + ); + } +} + async function testCappedQueueAndSerialDedup() { const { fixture, store } = await coordinatorFixture({ workers: 3, @@ -61,6 +199,8 @@ async function testCappedQueueAndSerialDedup() { handoffClaimToken: "claim-fixture", onComplete: async (draft) => void completedDrafts.push(structuredClone(draft)), + onFinalized: async () => + assert.fail("Legacy scans retain explicit parent completion"), }); assert.equal(terminal?.status, "succeeded"); assert.equal(terminal?.terminalReason, "capped"); @@ -1914,7 +2054,7 @@ async function testCancellationDuringDiscoveryAcceptanceRejectsLateSuccess() { const preserved = Promise.withResolvers(); const releasePreservation = Promise.withResolvers(); const coordinator = createCoordinator(fixture, store, executor, { - log: (event) => events.push(event), + log: (event: DeepScanLogEvent) => events.push(event), threadId: "checkpoint-owner", onStopped: async () => { const worker = [...store.workers.values()].find( @@ -2031,7 +2171,7 @@ async function testStoppedPublicationFailurePreservesOriginalDiagnostic() { const events: DeepScanLogEvent[] = []; const coordinator = createCoordinator(fixture, store, executor, { threadId: "checkpoint-owner", - log: (event) => events.push(event), + log: (event: DeepScanLogEvent) => events.push(event), onStopped: async () => { throw new Error( `fixture retained result publication failure ${"y".repeat(2_350)}`, @@ -2643,6 +2783,10 @@ async function testPausedDiscoverySurvivesCoordinatorRestart() { store.heartbeatCoordinator = async () => structuredClone(store.run); const replacementExecutor = new FakeExecutor(); const acceptedResult = await readFile(accepted.resultManifestPath!, "utf8"); + const acceptedWorkerId = await workerIdFromPrompt(accepted.promptPath); + await Promise.all( + persistedWorkers.map((worker) => rm(worker.promptPath, { force: true })), + ); const resumed = await startOrJoinDeepScanCoordinator({ run: structuredClone(store.run), registry: new DeepScanCoordinatorRegistry(), @@ -2663,9 +2807,7 @@ async function testPausedDiscoverySurvivesCoordinatorRestart() { assert.equal(store.failureMessages.length, 0); assert.equal(replacementExecutor.discoveryAttempts.size, 1); assert.equal( - replacementExecutor.discoveryAttempts.has( - await workerIdFromPrompt(accepted.promptPath), - ), + replacementExecutor.discoveryAttempts.has(acceptedWorkerId), false, ); assert.equal(store.dedupClaims.length, 1); @@ -2953,10 +3095,9 @@ async function testPersistedErrorLimitStopsBeforeRescheduling() { assert.equal(executor.discoveryCalls, 0); assert.match( terminal?.error ?? "", - promptExists ? /2 consecutive unsuccessful discovery workers/ : /ENOENT/, + /2 consecutive unsuccessful discovery workers/, ); - if (promptExists) - assert.match(terminal?.error ?? "", /persisted worker failure/); + assert.match(terminal?.error ?? "", /persisted worker failure/); assert.equal(terminal.manifestPath, undefined); assert.equal(store.run.persistedWorkers![0].id, failedWorker.id); } @@ -3320,12 +3461,14 @@ async function testSaturationOmitsWorkerAcceptedDuringCancellation() { ); } -async function testSuccessfulDeepCoverageIgnoresWorkerAndReducerReviewStatus() { - const { fixture, store } = await coordinatorFixture({ +async function testSuccessfulDeepCoveragePreservesWorkerReviewStatus() { + const fixture = await fixtureRun({ workers: 2, + subagents: 0, stopAfterNoNew: 2, maxDiscoveryRuns: 2, }); + const store = new FakeStore(fixture.run); const executor = new FakeExecutor(); const run = executor.run.bind(executor); const reviewed = { label: "Reviewed query", disposition: "no_issue_found" }; @@ -3340,7 +3483,7 @@ async function testSuccessfulDeepCoverageIgnoresWorkerAndReducerReviewStatus() { executor.run = async (request) => { const outcome = await run(request); const resultPath = path.join(request.artifactContext!.root, "result.json"); - const draft = await readJson(resultPath); + const draft = JSON.parse(await readFile(resultPath, "utf8")); draft.coverage = { completeness: request.kind === "discovery" && @@ -3353,24 +3496,50 @@ async function testSuccessfulDeepCoverageIgnoresWorkerAndReducerReviewStatus() { { reason: "An independent review left this question unresolved." }, ], }; - await writeJson(resultPath, draft); + await writeFile(resultPath, JSON.stringify(draft)); return outcome; }; const completed: ScanDraftInput[] = []; - const terminal = await runCoordinator(fixture, store, executor, { - onComplete: async (draft) => void completed.push(structuredClone(draft)), + const coordinator = new DeepScanCoordinator({ + run: fixture.run, + store, + executor, + pluginRoot: fixture.pluginRoot, + clock: immediateClock, + onComplete: async (draft: ScanDraftInput) => + void completed.push(structuredClone(draft)), }); + coordinator.start(); + const terminal = await coordinator.wait(undefined, 5_000); assert.equal(terminal?.status, "succeeded", terminal?.error); assert.equal(completed.length, 1); - assert.deepEqual(completed[0].coverage, { - completeness: "complete", - surfaces: [], - explicitExclusions: [], - deferred: [], - }); + const projection = completed[0].coverage as { + completeness: string; + deferred: { provenance: { attempt: number; workerId: string } }[]; + surfaces: unknown[]; + reviews: { completeness: string }[]; + }; + assert.equal(projection.completeness, "partial"); + assert.equal(projection.deferred.length, 2); + assert.equal(projection.surfaces.length, 4); + assert.equal(projection.reviews.length, 2); + assert.deepEqual( + new Set( + projection.reviews.map( + (review: { completeness: string }) => review.completeness, + ), + ), + new Set(["partial", "unknown"]), + ); + for (const item of projection.deferred) { + assert.equal(item.provenance.attempt, 1); + assert.ok(store.workers.has(item.provenance.workerId)); + } for (const worker of store.workers.values()) { if (worker.kind !== "discovery") continue; - const draft = await readJson(worker.resultManifestPath!); + const draft = JSON.parse( + await readFile(worker.resultManifestPath!, "utf8"), + ); assert.notEqual(draft.coverage.completeness, "complete"); assert.deepEqual(draft.coverage.surfaces, [workerReviewed, followUp]); assert.equal(draft.coverage.deferred.length, 1); @@ -3383,6 +3552,26 @@ async function testSaturationIgnoresDiscoveryCancellationWriteFailure() { stopAfterNoNew: 2, maxDiscoveryRuns: 6, }); + fixture.run.workflowVersion = "deep-security-scan/v2"; + store.run.workflowVersion = "deep-security-scan/v2"; + store.selectFinalization = async (input) => { + const checkpointRoot = path.join( + path.dirname(input.resultPath!), + "checkpoints", + ); + const [name] = await readdir(checkpointRoot); + const checkpoint = path.join(checkpointRoot, name); + const bytes = await readFile(checkpoint); + store.run.finalizationInput = { + version: 1, + resultPath: path.relative(fixture.run.scanDir, checkpoint), + resultSha256: createHash("sha256").update(bytes).digest("hex"), + terminalReason: input.reason, + omittedWorkerIds: input.omittedWorkerIds ?? [], + selectedAt: "2026-01-01T00:00:00Z", + }; + return structuredClone(store.run); + }; const executor = new FakeExecutor({ blockDiscoveryAfterCalls: 2, }); @@ -3438,7 +3627,7 @@ async function testSaturationIgnoresDiscoveryCancellationWriteFailure() { ); const { coverage, ...publishedReduction } = completed[0]; assert.deepEqual( - publishedReduction, + { ...publishedReduction, sourceCoverage: coverage }, await readJson(acceptedReducer!.resultManifestPath!), "the accepted aggregate still reaches publication when redundant cancellation writes fail", ); @@ -3479,7 +3668,7 @@ async function testPublicationUsesAcceptedReducerSnapshot() { assert.equal(completed[0].coverage.completeness, "complete"); } -async function testResumeRequiresHistoricalWorkerPrompt( +async function testResumeDoesNotRequireHistoricalWorkerPrompt( status: "failed" | "canceled", ) { const fixture = await fixtureRun({ @@ -3510,10 +3699,10 @@ async function testResumeRequiresHistoricalWorkerPrompt( }); coordinator.start(); const terminal = await coordinator.wait(undefined, 5_000); - assert.equal(terminal?.status, "failed"); - assert.match(terminal.error, /ENOENT/); - assert.ok(terminal.error.includes(promptPath)); - assert.equal(executor.discoveryCalls, 0); + assert.equal(terminal?.status, "succeeded", terminal?.error); + assert.equal(store.failureMessages.length, 0); + assert.equal(executor.discoveryCalls, 2); + assert.equal(store.run.persistedWorkers?.[0].promptPath, promptPath); } async function testRecoverableWorkerErrorsCannotFailScan() { @@ -3730,6 +3919,7 @@ async function testNonRetryableReducerAbortsScanWithoutRetry( } try { + await testSelectedFinalizationOwner(); await testCappedQueueAndSerialDedup(); await testStandardWorkersReceiveExistingFalsePositiveFeedback(); await testDiscoveryWorkersKeepOneContextAfterPersistedUpdate(); @@ -3742,7 +3932,7 @@ try { await testCompletionOrdering(); await testSaturationPreservesFindingAlreadyBuffered(); await testSaturationOmitsWorkerAcceptedDuringCancellation(); - await testSuccessfulDeepCoverageIgnoresWorkerAndReducerReviewStatus(); + await testSuccessfulDeepCoveragePreservesWorkerReviewStatus(); await testSaturationIgnoresDiscoveryCancellationWriteFailure(); await testPublicationUsesAcceptedReducerSnapshot(); await testDirectReducerCannotDropAcceptedFinding(); @@ -3814,8 +4004,8 @@ try { await testCoordinatorHeartbeatsContinueDuringBlockedOwnershipRead(); await testRemoteObserverRetriesTransientPersistenceFailures(); await testJoinAndOrphanRules(); - await testResumeRequiresHistoricalWorkerPrompt("failed"); - await testResumeRequiresHistoricalWorkerPrompt("canceled"); + await testResumeDoesNotRequireHistoricalWorkerPrompt("failed"); + await testResumeDoesNotRequireHistoricalWorkerPrompt("canceled"); await testPausedDiscoverySurvivesCoordinatorRestart(); await testResumedDiscoveryDeadlineUsesPersistedCreationTime(); await testResumedDiscoveryDeadlineUsesPersistedCreationTime(true); diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_coverage_descriptions.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_coverage_descriptions.mjs new file mode 100644 index 0000000000..60c396cd68 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_coverage_descriptions.mjs @@ -0,0 +1,81 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { publishCoverageFixture } from "./deep_scan_coverage_fixture.mjs"; + +const descriptions = { + source: "source-review", + worker: "focused-reviewer", + task: "Check the filesystem boundary.", +}; +const sourceProvenance = { + ...descriptions, + workerId: "worker-local-label", + attempt: 99, + sourceId: "worker-local-source", + candidateId: "worker-local-candidate", +}; + +for (const options of [ + {}, + { resume: true }, + { resume: true, continueAfterResume: true }, + { resume: true, selectedRecovery: true, discardMutableResults: true }, + { + resume: true, + selectedRecovery: true, + discardMutableResults: true, + splitSeededReducers: true, + }, +]) { + test(`coverage source descriptions survive publication and recovery: ${JSON.stringify(options)}`, async () => { + const root = await mkdtemp( + path.join(tmpdir(), "deep-coverage-descriptions-"), + ); + try { + const fixture = path.join(root, "fixture"); + await mkdir(fixture, { mode: 0o700 }); + const { scanDir } = await publishCoverageFixture(fixture, "partial", { + ...options, + immutableInputs: true, + materialFindings: true, + sourceProvenance, + }); + const coverage = JSON.parse( + await readFile(path.join(scanDir, "coverage.json"), "utf8"), + ); + assert.equal(coverage.completeness, "partial"); + assert.equal( + new Set(coverage.deferred.map((item) => item.candidateId)).size, + 2, + ); + for (const field of [ + "surfaces", + "explicitExclusions", + "deferred", + "openQuestions", + ]) { + assert.ok(coverage[field].length > 0); + for (const item of coverage[field]) { + const review = coverage.reviews.find( + (review) => review.workerId === item.provenance.workerId, + ); + assert.ok(review, "host identity names an actual independent review"); + assert.deepEqual(item.provenance, { + ...descriptions, + workerId: review.workerId, + attempt: review.attempt, + ...(field === "surfaces" ? { sourceId: "shared-surface" } : {}), + ...(field === "deferred" + ? { sourceId: "same-id", candidateId: "candidate-1" } + : {}), + }); + } + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts index d568af3c26..57553a86ec 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts @@ -1,3 +1,5 @@ +import { parse as parseToml } from "smol-toml"; +import { rm } from "node:fs/promises"; import { readJson, writeJson } from "./support/json.ts"; import { assertFlagPair } from "./assertions.ts"; import { createTemporaryDirectories } from "./support/temporary-directories.ts"; @@ -34,6 +36,10 @@ const { CodexSdkWorkerExecutor, resolveCodexPath, snapshotWorkerEnvironment, + WorkbenchDeepScanStore, + captureDeepScanExecutionSettings, + loadDeepScanExecutionSettings, + restoredDeepScanWorkerSettings, appendSafeItemDiagnostic, classifyCodexWorkerError, DeepScanNonRetryableError, @@ -46,7 +52,9 @@ const { // Test the environment snapshot without adding a production export. contents: `${await readFile(executorSource, "utf8")} export { snapshotWorkerEnvironment, appendSafeItemDiagnostic }; -export * from "./errors.js";`, +export * from "./errors.js"; +export * from "./recovery-settings.js"; +export { WorkbenchDeepScanStore } from "./store.js";`, loader: "ts", resolveDir: path.dirname(fileURLToPath(executorSource)), sourcefile: fileURLToPath(executorSource), @@ -91,6 +99,7 @@ try { await testOpenAiCredentialsReachWorker(); await testWorkerRuntimeSettings(); await testWorkerCyberAccessSettings(); + await testIsolatedReconstructedWorkers(); if (process.platform !== "win32") { await testMissingParentSandboxFailsBeforeWorkerLaunch(); await testDisallowedWorkerProfileFailsBeforeWorkerLaunch(); @@ -100,6 +109,9 @@ try { await testSdkInvocationAndThreadCapture(); await testBedrockCredentialsReachWorker(); await testArtifactServerUsesExtendedStartupTimeout(); + await testReducerCoveragePersistenceBinding(); + await testWorkerProviderSelection(); + await testNullUsageCompletion(); await testZeroSubagentsPreservesHostRestrictions(); await testSdkResumesExistingThread(); await testRetryNotificationDoesNotInterruptTurn(); @@ -378,7 +390,7 @@ async function testWindowsLongExecutableLaunches() { ); children.push(child); return child; - }) as typeof childProcess.spawn; + }) as unknown as typeof childProcess.spawn; syncBuiltinESMExports(); try { process.env.CODEX_HOME = codexHome; @@ -1040,7 +1052,7 @@ async function testOpenAiCredentialsReachWorker() { ? [fixture.executablePath, ...args] : args, options, - )) as typeof childProcess.spawn; + )) as unknown as typeof childProcess.spawn; syncBuiltinESMExports(); const promptPath = path.join(fixture.root, "prompt.md"); await writeFile( @@ -1213,7 +1225,7 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`, : args, { ...options, env: environment }, ); - }) as typeof childProcess.spawn; + }) as unknown as typeof childProcess.spawn; syncBuiltinESMExports(); const settings = [ { model: "gpt-5.6-sol", reasoningEffort: "xhigh" }, @@ -1471,7 +1483,7 @@ async function testWorkerCyberAccessSettings() { ...options, env: environment, }); - }) as typeof childProcess.spawn; + }) as unknown as typeof childProcess.spawn; syncBuiltinESMExports(); for (const testCase of cases) { testCase.configPath = path.join(fixture.root, `${testCase.name}.toml`); @@ -2201,7 +2213,11 @@ async function runFixtureWorker( } async function fakeCodexFixture( - preflightProfile = emptyWorkerPermissionProfile, + preflightProfile: { + extends: string; + filesystem: Record; + network: { enabled: boolean }; + } = emptyWorkerPermissionProfile, preflightAllowed = true, accountResult: { account: { type: string } | null; @@ -2224,7 +2240,7 @@ const preflightAllowed = ${JSON.stringify(preflightAllowed)}; const accountResult = ${JSON.stringify(accountResult)}; const preflightMarkerPath = process.env.FAKE_CODEX_PREFLIGHT_MARKER ?? ${JSON.stringify(preflightMarkerPath)}; if (process.argv.includes('app-server')) { - const preflight = { cwd: process.cwd(), codexHome: process.env.CODEX_HOME, runnerTrackingId: process.env.RUNNER_TRACKING_ID, libraryPath: process.env.LD_LIBRARY_PATH, gitEnvironment: Object.fromEntries(['PATH', 'CODEX_SECURITY_GIT', 'GIT_SSH_COMMAND', 'GIT_CONFIG_GLOBAL'].map(name => [name, process.env[name]])), requests: [] }; + const preflight = { argv: process.argv.slice(2), cwd: process.cwd(), codexHome: process.env.CODEX_HOME, runnerTrackingId: process.env.RUNNER_TRACKING_ID, libraryPath: process.env.LD_LIBRARY_PATH, gitEnvironment: Object.fromEntries(['PATH', 'CODEX_SECURITY_GIT', 'GIT_SSH_COMMAND', 'GIT_CONFIG_GLOBAL'].map(name => [name, process.env[name]])), requests: [] }; writeFileSync(preflightMarkerPath, JSON.stringify(preflight)); let buffer = ''; process.stdin.setEncoding('utf8'); @@ -2264,12 +2280,13 @@ if (process.argv.includes('app-server')) { } else { const stdin = (await process.stdin.toArray()).join(''); const openaiAuthentication = stdin.includes('CAPTURE_SYNTHETIC_OPENAI_AUTH') ? { OPENAI_API_KEY: process.env.OPENAI_API_KEY, CODEX_API_KEY: process.env.CODEX_API_KEY } : undefined; +const providerAuthentication = stdin.includes('CAPTURE_SYNTHETIC_PROVIDER_AUTH') ? Object.fromEntries(JSON.parse(process.env.FAKE_CODEX_PROVIDER_ENV_KEYS).map((name) => [name, process.env[name]])) : undefined; const bedrockAuthentication = stdin.includes('CAPTURE_SYNTHETIC_BEDROCK_AUTH') ? Object.fromEntries(JSON.parse(process.env.FAKE_CODEX_BEDROCK_ENV_KEYS).map((name) => [name, process.env[name]])) : undefined; const runtimeEnvironment = Object.fromEntries(['PATH', 'HOME', 'PYTHON', 'PYTHONUTF8', 'LD_LIBRARY_PATH', 'CODEX_SECURITY_STATE_DIR', 'RUNNER_TRACKING_ID'].map(name => [name, process.env[name]])); const pythonProbe = stdin.includes('CAPTURE_SYNTHETIC_PYTHON') ? spawnSync(process.env.PYTHON, ['-I', '-c', 'import json,os,sys; print(json.dumps([sys.prefix,os.environ.get("LD_LIBRARY_PATH")]))'], { encoding: 'utf8' }) : undefined; if (pythonProbe && pythonProbe.status !== 0) throw new Error(pythonProbe.stderr || String(pythonProbe.error)); const pythonRuntime = pythonProbe ? JSON.parse(pythonProbe.stdout) : undefined; -writeFileSync(process.env.FAKE_CODEX_MARKER, JSON.stringify({ argv: process.argv.slice(2), stdin, cwd: process.cwd(), codexHome: process.env.CODEX_HOME, gitEnvironment: Object.fromEntries(['PATH', 'CODEX_SECURITY_GIT', 'GIT_SSH_COMMAND', 'GIT_CONFIG_GLOBAL'].map(name => [name, process.env[name]])), configPath: process.env.CODEX_SECURITY_CONFIG_PATH, deepConfigPath: process.env.CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH, python: process.env.PYTHON, pythonPrefix: pythonRuntime?.[0], pythonLibraryPath: pythonRuntime?.[1], runtimeEnvironment, providerKey: process.env.SYNTHETIC_GATEWAY_KEY, originator: process.env.CODEX_INTERNAL_ORIGINATOR_OVERRIDE, ...(stdin.includes('COMPLETE_THEN_HANG') ? { pid: process.pid } : {}), ...(openaiAuthentication ? { openaiAuthentication } : {}), ...(bedrockAuthentication ? { bedrockAuthentication } : {}) })); +writeFileSync(process.env.FAKE_CODEX_MARKER, JSON.stringify({ executable: process.execPath, codexCliPath: process.env.CODEX_CLI_PATH, scanValue: process.env.FAKE_CODEX_SCAN_VALUE, argv: process.argv.slice(2), stdin, cwd: process.cwd(), codexHome: process.env.CODEX_HOME, gitEnvironment: Object.fromEntries(['PATH', 'CODEX_SECURITY_GIT', 'GIT_SSH_COMMAND', 'GIT_CONFIG_GLOBAL'].map(name => [name, process.env[name]])), configPath: process.env.CODEX_SECURITY_CONFIG_PATH, deepConfigPath: process.env.CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH, python: process.env.PYTHON, pythonPrefix: pythonRuntime?.[0], pythonLibraryPath: pythonRuntime?.[1], runtimeEnvironment, providerKey: process.env.SYNTHETIC_GATEWAY_KEY, originator: process.env.CODEX_INTERNAL_ORIGINATOR_OVERRIDE, ...(stdin.includes('COMPLETE_THEN_HANG') ? { pid: process.pid } : {}), ...(openaiAuthentication ? { openaiAuthentication } : {}), ...(bedrockAuthentication ? { bedrockAuthentication } : {}), ...(providerAuthentication ? { providerAuthentication } : {}) })); if (stdin.includes('COMPLETE_THEN_HANG')) process.on('SIGTERM', () => setTimeout(() => process.exit(0), 100)); if (stdin.includes('THREAD_START_CONFIG_ERROR')) { console.error('Error: thread/start: thread/start failed: agents.max_threads cannot be set when features.multi_agent_v2 is enabled (code -32600)'); process.exit(1); } if (stdin.includes('CONFIG_ERROR')) { console.error('failed to load configuration: invalid value'); process.exit(2); } @@ -2318,7 +2335,7 @@ if (stdin.includes('ARTIFACT_TOOL_')) { console.log(JSON.stringify({ type: 'item.completed', item })); } console.log(JSON.stringify({ type: 'item.completed', item: { id: 'message-1', type: 'agent_message', text: 'fixture final response' } })); -console.log(JSON.stringify({ type: 'turn.completed', usage: { input_tokens: 1, cached_input_tokens: 0, output_tokens: 1 } })); +console.log(JSON.stringify({ type: 'turn.completed', usage: stdin.includes('NULL_USAGE') ? null : { input_tokens: 1, cached_input_tokens: 0, output_tokens: 1 } })); if (stdin.includes('COMPLETE_THEN_HANG')) { setInterval(() => {}, 1_000); await new Promise(() => {}); } } `, @@ -2422,3 +2439,749 @@ async function withWorkerFixture( restoreEnv("CODEX_CLI_PATH", previousPath); } } + +async function testIsolatedReconstructedWorkers() { + const launchFailures: string[] = []; + const previousMarker = process.env.FAKE_CODEX_MARKER; + const originalSpawn = childProcess.spawn; + const scans: Awaited>[] = []; + async function prepareReconstructedScan(name: string) { + const uncappedSandbox = { + filesystemDenies: trustedParentSandboxWithDenials.filesystemDenies, + }; + const currentParentSandbox = + name === "first" ? uncappedSandbox : trustedParentSandboxWithDenials; + const fixture = await fakeCodexFixture({ + ...deniedWorkerPermissionProfile, + filesystem: { + ":root": "read", + "/repo/.env": "deny", + "/repo/**/*.pem": "deny", + "/repo/**/.secret": "deny", + }, + }); + const codexHome = path.join(fixture.root, "home"); + const configPath = path.join(fixture.root, "scan config.toml"); + const promptPath = path.join(fixture.root, "prompt.md"); + await mkdir(codexHome); + const config: Record = { + model: `fixture-${name}-inherited`, + model_provider: name === "first" ? "openrouter" : "amazon-bedrock", + model_reasoning_effort: "medium", + model_reasoning_summary: "concise", + service_tier: name === "first" ? "default" : "fast", + }; + await writeFile( + configPath, + Object.entries(config) + .filter( + ([key]) => + name !== "first" || + ![ + "model_provider", + "model_reasoning_summary", + "service_tier", + ].includes(key), + ) + .map(([key, value]) => `${key} = ${JSON.stringify(value)}\n`) + .join("") + + (name === "second" + ? '[model_providers.amazon-bedrock.aws]\nregion = "us-west-2"\nprofile = "fixture-profile"\n' + : "") + + `[codex_security]\ncyber_access_program = "${name === "first" ? "daybreak_blue" : "standard"}"\n[features]\napi_key_cyber_access_programs = ${name === "first"}\napi_key_model_discovery = ${name !== "first"}\n`, + ); + const providerKeys = + name === "first" + ? ["OPENROUTER_API_KEY"] + : ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]; + await writeFile( + promptPath, + "CAPTURE_SYNTHETIC_OPENAI_AUTH CAPTURE_SYNTHETIC_PROVIDER_AUTH NULL_USAGE\n", + ); + const executable = path.join( + fixture.root, + process.platform === "win32" ? "node.exe" : "node", + ); + // Keep dynamically linked Node beside its libraries on Unix. Each scan + // still selects a distinct executable path at the spawn boundary. + if (process.platform === "win32") + await copyFile(process.execPath, executable); + else await symlink(process.execPath, executable); + const codexOptions = { + codexPathOverride: executable, + baseUrl: `https://${name}.example.invalid/v1`, + env: { + PATH: path.dirname(process.execPath), + ...(process.env.SystemRoot + ? { SystemRoot: process.env.SystemRoot } + : {}), + CODEX_HOME: codexHome, + CODEX_SECURITY_CONFIG_PATH: configPath, + CODEX_API_KEY: `synthetic-${name}-credential`, + FAKE_CODEX_MARKER: fixture.markerPath, + FAKE_CODEX_PROVIDER_ENV_KEYS: JSON.stringify(providerKeys), + FAKE_CODEX_SCAN_VALUE: name, + }, + }; + const settings = { + codexOptions, + model: `fixture-${name}-override`, + reasoningEffort: "ultra", + usageOwner: { + threadId: `fixture-${name}-owner`, + turnId: "original-turn", + startedAt: "2026-01-01T00:00:00Z", + }, + parentSandbox: + name === "first" ? trustedParentSandboxWithDenials : uncappedSandbox, + }; + await mkdir(path.join(codexHome, "sessions")); + await writeFile( + path.join(codexHome, "sessions", "owner.jsonl"), + [ + { + type: "session_meta", + timestamp: "2026-01-01T00:00:00Z", + payload: { + id: `fixture-${name}-owner`, + model_provider: config.model_provider, + }, + }, + { + type: "event_msg", + timestamp: "2026-01-01T00:00:00Z", + payload: { + type: "thread_settings_applied", + thread_id: `fixture-${name}-owner`, + thread_settings: { + model: "native-parent-model", + model_provider_id: config.model_provider, + reasoning_effort: "medium", + reasoning_summary: config.model_reasoning_summary, + }, + }, + }, + { + type: "turn_context", + timestamp: "2026-01-01T00:00:01Z", + payload: { + turn_id: "original-turn", + model: "native-parent-model", + effort: "medium", + summary: "none", + }, + }, + { + type: "turn_context", + timestamp: "2026-01-01T00:02:00Z", + payload: { + turn_id: "later-turn", + model: "later-parent-model", + effort: "low", + summary: "detailed", + }, + }, + ] + .map((value) => JSON.stringify(value)) + .join("\n") + "\n", + ); + await writeFile( + path.join(codexHome, "sessions", "observer.jsonl"), + JSON.stringify({ + type: "session_meta", + timestamp: "2026-01-01T00:00:00Z", + payload: { + id: `fixture-${name}-observer`, + model_provider: "observer-provider", + }, + }) + "\n", + ); + const saved = await captureDeepScanExecutionSettings( + settings, + settings.parentSandbox, + { ...codexOptions.env, CODEX_CLI_PATH: executable }, + { + threadId: `fixture-${name}-observer`, + startedAt: "2026-01-01T00:01:00Z", + }, + ); + assert.equal( + saved.nativeServiceTierAbsent, + name === "first" ? true : undefined, + ); + const targetPath = path.join(fixture.root, "target"); + await mkdir(targetPath); + const workbenchPath = fileURLToPath( + new URL("../../scripts/workbench_db.py", import.meta.url), + ); + const runWorkbench = async ( + args: string[], + input?: string, + _selectFinalization?: boolean, + withExecutionSettings?: boolean, + ) => { + const pythonArgs = withExecutionSettings + ? [ + "-c", + "import runpy, sys; script = sys.argv.pop(1); runpy.run_path(script)['main'](with_execution_settings=True)", + workbenchPath, + ...args, + ] + : [workbenchPath, ...args]; + const result = spawnSync( + process.env.PYTHON?.trim() || "python3", + pythonArgs, + { + env: { + ...process.env, + CODEX_HOME: codexHome, + CODEX_SECURITY_STATE_DIR: path.join(fixture.root, "state"), + }, + input, + encoding: "utf8", + timeout: 30_000, + }, + ); + assert.equal(result.status, 0, result.stderr); + return JSON.parse(result.stdout); + }; + const store = new WorkbenchDeepScanStore(runWorkbench); + const beginInput = { + targetPath, + threadId: settings.usageOwner.threadId, + model: settings.model, + reasoningEffort: settings.reasoningEffort, + scanRoot: path.join(fixture.root, "scans"), + }; + const run: import("../src/deep-scan/types.js").DeepScanRunState = + await store.begin({ ...beginInput, executionSettings: saved }); + const recordedScanDir = run.scanDir; + const snapshotPath = path.join( + recordedScanDir, + "artifacts", + "deep_discovery", + "execution-settings.json", + ); + assert.equal(run.workflowVersion, "deep-security-scan/v2"); + assert.deepEqual(run.executionSettings, { version: 1, settings: saved }); + const snapshot = await readFile(snapshotPath, "utf8"); + const expectedProvider = + name === "first" + ? undefined + : { + "amazon-bedrock": { + aws: { region: "us-west-2", profile: "fixture-profile" }, + }, + }; + assert.deepEqual( + JSON.parse(snapshot).settings.providerConfig, + expectedProvider, + "recorded provider selections need no persisted catalog definitions", + ); + const claim = await store.claimCoordinator({ + scanId: run.scanId, + threadId: beginInput.threadId, + }); + assert.equal(claim.acquired, true); + assert.deepEqual( + await loadDeepScanExecutionSettings(recordedScanDir, claim.run), + saved, + ); + assert.deepEqual( + (await store.get(run.scanId, beginInput.threadId)).executionSettings, + run.executionSettings, + ); + const observer = await new WorkbenchDeepScanStore(runWorkbench).begin({ + ...beginInput, + executionSettings: null, + model: "observer-model", + reasoningEffort: "low", + }); + assert.equal(observer.startDisposition, "joined"); + assert.equal(await readFile(snapshotPath, "utf8"), snapshot); + assert.deepEqual(observer.executionSettings, run.executionSettings); + assert.equal(observer.model, settings.model); + assert.equal(snapshot.includes("synthetic-"), false); + const runtimeEnvironment: NodeJS.ProcessEnv = { ...codexOptions.env }; + const restored = restoredDeepScanWorkerSettings( + saved, + currentParentSandbox, + () => runtimeEnvironment, + ); + restored.codexOptions.baseUrl = codexOptions.baseUrl; + return { + name, + fixture, + run, + readRun: async (): Promise< + import("../src/deep-scan/types.js").DeepScanRunState + > => + ( + await new WorkbenchDeepScanStore(runWorkbench).claimCoordinator({ + scanId: run.scanId, + threadId: beginInput.threadId, + }) + ).run, + recordedScanDir, + currentParentSandbox, + config, + configPath, + promptPath, + settings, + runtimeEnvironment, + snapshotPath, + snapshot, + providerKeys, + expectedProvider, + executor: new CodexSdkWorkerExecutor(restored), + }; + } + try { + for (const name of ["first", "second"]) + scans.push(await prepareReconstructedScan(name)); + childProcess.spawn = (( + command: string, + args: readonly string[] = [], + options: SpawnOptions = {}, + ) => { + const scan = scans.find( + (scan) => options?.env?.FAKE_CODEX_MARKER === scan.fixture.markerPath, + ); + return originalSpawn( + command, + scan ? [scan.fixture.executablePath, ...args] : args, + options, + ); + }) as unknown as typeof childProcess.spawn; + syncBuiltinESMExports(); + + for (const phase of [ + "fresh", + "resume", + "reconstructed-fresh", + "reconstructed", + "incomplete", + ]) { + if (phase.startsWith("reconstructed") || phase === "incomplete") { + for (const scan of scans) { + scan.run = await scan.readRun(); + // The caller restores recorded selections. Its old config file need + // not exist; current credentials still come from the selected home/env. + if (phase === "reconstructed-fresh") await rm(scan.configPath); + if (phase.startsWith("reconstructed")) { + // The managed parent can edit its output files. Neither a substituted + // executable/home nor other settings in that file are launch authority. + const rewritten = JSON.parse(scan.snapshot); + rewritten.settings.codexPath = process.execPath; + rewritten.settings.codexHome = scans.find( + (other) => other !== scan, + )!.settings.codexOptions.env.CODEX_HOME; + await writeFile(scan.snapshotPath, JSON.stringify(rewritten)); + } + if (phase === "incomplete") { + const saved = JSON.parse(scan.snapshot); + for (const key of ["model", "reasoningEffort", "reasoningSummary"]) + delete saved.settings[key]; + // Native history restores the first provider. The second snapshot + // retains the provider binding for its saved AWS selectors. + if (scan.name === "first") delete saved.settings.modelProvider; + if (scan.name === "first") delete saved.settings.serviceTier; + await writeFile(scan.snapshotPath, JSON.stringify(saved)); + // Emulate an older trusted record with missing optional selections. + scan.run.executionSettings = saved; + } + const snapshotBeforeRead = await readFile(scan.snapshotPath, "utf8"); + const recorded = await loadDeepScanExecutionSettings( + scan.recordedScanDir, + { + ...scan.run, + ...scan.settings, + createdAt: "2026-01-01T00:01:00Z", + }, + ); + const restored = restoredDeepScanWorkerSettings( + recorded, + scan.currentParentSandbox, + () => scan.runtimeEnvironment, + ); + restored.codexOptions.baseUrl = scan.settings.codexOptions.baseUrl; + scan.executor = new CodexSdkWorkerExecutor(restored); + assert.equal( + await readFile(scan.snapshotPath, "utf8"), + snapshotBeforeRead, + "restoring original worker selections must not rewrite saved settings", + ); + } + } + for (const scan of scans) { + scan.runtimeEnvironment.CODEX_API_KEY = `synthetic-${scan.name}-${phase}`; + for (const key of scan.providerKeys) + scan.runtimeEnvironment[key] = + `synthetic-${scan.name}-${phase}-${key}`; + scan.runtimeEnvironment.FAKE_CODEX_SCAN_VALUE = `${scan.name}-${phase}`; + scan.runtimeEnvironment.CODEX_HOME = path.join( + scan.fixture.root, + "observer-home", + ); + scan.runtimeEnvironment.CODEX_CLI_PATH = path.join( + scan.fixture.root, + "observer-codex", + ); + } + for (const kind of ["discovery", "dedup"]) { + const launches = await Promise.allSettled( + scans.map(async (scan) => { + const resumeThreadId = ["fresh", "reconstructed-fresh"].includes( + phase, + ) + ? undefined + : `fixture-${scan.name}-resumed`; + const result = await scan.executor.run({ + kind, + promptPath: scan.promptPath, + workingDirectory: scan.fixture.root, + subagents: scan.name === "first" ? 0 : 2, + resumeThreadId, + continuationPrompt: + "CAPTURE_SYNTHETIC_OPENAI_AUTH CAPTURE_SYNTHETIC_PROVIDER_AUTH NULL_USAGE continuation", + signal: new AbortController().signal, + }); + assert.equal( + result.threadId, + resumeThreadId ?? "fixture-thread-id", + ); + const child = JSON.parse( + await readFile(scan.fixture.markerPath, "utf8"), + ); + const preflight = JSON.parse( + await readFile(scan.fixture.preflightMarkerPath, "utf8"), + ); + assert.equal( + await realpath(child.executable), + await realpath(scan.settings.codexOptions.codexPathOverride), + ); + assert.equal( + child.codexCliPath, + scan.settings.codexOptions.codexPathOverride, + ); + assert.equal( + child.codexHome, + scan.settings.codexOptions.env.CODEX_HOME, + ); + assert.equal(preflight.codexHome, child.codexHome); + assert.equal(child.scanValue, `${scan.name}-${phase}`); + assert.equal( + child.configPath, + scan.runtimeEnvironment.CODEX_SECURITY_CONFIG_PATH, + ); + assert.deepEqual(child.openaiAuthentication, { + CODEX_API_KEY: `synthetic-${scan.name}-${phase}`, + }); + assert.deepEqual( + child.providerAuthentication, + Object.fromEntries( + scan.providerKeys.map((key) => [ + key, + `synthetic-${scan.name}-${phase}-${key}`, + ]), + ), + ); + assertFlagPair(child.argv, "--model", scan.settings.model); + assertFlagPair( + child.argv, + "--cyber-access-program", + scan.name === "first" ? "daybreak_blue" : "standard", + ); + assertConfigOverrides(child.argv, { + "features.api_key_cyber_access_programs": scan.name === "first", + "features.api_key_model_discovery": scan.name !== "first", + }); + for (const key of [ + "model_provider", + "model_reasoning_summary", + "service_tier", + ]) { + const override = `${key}=${JSON.stringify(scan.config[key])}`; + assert.equal(child.argv.includes(override), true, override); + assert.equal(preflight.argv.includes(override), true, override); + } + assert.equal( + child.argv.includes('model_reasoning_effort="ultra"'), + true, + ); + assert.equal( + preflight.argv.includes('model_reasoning_effort="ultra"'), + true, + ); + assert.equal( + preflight.argv.includes( + `model=${JSON.stringify(scan.settings.model)}`, + ), + true, + ); + const baseUrl = `openai_base_url=${JSON.stringify(scan.settings.codexOptions.baseUrl)}`; + assert.equal(child.argv.includes(baseUrl), true); + assert.equal(preflight.argv.includes(baseUrl), true); + for (const launch of [child, preflight]) { + const provider = launch.argv.filter((argument: string) => + /^model_providers[.=]/u.test(argument), + ); + assert.ok( + provider.length > 0, + "both preflight and worker launch receive provider configuration", + ); + const providers = parseToml(provider.join("\n")) + .model_providers as Record>; + if (scan.expectedProvider) + assert.deepEqual( + JSON.parse(JSON.stringify(providers)), + scan.expectedProvider, + ); + else + assert.deepEqual(Object.keys(providers.openrouter).sort(), [ + "base_url", + "env_key", + "name", + "wire_api", + ]); + assert.equal( + workerPermissionProfileOverride(launch.argv).includes( + "glob_scan_max_depth", + ), + false, + "a resumed bounded cap must not truncate an original uncapped deny glob, in either order", + ); + } + assertReadOnlyWorkerPolicy(child.argv); + assertWorkerSubagentPolicy( + child.argv, + scan.name === "first" ? 0 : 2, + ); + assert.equal( + workerPermissionProfileOverride(child.argv).includes( + '"/repo/.env"="deny"', + ), + true, + ); + assert.equal( + child.argv.includes("resume"), + resumeThreadId !== undefined, + ); + assert.equal( + child.stdin.includes("continuation"), + resumeThreadId !== undefined, + ); + }), + ); + for (const [index, launch] of launches.entries()) { + if (launch.status === "rejected") { + launchFailures.push( + `${scans[index].name}/${phase}/${kind}: ${launch.reason.message}`, + ); + } + } + } + if (phase === "fresh") { + for (const scan of scans) { + const replacementConfig = path.join( + scan.fixture.root, + "replacement config.toml", + ); + await writeFile( + replacementConfig, + 'model_provider = "changed-provider"\nmodel_reasoning_summary = "detailed"\n[codex_security]\ncyber_access_program = "daybreak_red"\n[features]\napi_key_cyber_access_programs = false\napi_key_model_discovery = false\n', + ); + scan.runtimeEnvironment.CODEX_SECURITY_CONFIG_PATH = + replacementConfig; + } + } + } + assert.deepEqual( + launchFailures, + [], + "every actual preflight and worker must retain the original launch selection", + ); + } finally { + childProcess.spawn = originalSpawn; + syncBuiltinESMExports(); + restoreEnv("FAKE_CODEX_MARKER", previousMarker); + } +} + +async function testWorkerProviderSelection() { + const fixture = await fakeCodexFixture(); + const saved = Object.fromEntries( + [ + "CODEX_CLI_PATH", + "CODEX_SECURITY_CONFIG_PATH", + "OPENAI_API_KEY", + "CODEX_API_KEY", + ].map((name) => [name, process.env[name]]), + ); + const originalSpawn = childProcess.spawn; + try { + delete process.env.OPENAI_API_KEY; + delete process.env.CODEX_API_KEY; + const configPath = path.join(fixture.root, "scan config.toml"); + const promptPath = path.join(fixture.root, "prompt.md"); + await writeFile(configPath, 'model_provider = "fixture-provider"\n'); + await writeFile(promptPath, "fixture provider selection"); + process.env.CODEX_CLI_PATH = process.execPath; + process.env.CODEX_SECURITY_CONFIG_PATH = configPath; + childProcess.spawn = (( + command: string, + args: readonly string[] = [], + options: SpawnOptions = {}, + ) => + originalSpawn( + command, + command === process.execPath || + command === path.toNamespacedPath(process.execPath) + ? [fixture.executablePath, ...args] + : args, + options, + )) as unknown as typeof childProcess.spawn; + syncBuiltinESMExports(); + const executor = new CodexSdkWorkerExecutor({ + parentSandbox: trustedParentSandbox, + }); + for (const kind of ["discovery", "dedup"]) { + await executor.run({ + kind, + promptPath, + workingDirectory: fixture.root, + subagents: 0, + signal: new AbortController().signal, + }); + const invocation = JSON.parse(await readFile(fixture.markerPath, "utf8")); + assert.equal( + invocation.argv.includes('model_provider="fixture-provider"'), + true, + ); + } + } finally { + childProcess.spawn = originalSpawn; + syncBuiltinESMExports(); + for (const [name, value] of Object.entries(saved)) restoreEnv(name, value); + } +} + +async function testNullUsageCompletion() { + const fixture = await fakeCodexFixture(); + const previousPath = process.env.CODEX_CLI_PATH; + process.env.CODEX_CLI_PATH = fixture.executablePath; + try { + const promptPath = path.join(fixture.root, "prompt.md"); + await writeFile(promptPath, "NULL_USAGE\n"); + for (const kind of ["discovery", "dedup"]) { + for (const resumeThreadId of [undefined, "fixture-resumed-thread"]) { + const result = await new CodexSdkWorkerExecutor({ + parentSandbox: trustedParentSandbox, + }).run({ + kind, + promptPath, + workingDirectory: fixture.root, + subagents: 0, + resumeThreadId, + signal: new AbortController().signal, + }); + assert.equal(result.threadId, resumeThreadId ?? "fixture-thread-id"); + assert.ok(result.threadId, "null-usage completion retains its thread"); + } + } + } finally { + restoreEnv("CODEX_CLI_PATH", previousPath); + } +} + +async function testReducerCoveragePersistenceBinding() { + const fixture = await fakeCodexFixture(); + const previousPath = process.env.CODEX_CLI_PATH; + const previousMarker = process.env.FAKE_CODEX_MARKER; + process.env.CODEX_CLI_PATH = fixture.executablePath; + try { + const promptPath = path.join(fixture.root, "prompt.md"); + const workingDirectory = path.join(fixture.root, "artifacts"); + await mkdir(workingDirectory); + await writeFile(promptPath, "fixture reducer prompt\n"); + const launches: Promise[] = []; + for (const resume of [false, true]) { + for (const persistSourceCoverage of [false, true]) { + const markerPath = path.join( + fixture.root, + `coverage-${resume}-${persistSourceCoverage}.json`, + ); + process.env.FAKE_CODEX_MARKER = markerPath; + const scanRoot = path.join( + fixture.root, + `scan-${resume}-${persistSourceCoverage}`, + ); + const deepReducer = { + scanRoot, + claimedWorkers: [ + { + id: "worker-1", + attempt: 2, + resultPath: path.join( + scanRoot, + "worker", + "checkpoints", + "accepted.json", + ), + artifactDir: path.join(scanRoot, "worker"), + }, + ], + persistSourceCoverage, + }; + const launch = new CodexSdkWorkerExecutor({ + parentSandbox: trustedParentSandbox, + artifactContext: { + pluginRoot: fixture.root, + scanRoot: deepReducer.scanRoot, + repoRoot: fixture.root, + scanId: "fixture-scan-id", + }, + }).run({ + kind: "dedup", + promptPath, + workingDirectory, + subagents: 0, + signal: new AbortController().signal, + ...(resume + ? { + resumeThreadId: "fixture-existing-thread", + continuationPrompt: "continue the reducer\n", + } + : {}), + artifactContext: { + root: workingDirectory, + layout: "reducer", + deepReducer, + }, + }); + launches.push( + launch.then(async () => { + const invocation = JSON.parse(await readFile(markerPath, "utf8")); + const prefix = + "mcp_servers.cs_artifacts.env.CODEX_SECURITY_REDUCER_CONTEXT_JSON="; + const encoded = invocation.argv.find((arg: string) => + arg.startsWith(prefix), + ); + assert.ok( + encoded, + "the launched reducer receives its host-bound artifact context", + ); + assert.deepEqual( + JSON.parse(JSON.parse(encoded.slice(prefix.length))), + deepReducer, + ); + }), + ); + } + } + await Promise.all(launches); + } finally { + restoreEnv("CODEX_CLI_PATH", previousPath); + restoreEnv("FAKE_CODEX_MARKER", previousMarker); + } +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_finalization.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_finalization.mjs new file mode 100644 index 0000000000..57574cd582 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_finalization.mjs @@ -0,0 +1,198 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { + mkdir, + mkdtemp, + readFile, + readdir, + realpath, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { build } from "esbuild"; + +const bundle = await build({ + stdin: { + contents: `export * from "./src/deep-scan/finalization.ts"; + export * from "./src/deep-scan/artifacts.ts"; + export * from "./src/artifact-scan-draft.ts";`, + resolveDir: path.resolve(import.meta.dirname, ".."), + }, + bundle: true, + format: "esm", + platform: "node", + write: false, + footer: { js: "//# sourceURL=deep-scan-finalization-contract.js" }, +}); +const { + publishSelectedDeepScan, + readSelectedDeepScanDraft, + createDeepScanArtifacts, + saveScanDraftCheckpoint, +} = await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}` +); +const scanId = "4e7b4acb-ac80-4d68-98cd-3d5ac5581cd1"; + +for (const terminalReason of ["capped", "saturated"]) { + test(`replays the selected ${terminalReason} input after replaceable results change`, async () => { + const scanDir = await realpath( + await mkdtemp(path.join(tmpdir(), "selected-finalization-")), + ); + try { + const artifacts = createDeepScanArtifacts(scanDir); + const root = path.join(artifacts.dedupRoot, "dedup-0001", "output"); + await mkdir(root, { recursive: true }); + const draft = { + scanId, + complete: true, + findings: [], + coverage: { + completeness: "partial", + surfaces: [], + explicitExclusions: [], + deferred: [ + { id: "review", reason: "A dependency remains unreviewed." }, + ], + }, + }; + const { coverage, ...reduction } = draft; + await saveScanDraftCheckpoint( + { root, repoRoot: scanDir, layout: "reducer" }, + { + ...reduction, + sourceCoverage: coverage, + }, + ); + const [name] = await readdir(path.join(root, "checkpoints")); + const checkpoint = path.join(root, "checkpoints", name); + const contents = await readFile(checkpoint); + const selection = { + version: 1, + resultPath: path.relative(scanDir, checkpoint), + resultSha256: createHash("sha256").update(contents).digest("hex"), + terminalReason, + omittedWorkerIds: [], + selectedAt: "2026-01-01T00:00:00Z", + }; + // A replacement result is not a new finalization selection. + await writeFile( + path.join(root, "result.json"), + JSON.stringify({ ...draft, complete: false }), + ); + assert.deepEqual( + await readSelectedDeepScanDraft(artifacts, scanId, selection), + draft, + ); + assert.deepEqual( + await readSelectedDeepScanDraft(artifacts, scanId, selection), + draft, + ); + await assert.rejects( + readSelectedDeepScanDraft( + artifacts, + "e14e9229-653a-4385-bec0-8745f0b037cb", + selection, + ), + /complete result for this scan/, + ); + await writeFile(checkpoint, JSON.stringify({ ...draft, findings: [] })); + await assert.rejects( + readSelectedDeepScanDraft(artifacts, scanId, selection), + /changed after acceptance/, + ); + } finally { + await rm(scanDir, { recursive: true, force: true }); + } + }); +} + +test("recreates only partial coverage for a persisted zero-success deadline selection", async () => { + const selection = { + version: 1, + resultPath: null, + resultSha256: null, + terminalReason: "capped", + omittedWorkerIds: [], + selectedAt: "2026-01-01T00:00:00Z", + }; + const result = await readSelectedDeepScanDraft( + createDeepScanArtifacts("unused"), + scanId, + selection, + ); + assert.equal(result.scanId, scanId); + assert.deepEqual(result.findings, []); + assert.equal(result.coverage.completeness, "partial"); + assert.equal(result.coverage.deferred.length, 1); + await assert.rejects( + readSelectedDeepScanDraft(createDeepScanArtifacts("unused"), scanId, { + ...selection, + terminalReason: "saturated", + }), + /recorded discovery deadline/, + ); +}); + +for (const status of ["failed", "canceled", "interrupted"]) { + test(`saved selection does not turn a ${status} scan into success`, async () => { + await assert.rejects( + publishSelectedDeepScan({ + run: { + scanId, + scanDir: "unused", + workflowVersion: "deep-security-scan/v2", + status, + finalizationInput: { + version: 1, + resultPath: null, + resultSha256: null, + terminalReason: "capped", + omittedWorkerIds: [], + selectedAt: "2026-01-01T00:00:00Z", + }, + }, + artifacts: createDeepScanArtifacts("unused"), + signal: new AbortController().signal, + publish: async () => + assert.fail("Stopped work cannot publish successful results"), + finish: async () => + assert.fail("Stopped work cannot finish successfully"), + }), + /Stopped Deep Scan/, + ); + }); +} + +test("cancellation prevents selected publication and preserves its input", async () => { + const controller = new AbortController(); + controller.abort("cost limit or user cancellation"); + const selection = { + version: 1, + resultPath: null, + resultSha256: null, + terminalReason: "capped", + omittedWorkerIds: [], + selectedAt: "2026-01-01T00:00:00Z", + }; + await assert.rejects( + publishSelectedDeepScan({ + run: { + scanId, + scanDir: "unused", + workflowVersion: "deep-security-scan/v2", + status: "running", + finalizationInput: selection, + }, + artifacts: createDeepScanArtifacts("unused"), + signal: controller.signal, + publish: async () => assert.fail("Canceled work cannot publish"), + finish: async () => assert.fail("Canceled work cannot finish"), + }), + (error) => error === controller.signal.reason, + ); + assert.equal(selection.terminalReason, "capped"); +}); diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_material_coverage.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_material_coverage.mjs new file mode 100644 index 0000000000..67bc086266 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_material_coverage.mjs @@ -0,0 +1,78 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { + materialRemediations, + materialRemediationTests, + publishCoverageFixture, +} from "./deep_scan_coverage_fixture.mjs"; + +for (const [resume, continueAfterResume] of [ + [false, false], + [true, false], + [true, true], +]) { + test(`material fixes and unresolved coverage survive canonical publication (resume: ${resume}, continued: ${continueAfterResume})`, async () => { + const root = await mkdtemp(path.join(tmpdir(), "deep-material-coverage-")); + try { + const fixture = path.join(root, "fixture"); + await mkdir(fixture, { mode: 0o700 }); + const { scanDir } = await publishCoverageFixture(fixture, "partial", { + resume, + continueAfterResume, + immutableInputs: true, + materialFindings: true, + }); + const { findings } = JSON.parse( + await readFile(path.join(scanDir, "findings.json"), "utf8"), + ); + assert.equal(findings.length, 1); + const sources = findings[0].provenance.sourceFindings; + assert.deepEqual( + sources.map((source) => source.finding.remediation), + materialRemediations, + ); + assert.equal(new Set(sources.map((source) => source.id)).size, 2); + const report = await readFile(path.join(scanDir, "report.md"), "utf8"); + for (const text of [ + ...materialRemediations, + ...materialRemediationTests, + ]) { + assert.equal( + report.split(text).length - 1, + 1, + `canonical report retains ${text}`, + ); + } + const coverage = JSON.parse( + await readFile(path.join(scanDir, "coverage.json"), "utf8"), + ); + assert.equal(coverage.completeness, "partial"); + assert.deepEqual( + coverage.reviews.map((review) => review.completeness), + ["partial", "complete", "unknown"], + ); + assert.equal( + coverage.deferred.length, + 2, + "a finding does not discharge independent unresolved work", + ); + assert.equal( + new Set(coverage.deferred.map((item) => item.candidateId)).size, + 2, + ); + for (const item of coverage.deferred) + assert.ok(report.includes(item.reason)); + for (const surface of coverage.surfaces) { + assert.equal( + await readFile(path.join(scanDir, surface.receiptRefs[0]), "utf8"), + "Synthetic review evidence.\n", + ); + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_recorded_recovery.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_recorded_recovery.mjs new file mode 100644 index 0000000000..4f895afeb2 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_recorded_recovery.mjs @@ -0,0 +1,64 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { + materialRemediations, + materialRemediationTests, + publishCoverageFixture, +} from "./deep_scan_coverage_fixture.mjs"; + +for (const [continueAfterResume, legacyAttempts, splitSeededReducers] of [ + [false, false, false], + [true, false, false], + [true, true, false], + [false, false, true], + [false, true, true], +]) { + test(`recorded inputs preserve fixes and coverage after mutable outputs disappear (continued: ${continueAfterResume}, legacy attempts: ${legacyAttempts}, reducer chain: ${splitSeededReducers})`, async () => { + const root = await mkdtemp( + path.join(tmpdir(), "recorded-coverage-recovery-"), + ); + try { + const fixture = path.join(root, "fixture"); + await mkdir(fixture, { mode: 0o700 }); + const { scanDir } = await publishCoverageFixture(fixture, "partial", { + resume: true, + continueAfterResume, + legacyAttempts, + splitSeededReducers, + immutableInputs: true, + materialFindings: true, + discardMutableResults: true, + }); + const report = await readFile(path.join(scanDir, "report.md"), "utf8"); + for (const fix of [ + ...materialRemediations, + ...materialRemediationTests, + ]) { + assert.equal(report.split(fix).length - 1, 1); + } + const coverage = JSON.parse( + await readFile(path.join(scanDir, "coverage.json"), "utf8"), + ); + assert.equal(coverage.completeness, "partial"); + assert.deepEqual( + coverage.reviews.map((review) => review.completeness), + ["partial", "complete", "unknown"], + ); + assert.equal( + new Set(coverage.deferred.map((item) => item.candidateId)).size, + 2, + ); + for (const surface of coverage.surfaces) { + assert.equal( + await readFile(path.join(scanDir, surface.receiptRefs[0]), "utf8"), + "Synthetic review evidence.\n", + ); + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_recovery_settings.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_recovery_settings.mjs new file mode 100644 index 0000000000..61ae1747a3 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_recovery_settings.mjs @@ -0,0 +1,962 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { syncBuiltinESMExports } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { PassThrough } from "node:stream"; +import { fileURLToPath } from "node:url"; +import { build } from "esbuild"; + +const bundle = await build({ + bundle: true, + nodePaths: [fileURLToPath(new URL("../node_modules", import.meta.url))], + define: { + "import.meta.url": JSON.stringify( + new URL("../src/deep-scan/recovery-settings.ts", import.meta.url).href, + ), + }, + entryPoints: [ + new URL("../src/deep-scan/recovery-settings.ts", import.meta.url).pathname, + ], + format: "esm", + platform: "node", + write: false, +}); +const { + captureDeepScanExecutionSettings: captureSettings, + restoredDeepScanWorkerSettings: restoreSettings, + loadDeepScanExecutionSettings: loadSettings, +} = await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}` +); +const root = await mkdtemp(join(tmpdir(), "deep-settings-")); +try { + const settings = { + codexPath: "/fixture/runtime/codex", + codexHome: "/fixture/account", + model: "fixture-model", + modelProvider: "fixture-provider", + reasoningEffort: "high", + reasoningSummary: "detailed", + serviceTier: "fast", + }; + const globSandbox = (depth) => ({ + filesystemDenies: ["/fixture/**/*.secret"], + ...(depth === undefined ? {} : { globScanMaxDepth: depth }), + }); + for (const [originalDepth, currentDepth, expectedDepth] of [ + [2, 5, 5], + [5, 2, 5], + [undefined, 2, undefined], + [2, undefined, undefined], + ]) { + const restored = restoreSettings( + { ...settings, parentSandbox: globSandbox(originalDepth) }, + globSandbox(currentDepth), + ); + assert.equal( + restored.parentSandbox.globScanMaxDepth, + expectedDepth, + `deny expansion must preserve both policies: ${originalDepth}, ${currentDepth}`, + ); + } + assert.equal( + restoreSettings(settings, globSandbox(2)).parentSandbox.globScanMaxDepth, + 2, + "unavailable historical policy does not establish uncapped glob expansion", + ); + assert.equal( + restoreSettings( + { ...settings, parentSandbox: globSandbox(2) }, + { + filesystemDenies: ["/fixture/exact-denial"], + }, + ).parentSandbox.globScanMaxDepth, + 2, + "exact denials do not change glob expansion", + ); + const writeSnapshot = async (directory, value) => { + const path = join( + directory, + "artifacts", + "deep_discovery", + "execution-settings.json", + ); + await mkdir(join(directory, "artifacts", "deep_discovery"), { + recursive: true, + }); + await writeFile( + path, + JSON.stringify({ version: 1, settings: value }, null, 2) + "\n", + ); + }; + const boundRun = (value) => ({ + workflowVersion: "deep-security-scan/v2", + executionSettings: { version: 1, settings: value }, + }); + await assert.rejects( + loadSettings(join(root, "missing")), + /no recorded original execution settings/, + ); + await writeSnapshot(join(root, "one"), settings); + await writeSnapshot(join(root, "two"), { ...settings, model: "other-model" }); + const savedPath = join( + root, + "one", + "artifacts", + "deep_discovery", + "execution-settings.json", + ); + const saved = await readFile(savedPath, "utf8"); + const [recovered, concurrent] = await Promise.all([ + loadSettings(join(root, "one"), boundRun(settings)), + loadSettings( + join(root, "two"), + boundRun({ ...settings, model: "other-model" }), + ), + ]); + assert.deepEqual(recovered, settings); + assert.equal(concurrent.model, "other-model"); + assert.equal(await readFile(savedPath, "utf8"), saved); + recovered.model = "caller-mutation"; + assert.deepEqual( + await loadSettings(join(root, "one"), boundRun(settings)), + settings, + ); + await writeSnapshot(join(root, "one"), { + ...settings, + codexPath: "/fixture/substituted/codex", + codexHome: "/fixture/substituted/home", + }); + assert.deepEqual( + await loadSettings(join(root, "one"), boundRun(settings)), + settings, + ); + await assert.rejects( + loadSettings(join(root, "one")), + /no recorded original execution settings/, + ); + + const configPath = join(root, "runtime.toml"); + await writeFile( + configPath, + `model = "inherited-model" +model_provider = "custom" +profile = "scan" +[profiles.scan] +model_reasoning_summary = "concise" +service_tier = "flex" +[model_providers.custom] +name = "Fixture" +http_headers = { Authorization = "synthetic-secret" } +`, + ); + const captured = await captureSettings( + { model: "original-model", reasoningEffort: "ultra" }, + { + filesystemDenies: ["/fixture/original-deny"], + globScanMaxDepth: 3, + }, + { + CODEX_CLI_PATH: process.execPath, + CODEX_HOME: root, + CODEX_SECURITY_CONFIG_PATH: configPath, + }, + ); + assert.equal(captured.model, "original-model"); + assert.equal(captured.modelProvider, "custom"); + assert.equal(captured.reasoningSummary, "concise"); + assert.equal(captured.serviceTier, "flex"); + assert.equal(captured.providerConfig, undefined); + assert.equal(JSON.stringify(captured).includes("synthetic-secret"), false); + for (const modelProvider of ["openrouter", "fireworks", "amazon-bedrock"]) { + await writeFile( + configPath, + `model_provider = ${JSON.stringify(modelProvider)} +[model_providers.${modelProvider}.aws] +region = "us-west-2" +profile = "fixture-profile" +access_key_id = "synthetic-secret" +`, + ); + const selected = await captureSettings( + {}, + { filesystemDenies: [] }, + { + CODEX_CLI_PATH: process.execPath, + CODEX_HOME: root, + CODEX_SECURITY_CONFIG_PATH: configPath, + }, + ); + assert.equal(selected.modelProvider, modelProvider); + const expectedProvider = + modelProvider === "amazon-bedrock" + ? { + "amazon-bedrock": { + aws: { region: "us-west-2", profile: "fixture-profile" }, + }, + } + : undefined; + assert.deepEqual( + selected.providerConfig, + expectedProvider, + "saved selections exclude catalog definitions and retain Bedrock selectors", + ); + const providerDir = join(root, modelProvider); + await writeSnapshot(providerDir, selected); + const path = join( + providerDir, + "artifacts", + "deep_discovery", + "execution-settings.json", + ); + const bytes = await readFile(path, "utf8"); + assert.equal(bytes.includes("synthetic-secret"), false); + const restoredProvider = restoreSettings( + await loadSettings(providerDir, boundRun(selected)), + { filesystemDenies: [] }, + ).codexOptions.config.model_providers; + if (expectedProvider) assert.deepEqual(restoredProvider, expectedProvider); + else + assert.deepEqual(Object.keys(restoredProvider[modelProvider]).sort(), [ + "base_url", + "env_key", + "name", + "wire_api", + ]); + assert.equal(await readFile(path, "utf8"), bytes); + // Older snapshots can contain catalog definitions. Reading them must not + // rewrite their bytes or prevent the existing launch projection. + if (!expectedProvider) { + await writeSnapshot(providerDir, { + ...selected, + providerConfig: restoredProvider, + }); + const legacyBytes = await readFile(path, "utf8"); + const legacy = await loadSettings( + providerDir, + boundRun({ ...selected, providerConfig: restoredProvider }), + ); + assert.equal(legacy.providerConfig, undefined); + assert.deepEqual( + restoreSettings(legacy, { filesystemDenies: [] }).codexOptions.config + .model_providers, + restoredProvider, + ); + assert.equal(await readFile(path, "utf8"), legacyBytes); + } + } + let credential = "synthetic-first"; + const restored = restoreSettings( + captured, + { filesystemDenies: ["/fixture/current-deny"] }, + () => ({ + CODEX_API_KEY: credential, + CODEX_HOME: "/fixture/observer-home", + CODEX_CLI_PATH: "/fixture/observer-codex", + }), + ); + assert.equal(restored.codexOptions.env.CODEX_API_KEY, "synthetic-first"); + credential = "synthetic-refreshed"; + assert.equal(restored.codexOptions.env.CODEX_API_KEY, "synthetic-refreshed"); + assert.equal(restored.codexOptions.env.CODEX_HOME, root); + assert.equal(restored.codexOptions.env.CODEX_CLI_PATH, captured.codexPath); + assert.deepEqual(restored.parentSandbox.filesystemDenies, [ + "/fixture/original-deny", + "/fixture/current-deny", + ]); + assert.equal(restored.codexOptions.config.model_reasoning_effort, "ultra"); + await writeFile(join(root, "config.toml"), 'model = "native-home-model"\n'); + const native = await captureSettings( + {}, + { filesystemDenies: [] }, + { CODEX_CLI_PATH: process.execPath, CODEX_HOME: root }, + ); + assert.equal(native.model, "native-home-model"); + const sessionDirectory = join(root, "sessions"); + await mkdir(sessionDirectory); + await writeFile( + join(sessionDirectory, "parent.jsonl"), + [ + { + type: "session_meta", + timestamp: "2026-01-01T00:00:00Z", + payload: { id: "fixture-parent", model_provider: "openai" }, + }, + { + type: "turn_context", + timestamp: "2026-01-01T00:00:01Z", + payload: { + turn_id: "original-turn", + model: "parent-model", + effort: "high", + summary: "none", + }, + }, + { + type: "turn_context", + timestamp: "2026-01-01T00:02:00Z", + payload: { + turn_id: "later-turn", + model: "later-model", + effort: "low", + summary: "detailed", + }, + }, + ] + .map(JSON.stringify) + .join("\n") + "\n", + ); + await writeFile( + join(sessionDirectory, "other.jsonl"), + JSON.stringify({ + type: "session_meta", + payload: { id: "fixture-other", model_provider: "other-provider" }, + }) + "\n", + ); + const parentSettings = await captureSettings( + {}, + { filesystemDenies: [] }, + { + CODEX_CLI_PATH: process.execPath, + CODEX_HOME: root, + }, + { threadId: "fixture-parent", startedAt: "2026-01-01T00:01:00Z" }, + ); + assert.equal( + parentSettings.model, + "native-home-model", + "explicit config retains precedence", + ); + assert.equal(parentSettings.modelProvider, "openai"); + assert.equal( + parentSettings.reasoningSummary, + "none", + "later owner turns are not original discovery settings", + ); + assert.equal(parentSettings.reasoningEffort, "high"); + await writeFile(join(root, "config.toml"), ""); + const parentEnvironment = { + CODEX_CLI_PATH: process.execPath, + CODEX_HOME: root, + }; + const [originalParent, otherParent, unavailableParent] = await Promise.all([ + captureSettings({}, { filesystemDenies: [] }, parentEnvironment, { + threadId: "fixture-parent", + startedAt: "2026-01-01T00:00:01Z", + }), + captureSettings({}, { filesystemDenies: [] }, parentEnvironment, { + threadId: "fixture-other", + startedAt: "2026-01-01T00:00:01Z", + }), + captureSettings( + { model: "stored-model", reasoningEffort: "ultra" }, + { filesystemDenies: [] }, + parentEnvironment, + { threadId: "fixture-unavailable", startedAt: "2026-01-01T00:00:01Z" }, + ), + ]); + assert.equal(originalParent.model, "parent-model"); + assert.equal( + originalParent.reasoningSummary, + "none", + "the original turn is included at its timestamp", + ); + assert.equal(otherParent.modelProvider, "other-provider"); + assert.equal( + otherParent.model, + undefined, + "concurrent scans do not borrow another parent's model", + ); + assert.equal(otherParent.reasoningSummary, undefined); + assert.equal(unavailableParent.model, "stored-model"); + assert.equal(unavailableParent.reasoningEffort, "ultra"); + assert.equal( + unavailableParent.modelProvider, + undefined, + "missing history does not establish a provider", + ); + assert.equal(unavailableParent.reasoningSummary, undefined); + assert.equal(unavailableParent.serviceTier, undefined); + assert.equal( + unavailableParent.nativeServiceTierAbsent, + undefined, + "missing history does not prove native absence", + ); + const originalOwner = { + threadId: "fixture-parent", + turnId: "original-turn", + startedAt: "2026-01-01T00:00:00Z", + }; + for (const workflowVersion of ["deep-security-scan/v1", "deep-scan-mcp/v1"]) { + const legacyDir = join(root, workflowVersion.replaceAll("/", "-")); + const legacy = await loadSettings( + legacyDir, + { + workflowVersion, + model: "recorded-model", + reasoningEffort: "ultra", + createdAt: "2026-01-01T00:01:00Z", + usageOwner: null, + }, + async () => ({ + config: { model_reasoning_summary: "concise", service_tier: "flex" }, + usageOwner: originalOwner, + }), + parentEnvironment, + ); + assert.equal(legacy.model, "recorded-model"); + assert.equal( + legacy.reasoningSummary, + "concise", + "recorded recipe retains precedence", + ); + assert.equal( + legacy.modelProvider, + "openai", + "recorded original owner supplies native selections", + ); + assert.equal(legacy.serviceTier, "flex"); + assert.equal( + legacy.codexPath, + undefined, + "legacy metadata did not record an executable", + ); + assert.equal( + legacy.codexHome, + undefined, + "a history lookup home is not recorded execution provenance", + ); + const restoredLegacy = restoreSettings( + legacy, + { filesystemDenies: ["/fixture/current-deny"] }, + () => ({ + CODEX_HOME: "/fixture/runtime-home", + CODEX_CLI_PATH: "/fixture/runtime-codex", + CODEX_API_KEY: "synthetic-live-key", + }), + ); + assert.equal( + restoredLegacy.codexOptions.env.CODEX_HOME, + "/fixture/runtime-home", + ); + assert.equal( + restoredLegacy.codexOptions.env.CODEX_CLI_PATH, + "/fixture/runtime-codex", + ); + assert.equal( + restoredLegacy.codexOptions.config.model_reasoning_summary, + "concise", + ); + await assert.rejects( + readFile( + join(legacyDir, "artifacts/deep_discovery/execution-settings.json"), + ), + { code: "ENOENT" }, + ); + } + await assert.rejects( + loadSettings( + join(root, "missing-v2"), + { workflowVersion: "deep-security-scan/v2" }, + async () => + assert.fail( + "missing promised v2 settings must not become legacy recovery", + ), + ), + /no recorded original/, + ); + const [rebound, unboundLegacy] = await Promise.all([ + captureSettings( + { usageOwner: originalOwner }, + { filesystemDenies: [] }, + parentEnvironment, + { threadId: "fixture-other", startedAt: "2026-01-01T00:03:00Z" }, + ), + captureSettings( + { model: "stored-model", usageOwner: null }, + { filesystemDenies: [] }, + parentEnvironment, + { threadId: "fixture-other", startedAt: "2026-01-01T00:03:00Z" }, + ), + ]); + assert.equal( + rebound.modelProvider, + "openai", + "takeover uses the recorded owner, not the invoking conversation", + ); + assert.equal( + rebound.model, + "parent-model", + "the bound turn takes precedence over later turns", + ); + assert.equal(rebound.reasoningSummary, "none"); + assert.equal(unboundLegacy.model, "stored-model"); + assert.equal( + unboundLegacy.modelProvider, + undefined, + "unrecorded legacy ownership cannot recover caller selections", + ); + assert.equal(unboundLegacy.reasoningSummary, undefined); + await writeFile( + join(sessionDirectory, "legacy-auto.jsonl"), + [ + { + type: "session_meta", + payload: { + id: "fixture-legacy-auto", + cli_version: "0.132.0", + model_provider: "openai", + }, + }, + { + type: "turn_context", + payload: { + turn_id: "legacy-turn", + model: "legacy-model", + effort: "high", + summary: "auto", + }, + }, + ] + .map(JSON.stringify) + .join("\n") + "\n", + ); + const legacyAuto = await captureSettings( + { usageOwner: { threadId: "fixture-legacy-auto", turnId: "legacy-turn" } }, + { filesystemDenies: [] }, + parentEnvironment, + ); + assert.equal( + legacyAuto.reasoningSummary, + "auto", + "older native turn-context selections remain readable", + ); + for (const version of ["0.133.0", "0.154.0"]) { + const threadId = `fixture-fresh-${version}`; + await writeFile( + join(sessionDirectory, `${threadId}.jsonl`), + [ + { + type: "session_meta", + timestamp: "2026-01-01T00:00:00Z", + payload: { + id: threadId, + cli_version: version, + model_provider: "openai", + }, + }, + { + type: "turn_context", + timestamp: "2026-01-01T00:00:01Z", + payload: { + turn_id: "fresh-turn", + model: "fresh-model", + effort: "high", + summary: "auto", + }, + }, + { + type: "event_msg", + timestamp: "2026-01-01T00:02:00Z", + payload: { + type: "thread_settings_applied", + thread_id: threadId, + thread_settings: { + model: "fresh-model", + model_provider_id: "openai", + reasoning_summary: "detailed", + }, + }, + }, + ] + .map(JSON.stringify) + .join("\n") + "\n", + ); + const owner = { + threadId, + turnId: "fresh-turn", + startedAt: "2026-01-01T00:01:00Z", + }; + const fresh = await captureSettings( + { usageOwner: owner }, + { filesystemDenies: [] }, + parentEnvironment, + ); + assert.equal(fresh.model, "fresh-model"); + assert.equal( + fresh.reasoningSummary, + undefined, + "fresh native compatibility auto is not an original selection", + ); + assert.equal( + restoreSettings(fresh, { filesystemDenies: [] }).codexOptions.config + .model_reasoning_summary, + undefined, + ); + const freshDir = join(root, threadId); + await writeSnapshot(freshDir, fresh); + const freshPath = join( + freshDir, + "artifacts", + "deep_discovery", + "execution-settings.json", + ); + const freshBytes = await readFile(freshPath, "utf8"); + assert.deepEqual( + await loadSettings(freshDir, { + ...boundRun(fresh), + usageOwner: owner, + createdAt: owner.startedAt, + }), + fresh, + ); + assert.equal( + await readFile(freshPath, "utf8"), + freshBytes, + "unknown summary is not replaced by a compatibility field or a later selection", + ); + await writeFile( + join(root, "config.toml"), + 'model_reasoning_summary = "auto"\n', + ); + const explicit = await captureSettings( + { usageOwner: owner }, + { filesystemDenies: [] }, + parentEnvironment, + ); + assert.equal( + explicit.reasoningSummary, + "auto", + "an explicit original config selection still takes precedence", + ); + await writeFile(join(root, "config.toml"), ""); + } + await writeFile( + join(sessionDirectory, "applied.jsonl"), + [ + { + type: "session_meta", + timestamp: "2026-01-01T00:00:00Z", + payload: { id: "fixture-applied", model_provider: "previous-provider" }, + }, + { + type: "event_msg", + timestamp: "2026-01-01T00:00:01Z", + payload: { + type: "thread_settings_applied", + thread_id: "fixture-applied", + thread_settings: { + model: "applied-model", + model_provider_id: "openai", + service_tier: "default", + reasoning_effort: "high", + reasoning_summary: "concise", + }, + }, + }, + { + type: "turn_context", + timestamp: "2026-01-01T00:00:02Z", + payload: { + turn_id: "applied-turn", + model: "previous-model", + effort: "low", + summary: "none", + }, + }, + { + type: "event_msg", + timestamp: "2026-01-01T00:00:03Z", + payload: { + type: "thread_settings_applied", + thread_id: "fixture-copied-owner", + thread_settings: { + model: "copied-model", + model_provider_id: "copied-provider", + service_tier: "flex", + reasoning_summary: "detailed", + }, + }, + }, + { + type: "event_msg", + timestamp: "2026-01-01T00:02:00Z", + payload: { + type: "thread_settings_applied", + thread_id: "fixture-applied", + thread_settings: { + model: "later-model", + model_provider_id: "later-provider", + service_tier: "fast", + reasoning_summary: "detailed", + }, + }, + }, + ] + .map(JSON.stringify) + .join("\n") + "\n", + ); + const appliedOwner = { + threadId: "fixture-applied", + turnId: "applied-turn", + startedAt: "2026-01-01T00:00:00Z", + }; + const applied = await captureSettings( + { usageOwner: appliedOwner }, + { filesystemDenies: [] }, + parentEnvironment, + { threadId: "fixture-other", startedAt: "2026-01-01T00:01:00Z" }, + ); + assert.equal( + applied.serviceTier, + "default", + "original explicit standard routing survives later and copied snapshots", + ); + assert.equal( + applied.reasoningSummary, + "concise", + "native applied summary overrides the legacy compatibility field", + ); + assert.equal( + applied.modelProvider, + "openai", + "complete native snapshot replaces the session metadata provider", + ); + assert.equal( + applied.model, + "applied-model", + "complete native snapshot replaces compatibility turn settings", + ); + assert.equal(applied.reasoningEffort, "high"); + assert.equal( + applied.nativeServiceTierAbsent, + undefined, + "explicit native standard remains an explicit selection", + ); + const tierDir = join(root, "missing-tier"); + const { serviceTier: omittedTier, ...withoutTier } = applied; + assert.equal(omittedTier, "default"); + await writeSnapshot(tierDir, withoutTier); + const repairedTier = await loadSettings(tierDir, { + ...boundRun(withoutTier), + usageOwner: appliedOwner, + createdAt: "2026-01-01T00:01:00Z", + }); + assert.equal(repairedTier.serviceTier, "default"); + await writeFile( + join(sessionDirectory, "applied.jsonl"), + (await readFile(join(sessionDirectory, "applied.jsonl"), "utf8")) + + JSON.stringify({ + type: "event_msg", + timestamp: "2026-01-01T00:00:04Z", + payload: { + type: "thread_settings_applied", + thread_id: "fixture-applied", + thread_settings: { + model: "applied-model", + model_provider_id: "openai", + reasoning_effort: "high", + service_tier: "priority", + }, + }, + }) + + "\n", + ); + const nativeTier = await captureSettings( + { usageOwner: appliedOwner }, + { filesystemDenies: [] }, + parentEnvironment, + { threadId: "fixture-other", startedAt: "2026-01-01T00:01:00Z" }, + ); + assert.equal( + nativeTier.serviceTier, + "priority", + "an effective tier selected by native remains unchanged", + ); + assert.equal(nativeTier.nativeServiceTierAbsent, undefined); + await writeFile( + join(sessionDirectory, "applied.jsonl"), + (await readFile(join(sessionDirectory, "applied.jsonl"), "utf8")) + + JSON.stringify({ + type: "event_msg", + timestamp: "2026-01-01T00:00:05Z", + payload: { + type: "thread_settings_applied", + thread_id: "fixture-applied", + thread_settings: { + model: "applied-model", + model_provider_id: "openai", + reasoning_effort: "high", + }, + }, + }) + + "\n", + ); + const nativeDefaults = await captureSettings( + { usageOwner: appliedOwner }, + { filesystemDenies: [] }, + parentEnvironment, + { threadId: "fixture-other", startedAt: "2026-01-01T00:01:00Z" }, + ); + assert.equal( + nativeDefaults.model, + "applied-model", + "absent optional selections do not erase the required model", + ); + assert.equal( + nativeDefaults.modelProvider, + "openai", + "absent optional selections do not erase the required provider", + ); + assert.equal(nativeDefaults.reasoningEffort, "high"); + assert.equal( + nativeDefaults.serviceTier, + "default", + "known native absence retains its omitted request tier", + ); + assert.equal( + nativeDefaults.nativeServiceTierAbsent, + true, + "known native absence is recorded separately from explicit standard", + ); + assert.equal( + nativeDefaults.reasoningSummary, + undefined, + "a compatibility summary is not a recorded native default", + ); + const incompleteDir = join(root, "incomplete"); + const incomplete = { + codexPath: process.execPath, + codexHome: root, + serviceTier: "flex", + }; + await writeSnapshot(incompleteDir, incomplete); + await writeFile( + join(root, "config.toml"), + 'model_provider = "observer-provider"\nmodel_reasoning_summary = "detailed"\n', + ); + const originalRun = { + ...boundRun(incomplete), + model: "stored-model", + reasoningEffort: "ultra", + usageOwner: originalOwner, + createdAt: "2026-01-01T00:01:00Z", + }; + const raceDir = join(root, "concurrent-recovery"); + await writeSnapshot(raceDir, incomplete); + const racePath = join( + raceDir, + "artifacts", + "deep_discovery", + "execution-settings.json", + ); + const originalCreateReadStream = fs.createReadStream; + const readingHistory = Promise.withResolvers(); + const releaseHistory = Promise.withResolvers(); + let held = false; + let pendingRead; + fs.createReadStream = (path, options) => { + const source = originalCreateReadStream(path, options); + if (path !== join(sessionDirectory, "parent.jsonl") || held) return source; + held = true; + const delayed = new PassThrough(); + source.once("error", (error) => delayed.destroy(error)); + delayed.once("close", () => source.destroy()); + void releaseHistory.promise.then(() => source.pipe(delayed)); + readingHistory.resolve(); + return delayed; + }; + syncBuiltinESMExports(); + try { + pendingRead = loadSettings(raceDir, originalRun); + await Promise.race([ + readingHistory.promise, + pendingRead.then(() => + assert.fail( + "historical recovery must reach the controlled history read", + ), + ), + ]); + const newer = { + ...settings, + modelProvider: "newer-provider", + reasoningSummary: "concise", + }; + await writeSnapshot(raceDir, newer); + const newerBytes = await readFile(racePath, "utf8"); + releaseHistory.resolve(); + const delayedProjection = await pendingRead; + assert.equal(delayedProjection.modelProvider, "openai"); + assert.equal(delayedProjection.reasoningSummary, "none"); + assert.equal( + await readFile(racePath, "utf8"), + newerBytes, + "a delayed historical projection must not overwrite a newer snapshot", + ); + assert.deepEqual(await loadSettings(raceDir, boundRun(newer)), newer); + } finally { + releaseHistory.resolve(); + await pendingRead?.catch(() => {}); + fs.createReadStream = originalCreateReadStream; + syncBuiltinESMExports(); + } + const incompletePath = join( + incompleteDir, + "artifacts", + "deep_discovery", + "execution-settings.json", + ); + const incompleteBytes = await readFile(incompletePath, "utf8"); + const repaired = await loadSettings(incompleteDir, originalRun); + assert.deepEqual(repaired, { + ...incomplete, + model: "stored-model", + reasoningEffort: "ultra", + modelProvider: "openai", + reasoningSummary: "none", + }); + assert.equal( + await readFile(incompletePath, "utf8"), + incompleteBytes, + "recovering historical fields is read-only", + ); + await rm(sessionDirectory, { recursive: true }); + const unavailable = await loadSettings(incompleteDir, originalRun); + assert.equal(unavailable.model, "stored-model"); + assert.equal(unavailable.reasoningEffort, "ultra"); + assert.equal( + unavailable.modelProvider, + undefined, + "unavailable history remains unknown", + ); + assert.equal(unavailable.reasoningSummary, undefined); + assert.equal(await readFile(incompletePath, "utf8"), incompleteBytes); + const unknownDir = join(root, "unknown"); + await writeSnapshot(unknownDir, incomplete); + const unknown = await loadSettings(unknownDir, { + ...originalRun, + usageOwner: null, + }); + assert.equal(unknown.model, "stored-model"); + assert.equal( + unknown.modelProvider, + undefined, + "missing original ownership is not current config", + ); + assert.equal(unknown.reasoningSummary, undefined); + assert.equal(unknown.nativeServiceTierAbsent, undefined); + const unsupported = JSON.stringify({ version: 99, settings }); + await writeFile(savedPath, unsupported); + await assert.rejects( + loadSettings(join(root, "one"), { + workflowVersion: "deep-security-scan/v2", + executionSettings: { version: 99, settings }, + }), + /unsupported/, + ); + assert.equal(await readFile(savedPath, "utf8"), unsupported); +} finally { + await rm(root, { recursive: true, force: true }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_coverage.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_coverage.mjs new file mode 100644 index 0000000000..ef5756cc2d --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_coverage.mjs @@ -0,0 +1,65 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { + materialRemediations, + materialRemediationTests, + publishCoverageFixture, +} from "./deep_scan_coverage_fixture.mjs"; + +for (const splitSeededReducers of [false, true]) { + test(`selected publication retains fixes and coverage after failure and a rejected checkpoint (reducer chain: ${splitSeededReducers})`, async () => { + const root = await mkdtemp( + path.join(tmpdir(), "selected-coverage-publication-"), + ); + try { + const fixture = path.join(root, "fixture"); + await mkdir(fixture, { mode: 0o700 }); + const { scanDir, terminal } = await publishCoverageFixture( + fixture, + "partial", + { + resume: true, + immutableInputs: true, + materialFindings: true, + discardMutableResults: true, + splitSeededReducers, + selectedRecovery: true, + }, + ); + assert.equal(terminal.workflowVersion, "deep-security-scan/v2"); + assert.match(terminal.finalizationInput.resultPath, /checkpoints/); + const report = await readFile(path.join(scanDir, "report.md"), "utf8"); + for (const fix of [ + ...materialRemediations, + ...materialRemediationTests, + ]) { + assert.equal(report.split(fix).length - 1, 1); + } + const coverage = JSON.parse( + await readFile(path.join(scanDir, "coverage.json"), "utf8"), + ); + assert.equal(coverage.completeness, "partial"); + assert.deepEqual( + coverage.reviews.map((review) => review.completeness), + ["partial", "complete", "unknown"], + ); + assert.equal( + new Set(coverage.deferred.map((item) => item.candidateId)).size, + 2, + ); + for (const item of coverage.deferred) + assert.ok(report.includes(item.reason)); + for (const surface of coverage.surfaces) { + assert.equal( + await readFile(path.join(scanDir, surface.receiptRefs[0]), "utf8"), + "Synthetic review evidence.\n", + ); + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_replay.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_replay.mjs new file mode 100644 index 0000000000..eebc660e50 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_replay.mjs @@ -0,0 +1,168 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { build } from "esbuild"; + +const bundle = await build({ + entryPoints: [ + path.resolve(import.meta.dirname, "../src/deep-scan/coordinator.ts"), + ], + loader: { ".md": "text" }, + bundle: true, + format: "esm", + platform: "node", + write: false, + footer: { js: "//# sourceURL=deep-scan-selected-replay.js" }, +}); +const { DeepScanCoordinator } = await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}` +); + +for (const terminalReason of ["saturated", "capped"]) { + test(`restart publishes the selected ${terminalReason} result after an output failure`, async () => { + const scanDir = await realpath( + await mkdtemp(path.join(tmpdir(), "selected-replay-")), + ); + try { + const scanId = "4e7b4acb-ac80-4d68-98cd-3d5ac5581cd1"; + const draft = { + scanId, + complete: true, + findings: [], + coverage: { + completeness: "partial", + surfaces: [], + explicitExclusions: [], + deferred: [ + { id: "review", reason: "A dependency remains unreviewed." }, + ], + }, + }; + const { coverage, ...reduction } = draft; + const bytes = JSON.stringify({ ...reduction, sourceCoverage: coverage }); + const digest = createHash("sha256").update(bytes).digest("hex"); + const resultPath = `artifacts/deep_discovery/dedup/dedup-0001/output/checkpoints/${digest}.json`; + await mkdir(path.dirname(path.join(scanDir, resultPath)), { + recursive: true, + }); + await writeFile(path.join(scanDir, resultPath), bytes); + const selection = { + version: 1, + resultPath, + resultSha256: digest, + terminalReason, + omittedWorkerIds: [], + selectedAt: "2026-01-01T00:00:00Z", + }; + let run = { + scanId, + scanDir, + targetPath: scanDir, + scope: ".", + workflowVersion: "deep-security-scan/v2", + status: "running", + phase: "terminal", + coordinatorGeneration: 3, + finalizationInput: selection, + terminalReason, + createdAt: "2026-01-01T00:00:00Z", + config: { + workers: 2, + subagents: 0, + stopAfterNoNew: 2, + stopAfterConsecutiveErrors: 2, + maxDiscoveryRuns: 4, + maxTimeHours: 1, + }, + dispatchedCount: 4, + noNewStreak: 2, + consecutiveErrors: 0, + }; + const mutations = []; + const store = new Proxy( + { + get: async () => structuredClone(run), + finish: async (input) => { + mutations.push("finish"); + assert.equal(input.reason, terminalReason); + assert.deepEqual( + input.omittedWorkerIds, + selection.omittedWorkerIds, + ); + run = { + ...run, + status: "succeeded", + manifestPath: input.manifestPath, + }; + return structuredClone(run); + }, + fail: async () => { + mutations.push("fail"); + run = { ...run, status: "failed" }; + return run; + }, + }, + { + get: (target, key) => + key in target + ? target[key] + : async () => { + mutations.push(key); + throw new Error( + `Unexpected scheduler operation: ${String(key)}`, + ); + }, + }, + ); + let executions = 0; + let publications = 0; + const options = { + store, + executor: { + run: async () => { + executions++; + throw new Error("Unexpected model work"); + }, + }, + pluginRoot: scanDir, + threadId: "original-result-conversation", + retryDelaysMs: [], + // Already expired: replay must not start a discovery deadline timer. + discoveryTimeoutMs: 1, + onComplete: async (actual, _signal, publication) => { + publications++; + assert.deepEqual(actual, draft); + assert.equal(publication.coordinatorGeneration, 3); + assert.equal(publication.resultPath, path.join(scanDir, resultPath)); + if (publications === 1) + throw new Error("Synthetic publication write failure"); + }, + }; + const first = new DeepScanCoordinator({ + ...options, + run: structuredClone(run), + }); + first.start(); + await assert.rejects(first.wait(), /Synthetic publication write failure/); + assert.equal(run.status, "running"); + assert.equal(run.terminalReason, terminalReason); + assert.deepEqual(run.finalizationInput, selection); + const restarted = new DeepScanCoordinator({ + ...options, + run: structuredClone(run), + }); + restarted.start(); + const completed = await restarted.wait(); + assert.equal(completed.status, "succeeded"); + assert.equal(completed.terminalReason, terminalReason); + assert.equal(executions, 0); + assert.deepEqual(mutations, ["finish"]); + assert.equal(publications, 2); + } finally { + await rm(scanDir, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_selection_store.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_selection_store.mjs new file mode 100644 index 0000000000..7939a49310 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_selection_store.mjs @@ -0,0 +1,88 @@ +import assert from "node:assert/strict"; +import path from "node:path"; +import { test } from "node:test"; +import { build } from "esbuild"; + +const bundle = await build({ + entryPoints: [path.resolve(import.meta.dirname, "../src/deep-scan/store.ts")], + bundle: true, + format: "esm", + platform: "node", + write: false, + footer: { js: "//# sourceURL=deep-scan-selection-store.js" }, +}); +const { WorkbenchDeepScanStore } = await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}` +); + +test("selection uses the dedicated function bridge and the store's existing replay policy", async () => { + const scanId = "ed8ff2da-01d9-4338-aeba-8bcfd4b530a9"; + const selection = { + version: 1, + resultPath: "artifacts/merge/checkpoints/aggregate.json", + resultSha256: "a".repeat(64), + terminalReason: "saturated", + omittedWorkerIds: [], + selectedAt: "2026-01-01T00:00:00Z", + }; + const response = { + deepScan: { + scanId, + targetPath: "/target", + scope: ".", + scanDir: "/scan", + status: "running", + schemaVersion: 1, + workflowVersion: "deep-security-scan/v2", + coordinatorGeneration: 2, + config: { + workers: 2, + subagents: 0, + stopAfterNoNew: 2, + stopAfterConsecutiveErrors: 2, + maxDiscoveryRuns: 4, + }, + dispatchedCount: 2, + noNewStreak: 2, + consecutiveErrors: 0, + finalizationInput: selection, + }, + }; + const calls = []; + const store = new WorkbenchDeepScanStore(async (...args) => { + calls.push(structuredClone(args)); + if (calls.length === 1) + throw Object.assign(new Error("Synthetic lost selection response"), { + code: "ETIMEDOUT", + }); + return response; + }); + const result = await store.selectFinalization({ + scanId, + coordinatorGeneration: 2, + reason: "saturated", + manifestPath: "/scan/scan-manifest.json", + resultPath: "/scan/artifacts/merge/checkpoints/aggregate.json", + omittedWorkerIds: [], + }); + assert.deepEqual(result.finalizationInput, selection); + assert.equal(calls.length, 2); + assert.deepEqual(calls[1], calls[0]); + assert.deepEqual(calls[0], [ + [ + "finish-deep-scan", + "--scan-id", + scanId, + "--coordinator-generation", + "2", + "--terminal-reason", + "saturated", + "--manifest-path", + "/scan/scan-manifest.json", + ], + JSON.stringify({ + resultPath: "/scan/artifacts/merge/checkpoints/aggregate.json", + }), + true, + ]); +}); diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.ts index beb79f0d41..4b0c4cbff6 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.ts +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.ts @@ -5,8 +5,9 @@ import { readOnlyParentSandboxState } from "./sandbox-state.ts"; import { temporaryDirectory } from "./support/temporary-directories.ts"; import assert from "node:assert/strict"; import { execFile } from "node:child_process"; -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { + appendFile, chmod, mkdir, readFile, @@ -24,7 +25,10 @@ import * as streams from "./support/streams.ts"; const execFileAsync = promisify(execFile); -const pluginRoot = path.resolve(mcpAppRoot, ".."); +const installedPluginRoot = process.env.CODEX_SECURITY_TEST_PLUGIN_ROOT; +const pluginRoot = installedPluginRoot + ? path.resolve(installedPluginRoot) + : path.resolve(mcpAppRoot, ".."); const workbenchPath = path.join(pluginRoot, "scripts", "workbench_db.py"); const parentSandboxState = readOnlyParentSandboxState(pluginRoot); @@ -33,9 +37,750 @@ if (process.platform === "win32") { "deep scan stdio lifecycle test skipped on Windows (POSIX fake Codex executable)", ); } else { + for (const mode of [ + "cancel-after-seal", + "cancel-finalizer", + "cancel-finalizer-failure", + "late-rejoin", + "late-rejoin-failure", + "detached", + "joined", + "remote", + "failure", + "active-failure", + "remote-replay", + "snapshot-observer", + "native-owner-usage", + "native-detached-usage", + "lost-response", + "lost-response-corrupt", + "lost-response-remove", + ]) { + await testDeepScanDetachedCompletion(mode); + } await testDeepScanStdioLifecycle(); } +async function testDeepScanDetachedCompletion(mode: string) { + const fixtureRoot = await temporaryDirectory("codex-security-deep-detached-"); + const targetPath = path.join(fixtureRoot, "target"); + const stateDir = path.join(fixtureRoot, "state"); + const scanRoot = path.join(fixtureRoot, "scans"); + const codexHome = path.join(fixtureRoot, "codex-home"); + const startLogPath = path.join(fixtureRoot, "started.jsonl"); + const exitLogPath = path.join(fixtureRoot, "exited.jsonl"); + const controlPath = path.join(fixtureRoot, "completion-control"); + const finalizerControlPath = path.join(fixtureRoot, "finalizer-control"); + const finalizerLogPath = path.join(fixtureRoot, "finalizer.jsonl"); + const committedControlPath = path.join(fixtureRoot, "committed-control"); + const committedLogPath = path.join(fixtureRoot, "committed.jsonl"); + const pythonWrapperPath = path.join(fixtureRoot, "python-wrapper.mjs"); + const fakeCodexPath = path.join(fixtureRoot, "fake-codex.mjs"); + const serverBundlePath = path.join( + pluginRoot, + "mcp", + installedPluginRoot + ? "server.mjs" + : `.deep-scan-detached-${randomUUID()}.cjs`, + ); + const threadId = "deep-scan-detached-result-conversation"; + const usageMode = + mode === "native-owner-usage" || mode === "native-detached-usage"; + const explicitCompletion = + mode === "active-failure" || + mode === "cancel-after-seal" || + mode.startsWith("cancel-finalizer") || + mode.startsWith("late-rejoin"); + const ownerRolloutPath = path.join(codexHome, "owner-rollout.jsonl"); + for (const directory of [ + targetPath, + stateDir, + scanRoot, + path.join(codexHome, "codex-security"), + ]) { + await mkdir(directory, { recursive: true }); + } + await writeFile(path.join(targetPath, "fixture.py"), "print('fixture')\n"); + await writeFile( + path.join(codexHome, "codex-security", "config.toml"), + "[deep_scan]\nworkers = 1\nsubagents = 0\nstop_after_no_new = 1\nmax_discovery_runs = 2\n", + ); + await writeFile(controlPath, "wait-for-completion"); + await writePythonWrapper(pythonWrapperPath); + if (mode === "cancel-after-seal") + await writeFile(committedControlPath, "wait"); + if (mode !== "detached" && !usageMode) + await writeFile( + finalizerControlPath, + mode === "joined" || + mode === "remote" || + mode.startsWith("cancel-finalizer") || + mode.startsWith("late-rejoin") + ? "wait" + : mode === "active-failure" || + mode === "remote-replay" || + mode === "snapshot-observer" + ? "failure" + : mode.startsWith("lost-response") + ? "lost-response" + : mode, + ); + await writeFakeCodex(fakeCodexPath); + if (!installedPluginRoot) + await buildServer(serverBundlePath, { target: "node20" }); + const environment: NodeJS.ProcessEnv = { + ...process.env, + OPENAI_API_KEY: "synthetic-stdio-key", + CODEX_API_KEY: "", + CODEX_CLI_PATH: fakeCodexPath, + CODEX_HOME: codexHome, + CODEX_SECURITY_SCAN_ROOT: scanRoot, + CODEX_SECURITY_STATE_DIR: stateDir, + FAKE_CODEX_START_LOG: startLogPath, + FAKE_CODEX_EXIT_LOG: exitLogPath, + FAKE_CODEX_RESTART_CONTROL: controlPath, + PYTHON: pythonWrapperPath, + REAL_PYTHON: process.env.PYTHON?.trim() || "python3", + FAKE_WORKBENCH_FINALIZER_CONTROL: finalizerControlPath, + FAKE_WORKBENCH_FINALIZER_LOG: finalizerLogPath, + FAKE_WORKBENCH_CANCEL_LOG: path.join(fixtureRoot, "cancel.jsonl"), + FAKE_WORKBENCH_COMMITTED_CONTROL: committedControlPath, + FAKE_WORKBENCH_COMMITTED_LOG: committedLogPath, + FAKE_CODEX_SIGNAL_CHECKPOINT_CONTROL: path.join( + fixtureRoot, + "unused-signal-control", + ), + }; + if (usageMode) { + environment.CODEX_SQLITE_HOME = codexHome; + environment.CODEX_STATE_DB = ""; + await writeFile( + ownerRolloutPath, + [ + { type: "session_meta", payload: { id: threadId, source: "cli" } }, + { + timestamp: new Date().toISOString(), + type: "turn_context", + payload: { turn_id: "native-owner-turn", model: "gpt-5.6-sol" }, + }, + ] + .map((record) => JSON.stringify(record) + "\n") + .join(""), + ); + await execFileAsync(environment.REAL_PYTHON!, [ + "-c", + ` +import sqlite3, sys +with sqlite3.connect(sys.argv[1]) as connection: + connection.execute("CREATE TABLE threads (id TEXT PRIMARY KEY, rollout_path TEXT NOT NULL)") + connection.execute("CREATE TABLE thread_spawn_edges (parent_thread_id TEXT NOT NULL, child_thread_id TEXT NOT NULL)") + connection.execute("INSERT INTO threads VALUES (?, ?)", (sys.argv[2], sys.argv[3])) +`, + path.join(codexHome, "state_5.sqlite"), + threadId, + ownerRolloutPath, + ]); + } + const server = startServer(serverBundlePath, environment); + let remote: ReturnType | undefined; + try { + assertNoError( + await server.request(1, "initialize", { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "deep-scan-detached-completion", version: "0.1.0" }, + }), + ); + server.sendRequest( + 2, + "tools/call", + toolCall( + "start_codex_security_deep_scan", + { targetPath, scope: ".", userContext: "Original discovery input" }, + threadId, + ), + ); + const scanId = await waitForScanId({ server, requestId: 2 }); + await waitForDeepScanWorker({ environment, scanId, threadId }); + const [worker] = await waitForJsonLines(startLogPath, 1); + if (usageMode) await appendOwnerUsage(10); + if (mode === "joined") { + server.sendRequest( + 3, + "tools/call", + toolCall("start_codex_security_deep_scan", { scanId }, threadId), + ); + await waitFor( + () => + server + .stderrEvents() + .some((event) => event.event === "coordinator_joined"), + "second observer to join", + ); + } + if (mode === "remote") { + remote = startServer(serverBundlePath, environment); + assertNoError( + await remote.request(1, "initialize", { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "remote-observer", version: "0.1.0" }, + }), + ); + remote.sendRequest( + 2, + "tools/call", + toolCall("start_codex_security_deep_scan", { scanId }, threadId), + ); + await waitFor( + () => + remote! + .stderrEvents() + .some((event) => event.event === "coordinator_joined"), + "remote request to observe the existing owner", + ); + } + if ( + mode !== "active-failure" && + mode !== "native-owner-usage" && + mode !== "cancel-after-seal" && + !mode.startsWith("cancel-finalizer") && + !mode.startsWith("late-rejoin") + ) + server.notify("notifications/cancelled", { + requestId: 2, + reason: "detach original observer", + }); + const detached = await getDeepScan({ environment, scanId, threadId }); + const originalPublicScan = await runWorkbench(environment, [ + "get-scan", + "--scan-id", + scanId, + ]); + assert.equal(detached.status, "running"); + assert.equal(detached.cancelRequested, false); + assertProcessAlive(worker.pid); + assertProcessAlive(server.pid!); + + // The already-owned workers proceed after observation ends. + await writeFile(controlPath, "after-restart"); + let finished: Awaited>; + await waitFor(async () => { + finished = await getDeepScan({ environment, scanId, threadId }); + return finished.status === "succeeded"; + }, "detached discovery and reducer to finish"); + assert.equal(finished.workflowVersion, "deep-security-scan/v2"); + assert.equal(finished.finalizationInput.version, 1); + assert.equal( + finished.coordinatorGeneration, + detached.coordinatorGeneration, + ); + assert.equal(finished.userContext, detached.userContext); + assert.deepEqual(finished.usageOwner, detached.usageOwner); + assert.equal( + finished.workers.filter( + (row: PersistedDeepScanWorker) => + row.kind === "discovery" && row.status === "succeeded", + ).length, + 2, + ); + assert.equal( + finished.workers.filter( + (row: PersistedDeepScanWorker) => + row.kind === "dedup" && row.status === "succeeded", + ).length, + 1, + ); + const selectedBytes = await readFile( + path.join(finished.scanDir, finished.finalizationInput.resultPath), + ); + assert.equal( + createHash("sha256").update(selectedBytes).digest("hex"), + finished.finalizationInput.resultSha256, + ); + if (usageMode) { + if (mode === "native-owner-usage") { + assertNoError(await server.waitForResponse(2)); + } else { + await waitFor( + async () => + (await runWorkbench(environment, ["get-scan", "--scan-id", scanId])) + .scan.progress.status === "complete", + "detached native publication", + ); + await appendFile( + ownerRolloutPath, + JSON.stringify({ + timestamp: new Date().toISOString(), + type: "turn_context", + payload: { turn_id: "unrelated-later-turn", model: "gpt-5.6-sol" }, + }) + "\n", + ); + } + await appendOwnerUsage(1010); + const completed = await server.request( + 60, + "tools/call", + toolCall("complete_codex_security_scan", { scanId }, threadId), + ); + assertNoError(completed); + const afterOwner = await runWorkbench(environment, [ + "get-scan", + "--scan-id", + scanId, + ]); + assert.equal(afterOwner.scan.progress.status, "complete"); + assert.equal( + afterOwner.scan.usage.inputTokens, + mode === "native-owner-usage" ? 1010 : 10, + "completion accounts for the owning continuation, excluding later conversation work", + ); + console.log("native owning continuation usage passed", mode, scanId); + return; + } + if (explicitCompletion) { + assertNoError(await server.waitForResponse(2)); + server.sendRequest( + 60, + "tools/call", + toolCall("complete_codex_security_scan", { scanId }, threadId), + ); + } + if (mode === "cancel-after-seal") { + await waitForJsonLines(committedLogPath, 1); + const beforeCancel = await runWorkbench(environment, [ + "get-scan", + "--scan-id", + scanId, + ]); + assert.equal(beforeCancel.scan.progress.status, "complete"); + const sealedBytes = await readFile( + path.join(finished.scanDir, "scan-manifest.json"), + ); + server.sendRequest( + 6, + "tools/call", + toolCall("cancel_codex_security_scan", { scanId }, threadId), + ); + const cancellation = await server.waitForResponse(6); + await rm(committedControlPath, { force: true }); + assertNoError(await server.waitForResponse(60)); + assert.equal(cancellation.result?.isError, true); + const afterCancel = await runWorkbench(environment, [ + "get-scan", + "--scan-id", + scanId, + ]); + assert.equal(afterCancel.scan.progress.status, "complete"); + assert.equal( + afterCancel.scan.executionAttribution.completedAt, + beforeCancel.scan.executionAttribution.completedAt, + ); + assert.deepEqual( + await readFile(path.join(finished.scanDir, "scan-manifest.json")), + sealedBytes, + ); + return; + } + if (mode.startsWith("cancel-finalizer")) { + await waitForJsonLines(finalizerLogPath, 1); + server.sendRequest( + 6, + "tools/call", + toolCall("cancel_codex_security_scan", { scanId }, threadId), + ); + const canceled = await server.waitForResponse(6); + if (mode === "cancel-finalizer-failure") + await writeFile(finalizerControlPath, "failure"); + else await rm(finalizerControlPath, { force: true }); + const ownerResponse = await server.waitForResponse(60); + const publicScan = await runWorkbench(environment, [ + "get-scan", + "--scan-id", + scanId, + ]); + const stopped = await getDeepScan({ environment, scanId, threadId }); + assert.equal( + publicScan.scan.progress.status, + "canceled", + "cancellation observed before parent completion must remain canceled", + ); + assertNoError(canceled); + assert.equal(ownerResponse.result?.isError, true); + assert.match( + ownerResponse.result.content + .map((item: { text: string }) => item.text) + .join(" "), + /Only a running scan can be completed|injected complete-scan failure/, + ); + assert.equal(stopped.status, "canceled"); + assert.deepEqual( + await readFile( + path.join(finished.scanDir, finished.finalizationInput.resultPath), + ), + selectedBytes, + ); + assert.deepEqual(stopped.finalizationInput, finished.finalizationInput); + assert.equal( + publicScan.scan.continuationThreadId, + originalPublicScan.scan.continuationThreadId, + ); + return; + } + if (mode.startsWith("late-rejoin")) { + await waitForJsonLines(finalizerLogPath, 1); + remote = startServer(serverBundlePath, { + ...environment, + FAKE_WORKBENCH_FINALIZER_CONTROL: path.join( + fixtureRoot, + "remote-finalizer-control", + ), + }); + assertNoError( + await remote.request(1, "initialize", { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "late-remote-observer", version: "0.1.0" }, + }), + ); + remote.sendRequest( + 2, + "tools/call", + toolCall("start_codex_security_deep_scan", { scanId }, threadId), + ); + const observed = await remote!.waitForResponse(2); + const during = await runWorkbench(environment, [ + "get-scan", + "--scan-id", + scanId, + ]); + if (mode === "late-rejoin-failure") + await writeFile(finalizerControlPath, "failure"); + else await rm(finalizerControlPath); + const ownerResponse = await server.waitForResponse(60); + const invocations = await readJsonLines(finalizerLogPath); + assertNoError(observed); + assert.equal( + observed.result.structuredContent.manifestPath, + path.join(finished.scanDir, "scan-manifest.json"), + ); + assert.deepEqual( + await readFile( + path.join(finished.scanDir, finished.finalizationInput.resultPath), + ), + selectedBytes, + ); + assert.equal( + invocations.length, + 1, + "late remote observation must not invoke a competing parent finalizer", + ); + assert.equal( + during.scan.progress.status, + "running", + "only the existing owner may finish pending parent completion", + ); + if (mode === "late-rejoin-failure") { + assert.equal(ownerResponse.result?.isError, true); + assert.equal( + (await runWorkbench(environment, ["get-scan", "--scan-id", scanId])) + .scan.progress.status, + "running", + ); + return; + } + assertNoError(ownerResponse); + } + if (mode === "joined") { + await waitForJsonLines(finalizerLogPath, 1); + // Rejoin while finish-deep-scan is committed but public sealing is blocked. + server.sendRequest( + 4, + "tools/call", + toolCall("start_codex_security_deep_scan", { scanId }, threadId), + ); + await waitFor( + () => + server + .stderrEvents() + .filter((event) => event.event === "coordinator_joined").length === + 2, + "observer to join pending public completion", + ); + await delay(5_100); // Exercise an actual coordinator heartbeat during finalization. + assert.equal(server.response(3), undefined); + assert.equal(server.response(4), undefined); + assert.equal( + (await runWorkbench(environment, ["get-scan", "--scan-id", scanId])) + .scan.progress.status, + "running", + ); + assert.equal((await readJsonLines(finalizerLogPath)).length, 1); + await rm(finalizerControlPath); + for (const id of [3, 4]) assertNoError(await server.waitForResponse(id)); + } + if (mode === "remote") { + // A remote observer retains the aggregate-ready response. It cannot run + // the owning process's public finalizer, even after Deep itself succeeds. + const observed = await remote!.waitForResponse(2); + await waitForJsonLines(finalizerLogPath, 1); + assertNoError(observed); + assert.equal( + observed.result.structuredContent.manifestPath, + path.join(finished.scanDir, "scan-manifest.json"), + ); + assert.equal((await readJsonLines(finalizerLogPath)).length, 1); + await rm(finalizerControlPath); + } + if ( + mode === "failure" || + mode === "active-failure" || + mode === "remote-replay" || + mode === "snapshot-observer" || + mode.startsWith("lost-response") + ) { + if (mode !== "active-failure") + await waitFor( + () => + server + .stderrEvents() + .some( + (event) => event.event === "coordinator_publication_pending", + ), + "public finalization failure to remain pending", + ); + if (mode === "active-failure") { + const failed = await server.waitForResponse(60); + assert.equal(failed.result?.isError, true); + const message = failed.result.content + .map((item: { text: string }) => item.text) + .join(" "); + assert.match(message, /injected complete-scan failure/); + assert.match(message, /Do not retry completion/); + assert.match(message, /no final|Do not.*final/i); + } + const pending = await getDeepScan({ environment, scanId, threadId }); + assert.equal(pending.status, "succeeded"); + assert.equal(pending.terminalReason, finished.terminalReason); + assert.deepEqual(pending.finalizationInput, finished.finalizationInput); + assert.equal( + (await readJsonLines(finalizerLogPath)).length, + 1, + "the owner does not add a retry layer", + ); + const beforeReplay = await runWorkbench(environment, [ + "get-scan", + "--scan-id", + scanId, + ]); + assert.equal( + beforeReplay.scan.progress.status, + !mode.startsWith("lost-response") ? "running" : "complete", + ); + const manifestBeforeReplay = await readFile( + path.join(finished.scanDir, "scan-manifest.json"), + ); + if (mode === "snapshot-observer") { + const observed = await server.request( + 50, + "tools/call", + toolCall( + "start_codex_security_deep_scan", + { targetPath, scope: "." }, + "independent-snapshot-observer", + ), + ); + assertNoError(observed); + assert.equal(observed.result.structuredContent.scanId, scanId); + assert.equal( + observed.result.structuredContent.manifestPath, + path.join(finished.scanDir, "scan-manifest.json"), + ); + assert.equal( + (await readJsonLines(finalizerLogPath)).length, + 1, + "a snapshot observer must not acquire publication ownership", + ); + } + let replayServer = server; + if (mode === "remote-replay") { + await server.stop(); + remote = startServer(serverBundlePath, environment); + replayServer = remote; + assertNoError( + await remote.request(1, "initialize", { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "reconstructed-finalizer", version: "0.1.0" }, + }), + ); + } + const damagedArtifact = path.join(finished.scanDir, "findings.json"); + if (mode === "lost-response-corrupt") { + await writeFile( + damagedArtifact, + Buffer.concat([await readFile(damagedArtifact), Buffer.from(" ")]), + ); + } else if (mode === "lost-response-remove") { + await rm(damagedArtifact); + } + const rejoined = await replayServer.request( + 5, + "tools/call", + toolCall("start_codex_security_deep_scan", { scanId }, threadId), + ); + if (mode === "lost-response-corrupt" || mode === "lost-response-remove") { + assert.equal( + rejoined.result?.isError, + true, + "completion replay must reject altered sealed artifacts", + ); + assert.match( + rejoined.result.content + .map((item: { text: string }) => item.text) + .join(" "), + /sealed artifact changed or is missing|findings\.json: expected a file/, + ); + assert.deepEqual( + await readFile(path.join(finished.scanDir, "scan-manifest.json")), + manifestBeforeReplay, + ); + assert.deepEqual( + await readFile( + path.join(finished.scanDir, finished.finalizationInput.resultPath), + ), + selectedBytes, + ); + assert.equal( + (await readJsonLines(startLogPath)).length, + 3, + "rejected replay launches no workers", + ); + const afterReplay = await runWorkbench(environment, [ + "get-scan", + "--scan-id", + scanId, + ]); + assert.equal(afterReplay.scan.progress.status, "complete"); + assert.equal( + afterReplay.scan.executionAttribution.completedAt, + beforeReplay.scan.executionAttribution.completedAt, + ); + console.log( + "native selected completion integrity passed", + mode, + scanId, + ); + return; + } + assertNoError(rejoined); + if (!mode.startsWith("lost-response")) + assertNoError( + await replayServer.request( + 61, + "tools/call", + toolCall("complete_codex_security_scan", { scanId }, threadId), + ), + ); + assert.equal( + rejoined.result.structuredContent.manifestPath, + path.join(finished.scanDir, "scan-manifest.json"), + ); + if (mode.startsWith("lost-response")) { + assert.deepEqual( + await readFile(path.join(finished.scanDir, "scan-manifest.json")), + manifestBeforeReplay, + ); + const afterReplay = await runWorkbench(environment, [ + "get-scan", + "--scan-id", + scanId, + ]); + assert.equal(afterReplay.scan.progress.status, "complete"); + assert.equal( + afterReplay.scan.executionAttribution.completedAt, + beforeReplay.scan.executionAttribution.completedAt, + ); + } + assert.deepEqual( + (await getDeepScan({ environment, scanId, threadId })) + .finalizationInput, + finished.finalizationInput, + ); + } + let publicScan: Awaited>; + await waitFor(async () => { + publicScan = await runWorkbench(environment, [ + "get-scan", + "--scan-id", + scanId, + ]); + return publicScan.scan.progress.status === "complete"; + }, "public completion without another observer"); + assert.equal(publicScan.scan.progress.status, "complete"); + const manifest = JSON.parse( + await readFile(path.join(finished.scanDir, "scan-manifest.json"), "utf8"), + ); + assert.equal(typeof manifest.scan.sealedAt, "string"); + assert.equal(typeof manifest.scan.completedAt, "string"); + assert.equal( + publicScan.scan.continuationThreadId, + originalPublicScan.scan.continuationThreadId, + ); + assert.deepEqual( + publicScan.scan.executionAttribution.owner, + originalPublicScan.scan.executionAttribution.owner, + ); + assert.equal(publicScan.scan.executionAttribution.owner.threadId, threadId); + assert.equal( + (await readJsonLines(startLogPath)).length, + 3, + "completion and replay launch no extra model workers", + ); + assert.equal( + (await readJsonLines(finalizerLogPath)).length, + mode === "failure" || + mode === "active-failure" || + mode === "remote-replay" || + mode === "snapshot-observer" + ? 2 + : 1, + ); + assertProcessAlive(mode === "remote-replay" ? remote!.pid! : server.pid!); + console.log("native selected completion passed", mode, scanId); + } catch (error) { + (error as Error).message += `\nMCP stderr:\n${server.stderrText()}`; + throw error; + } finally { + await remote?.stop(); + await server.stop(); + if (!installedPluginRoot) await rm(serverBundlePath, { force: true }); + await rm(fixtureRoot, { recursive: true, force: true }); + } + async function appendOwnerUsage(inputTokens: number) { + await appendFile( + ownerRolloutPath, + JSON.stringify({ + timestamp: new Date().toISOString(), + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: inputTokens, + cached_input_tokens: 0, + cache_write_input_tokens: 0, + output_tokens: 1, + reasoning_output_tokens: 0, + total_tokens: inputTokens + 1, + }, + }, + }, + }) + "\n", + ); + } +} + async function testDeepScanStdioLifecycle() { const fixtureRoot = await temporaryDirectory("codex-security-deep-stdio-"); const targetPath = path.join(fixtureRoot, "target"); @@ -63,7 +808,9 @@ async function testDeepScanStdioLifecycle() { const serverBundlePath = path.join( pluginRoot, "mcp", - `.deep-scan-stdio-test-${randomUUID()}.cjs`, + installedPluginRoot + ? "server.mjs" + : `.deep-scan-stdio-test-${randomUUID()}.cjs`, ); const threadId = "deep-scan-stdio-lifecycle-thread"; @@ -94,7 +841,8 @@ model_reasoning_summary = "none" `, ); await writePythonWrapper(pythonWrapperPath); - await buildServer(serverBundlePath, { target: "node20" }); + if (!installedPluginRoot) + await buildServer(serverBundlePath, { target: "node20" }); const environment = { ...process.env, @@ -607,7 +1355,7 @@ model_reasoning_summary = "none" "the MCP server must remain responsive after canceling one scan", ); - const resumedThreadId = "deep-scan-stdio-resumed-thread"; + let resumedThreadId = "deep-scan-stdio-resumed-thread"; const opened = await server.request( 24, "tools/call", @@ -625,7 +1373,13 @@ model_reasoning_summary = "none" "tools/call", toolCall( "submit_codex_security_setup", - { sessionId, targetPath, scope: ".", mode: "deep" }, + { + sessionId, + targetPath, + scope: ".", + mode: "deep", + userContext: "Original discovery focus", + }, resumedThreadId, ), ), @@ -638,7 +1392,7 @@ model_reasoning_summary = "none" assertNoError(started); const resumedScan = started.result.structuredContent.workspace.results; const resumedScanId = resumedScan.scanId; - const handoffClaimToken = randomUUID(); + let handoffClaimToken = randomUUID(); for (const [id, name, arguments_] of [ [ 27, @@ -710,6 +1464,35 @@ model_reasoning_summary = "none" const completedDraft = await readJson(completedWorker.resultManifestPath); assert.equal(completedDraft.scanId, resumedScanId); assert.deepEqual(completedDraft.findings, []); + assert.equal( + partial!.workflowVersion, + "deep-security-scan/v2", + "new scans use selected finalization by default", + ); + assert.equal(partial!.userContext, "Original discovery focus"); + assert.equal(partial!.usageOwner.threadId, resumedThreadId); + const settingsPath = path.join( + resumedScan.scanDir, + "artifacts", + "deep_discovery", + "execution-settings.json", + ); + const originalSettings = await readFile(settingsPath, "utf8"); + assertNoError( + await server.request( + 30, + "tools/call", + toolCall( + "update_codex_security_scan_context", + { + scanId: resumedScanId, + handoffClaimToken, + userContext: "Later result discussion", + }, + resumedThreadId, + ), + ), + ); await server.stop(); assert.throws( () => process.kill(server.pid!, 0), @@ -755,7 +1538,36 @@ model_reasoning_summary = "none" path.join(stateDir, "workbench.sqlite3"), resumedScanId, ]); + await runWorkbench(environment, [ + "release-handoff-delivery", + "--scan-id", + resumedScanId, + "--claim-token", + handoffClaimToken, + ]); + handoffClaimToken = randomUUID(); + resumedThreadId = "deep-scan-stdio-replacement-thread"; + await runWorkbench(environment, [ + "claim-handoff-delivery", + "--scan-id", + resumedScanId, + "--claim-token", + handoffClaimToken, + ]); + await runWorkbench(environment, [ + "attach-scan-continuation-thread", + "--scan-id", + resumedScanId, + "--claim-token", + handoffClaimToken, + "--thread-id", + resumedThreadId, + ]); await writeFile(restartControlPath, "after-restart"); + await writeFile( + runtimeConfigPath, + 'model_reasoning_summary = "detailed"\n', + ); const restartedServer = startServer(serverBundlePath, environment); try { @@ -819,6 +1631,25 @@ model_reasoning_summary = "none" partial!.coordinatorGeneration + 1, ); assert.equal(finished.dispatchedCount, 2); + assert.equal(finished.workflowVersion, partial!.workflowVersion); + assert.equal( + finished.finalizationInput.version, + 1, + "recovery selects a persisted finalization input", + ); + assert.equal(finished.userContext, partial!.userContext); + assert.equal( + finished.createdAt, + partial!.createdAt, + "recovery retains the original deadline origin", + ); + assert.equal(finished.config.maxTimeHours, partial!.config.maxTimeHours); + assert.deepEqual( + finished.usageOwner, + partial!.usageOwner, + "a replacement continuation does not rebind original usage", + ); + assert.equal(await readFile(settingsPath, "utf8"), originalSettings); const successfulDiscoveries = finished.workers.filter( (worker: PersistedDeepScanWorker) => worker.kind === "discovery" && worker.status === "succeeded", @@ -883,6 +1714,9 @@ model_reasoning_summary = "none" execution.argv.includes('model_reasoning_summary="none"'), true, ); + const context = discoveryPromptContext(execution.stdin); + if (context.workerLabel) + assert.equal(context.userContext, "Original discovery focus"); } assert.equal( executions.filter( @@ -900,7 +1734,7 @@ model_reasoning_summary = "none" throw error; } finally { await server.stop(); - await rm(serverBundlePath, { force: true }); + if (!installedPluginRoot) await rm(serverBundlePath, { force: true }); await rm(fixtureRoot, { recursive: true, force: true }); } } @@ -1082,6 +1916,9 @@ const context = JSON.parse(stdin.match(/\`\`\`json\\n([\\s\\S]*?)\\n\`\`\`/u)[1] const root = process.argv[process.argv.indexOf('--cd') + 1]; appendFileSync(process.env.FAKE_CODEX_START_LOG, JSON.stringify({ pid: process.pid, argv: process.argv.slice(2), stdin, hasExpectedApiKey: process.env.CODEX_API_KEY === 'synthetic-stdio-key' }) + '\\n'); console.log(JSON.stringify({ type: 'thread.started', thread_id: \`stdio-fixture-\${process.pid}\` })); +while (existsSync(process.env.FAKE_CODEX_RESTART_CONTROL) && readFileSync(process.env.FAKE_CODEX_RESTART_CONTROL, 'utf8') === 'wait-for-completion') { + await new Promise((resolve) => setTimeout(resolve, 25)); +} if (existsSync(process.env.FAKE_CODEX_RESTART_CONTROL)) { const phase = readFileSync(process.env.FAKE_CODEX_RESTART_CONTROL, 'utf8'); if (phase === 'after-restart' || context.workerLabel === 'discovery-0001') { @@ -1120,23 +1957,44 @@ process.once('SIGTERM', () => stop('SIGTERM')); async function writePythonWrapper(executablePath: string) { await writeFile( executablePath, - `#!/usr/bin/env node -import { appendFileSync, existsSync, unlinkSync } from "node:fs"; -import { spawnSync } from "node:child_process"; -const args = process.argv.slice(2); -const control = process.env.FAKE_WORKBENCH_CANCEL_FAILURE_CONTROL; -if (args[1] === 'cancel-scan') { - appendFileSync(process.env.FAKE_WORKBENCH_CANCEL_LOG, JSON.stringify(args) + '\\n'); -} -if (args[1] === 'cancel-scan' && control && existsSync(control)) { - unlinkSync(control); - console.error('injected cancel-scan failure'); - process.exit(1); -} -const result = spawnSync(process.env.REAL_PYTHON || 'python3', args, { stdio: 'inherit' }); -if (result.error) throw result.error; -process.exit(result.status ?? 1); -`, + [ + "#!/usr/bin/env node", + 'import { appendFileSync, existsSync, readFileSync, unlinkSync } from "node:fs";', + 'import { spawnSync } from "node:child_process";', + "const args = process.argv.slice(2);", + "const control = process.env.FAKE_WORKBENCH_CANCEL_FAILURE_CONTROL;", + "if (args[1] === 'cancel-scan') {", + " appendFileSync(process.env.FAKE_WORKBENCH_CANCEL_LOG, JSON.stringify(args) + '\\n');", + "}", + "if (args[1] === 'cancel-scan' && control && existsSync(control)) {", + " unlinkSync(control);", + " console.error('injected cancel-scan failure');", + " process.exit(1);", + "}", + "let finalizerMode;", + "if (args[1] === 'complete-scan' && process.env.FAKE_WORKBENCH_FINALIZER_LOG) {", + " appendFileSync(process.env.FAKE_WORKBENCH_FINALIZER_LOG, JSON.stringify(args) + '\\n');", + " const finalizerControl = process.env.FAKE_WORKBENCH_FINALIZER_CONTROL;", + " while (existsSync(finalizerControl) && readFileSync(finalizerControl, 'utf8') === 'wait') {", + " await new Promise((resolve) => setTimeout(resolve, 25));", + " }", + " if (existsSync(finalizerControl)) {", + " finalizerMode = readFileSync(finalizerControl, 'utf8');", + " unlinkSync(finalizerControl);", + " }", + " if (finalizerMode === 'failure') { console.error('injected complete-scan failure'); process.exit(1); }", + "}", + "const result = spawnSync(process.env.REAL_PYTHON || 'python3', args, { stdio: 'inherit' });", + "const committedControl = process.env.FAKE_WORKBENCH_COMMITTED_CONTROL;", + "if (args[1] === 'complete-scan' && result.status === 0 && committedControl && existsSync(committedControl)) {", + " appendFileSync(process.env.FAKE_WORKBENCH_COMMITTED_LOG, JSON.stringify({ pid: process.pid, committed: true }) + '\\n');", + " while (existsSync(committedControl)) await new Promise((resolve) => setTimeout(resolve, 25));", + "}", + "if (result.error) throw result.error;", + "if (result.status === 0 && finalizerMode === 'lost-response') { console.error('injected lost completion response'); process.exit(1); }", + "process.exit(result.status ?? 1);", + "", + ].join("\n"), ); await chmod(executablePath, 0o755); } diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_store.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_store.ts index 129f1fbc25..8af6524c0a 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_store.ts +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_store.ts @@ -20,10 +20,13 @@ const canonical = { }; await testBeginProtocolAndParsing(); +await testBeginCarriesOriginalSettingsWithUserContext(); +await testClaimChecksOriginalSettingsInsideTheTransaction(); await testCanonicalCommitProtocol(); await testTerminalProtocol(); testCanonicalNullAndPartialParsing(); testRunErrorParsing(); +testWorkflowVersionParsing(); testConfiguredMaximumDurationParsing(); await testWriteSerializationAndRecovery(); await testBeginUsesTheWriteQueue(); @@ -38,6 +41,94 @@ await testPersistenceRetryExhaustionPreservesDiagnostics(); await testDeterministicPersistenceFailuresAreNotRetried(); await testNonIdempotentMutationsAreNotRetried(); testInvalidPersistedConfig(); +testOriginalUsageOwnerParsing(); + +function testOriginalUsageOwnerParsing() { + const value = stateResult(randomUUID()); + const usageOwner = { + threadId: "original-thread", + turnId: "original-turn", + startedAt: "2026-01-01T00:00:00Z", + }; + assert.deepEqual( + parseDeepScan({ deepScan: { ...value.deepScan, usageOwner } }).usageOwner, + usageOwner, + ); + assert.equal( + parseDeepScan({ deepScan: { ...value.deepScan, usageOwner: null } }) + .usageOwner, + null, + ); + assert.equal( + parseDeepScan(value).usageOwner, + null, + "old readers do not establish an original owner", + ); +} + +async function testBeginCarriesOriginalSettingsWithUserContext() { + const executionSettings = { + codexPath: "/fixture/codex", + codexHome: "/fixture/home", + model: "original-model", + reasoningSummary: "concise", + }; + const userContext = "Review the parser.\nKeep this second line."; + const runner = async ( + args: string[], + input?: string, + selectFinalization?: boolean, + withExecutionSettings?: boolean, + ) => { + assert.equal(selectFinalization, false); + assert.equal(withExecutionSettings, true); + assert.deepEqual(JSON.parse(input!), { executionSettings, userContext }); + assert.equal( + args.includes("--user-context-stdin"), + false, + "the private structured input carries context without a second stdin consumer", + ); + assert.equal( + args.some((arg) => arg.includes("execution-settings")), + false, + "no public argument is added for the internal settings handoff", + ); + return stateResult(randomUUID(), { startDisposition: "created" }); + }; + await new WorkbenchDeepScanStore(runner).begin({ + targetPath: "/fixture/repository", + threadId: "fixture-thread", + scanRoot: "/fixture/scans", + userContext, + executionSettings, + }); +} + +async function testClaimChecksOriginalSettingsInsideTheTransaction() { + const scanId = randomUUID(); + const store = new WorkbenchDeepScanStore( + async ( + args: string[], + input?: string, + selectFinalization?: boolean, + withExecutionSettings?: boolean, + ) => { + assert.equal(args[0], "claim-deep-scan-coordinator"); + assert.equal(input, undefined); + assert.equal(selectFinalization, false); + assert.equal(withExecutionSettings, true); + return { + ...stateResult(scanId, { deepScan: { coordinatorGeneration: 2 } }), + coordinatorDisposition: "claimed", + }; + }, + ); + assert.equal( + (await store.claimCoordinator({ scanId, threadId: "fixture-thread" })) + .acquired, + true, + ); +} async function testBeginProtocolAndParsing() { const scanId = randomUUID(); @@ -85,7 +176,7 @@ async function testBeginProtocolAndParsing() { ); assert.equal( flagValue(calls[0].args, "--workflow-version"), - "deep-scan-mcp/v1", + "deep-security-scan/v2", ); const claimToken = randomUUID(); @@ -519,7 +610,7 @@ async function testPersistenceRetriesRemainInsideTheWriteQueue() { if (args[0] === "claim-deep-scan-dedup" && calls.length === 1) { throw new Error("sqlite3.OperationalError: database is locked"); } - return {}; + return stateResult(scanId); }); const claim = store.claimDedup({ @@ -786,7 +877,7 @@ function idempotentPersistenceScenarios() { })), { operation: "claim-deep-scan-dedup", - result: {}, + result: stateResult(scanId), invoke: (store: Store) => store.claimDedup({ id: reducerId, @@ -1007,3 +1098,47 @@ function repeatedFlagValues(args: string[], flag: string) { value === flag ? [args[index + 1]] : [], ); } + +function testWorkflowVersionParsing() { + const value = stateResult(randomUUID()).deepScan; + const run = parseDeepScan({ + deepScan: { + ...value, + schemaVersion: 1, + workflowVersion: "deep-scan-mcp/v1", + model: "original-model", + reasoningEffort: "high", + }, + }); + assert.equal(run.model, "original-model"); + assert.equal(run.reasoningEffort, "high"); + assert.equal(run.schemaVersion, 1); + assert.equal(run.workflowVersion, "deep-scan-mcp/v1"); + const future = parseDeepScan({ + deepScan: { ...value, schemaVersion: 99, workflowVersion: "future/v99" }, + }); + assert.equal(future.schemaVersion, 99); + assert.equal( + future.workflowVersion, + "future/v99", + "inspection preserves unsupported versions", + ); +} + +for (const version of [1, 99]) { + const finalizationInput = { + version, + resultPath: null, + resultSha256: null, + terminalReason: "capped", + omittedWorkerIds: ["fixture-worker"], + selectedAt: "2026-01-01T00:00:00Z", + }; + assert.deepEqual( + parseDeepScan( + stateResult(randomUUID(), { deepScan: { finalizationInput } }), + ).finalizationInput, + finalizationInput, + "inspection preserves finalization input and version before execution compatibility checks", + ); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.ts index 017fda701f..24665caf20 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.ts +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.ts @@ -94,6 +94,11 @@ async function testRecoveredPublicationRejectsLateFailure() { "Publication recovery remains pending.", ), runWorkbench, + undefined, + { + coordinatorGeneration: claim.run.coordinatorGeneration, + resultPath: null, + }, ); await runWorkbench([ "cancel-scan", @@ -333,7 +338,20 @@ async function testReducerCommitAndFinishAgainstRealWorkbench( const codexHome = path.join(fixtureRoot, "codex-home"); const threadId = "deep-scan-store-integration-thread"; const python = process.env.PYTHON?.trim() || "python3"; - const runWorkbench = createWorkbenchRunner(environment); + const finishCalls: string[][] = []; + const executeWorkbench = createWorkbenchRunner(environment); + const runWorkbench = async (args: string[]) => { + const result = await executeWorkbench(args); + if (args[0] === "finish-deep-scan") { + finishCalls.push([...args]); + if (finishCalls.length === 1) + throw Object.assign( + new Error("Synthetic lost committed finish response"), + { code: "ETIMEDOUT" }, + ); + } + return result; + }; const store = new WorkbenchDeepScanStore(runWorkbench); try { @@ -553,6 +571,8 @@ connection.rollback()`, omittedWorkerIds: [late.id], }); assert.equal(finished.status, "succeeded"); + assert.equal(finishCalls.length, 2); + assert.deepEqual(finishCalls[1], finishCalls[0]); assert.equal(finished.terminalReason, "saturated"); assert.equal(finished.manifestPath, manifestPath); diff --git a/plugins/codex-security/mcp-app/tsconfig.json b/plugins/codex-security/mcp-app/tsconfig.json index 1a51147297..7b8116b0b0 100644 --- a/plugins/codex-security/mcp-app/tsconfig.json +++ b/plugins/codex-security/mcp-app/tsconfig.json @@ -3,6 +3,10 @@ "module": "ESNext", "moduleResolution": "Bundler", "noEmit": true, + "paths": { + "@openai/codex-sdk": ["./node_modules/@openai/codex-sdk"], + "smol-toml": ["./node_modules/smol-toml/dist/index"] + }, "resolveJsonModule": true, "skipLibCheck": true, "strict": true, diff --git a/plugins/codex-security/native/proof-policy-windows.mts b/plugins/codex-security/native/proof-policy-windows.mts index d649cde927..ace760696e 100644 --- a/plugins/codex-security/native/proof-policy-windows.mts +++ b/plugins/codex-security/native/proof-policy-windows.mts @@ -19,10 +19,11 @@ import { const testDirectory = join(output, "policy-proof"); const helper = join(testDirectory, "helpers.cjs"); if (process.argv[2] === "build") { + const sdkModules = join(root, "../../../sdk/typescript/node_modules"); execFileSync( process.execPath, [ - join(root, "../../../sdk/typescript/node_modules/esbuild/bin/esbuild"), + join(sdkModules, "esbuild/bin/esbuild"), join(root, "../mcp-app/helpers-main.ts"), "--bundle", "--platform=node", @@ -31,7 +32,11 @@ if (process.argv[2] === "build") { "--define:import.meta.url=__filename", `--outfile=${helper}`, ], - { stdio: "inherit" }, + { + stdio: "inherit", + // Native CI installs the helper's dependencies only in the SDK. + env: { ...process.env, NODE_PATH: sdkModules }, + }, ); const nativeDirectory = join(testDirectory, "native", nativeTarget); mkdirSync(nativeDirectory, { recursive: true }); diff --git a/plugins/codex-security/references/core-scan.md b/plugins/codex-security/references/core-scan.md index 5860bedbeb..21339b4e2d 100644 --- a/plugins/codex-security/references/core-scan.md +++ b/plugins/codex-security/references/core-scan.md @@ -9,7 +9,7 @@ Perform one complete, evidence-backed security audit of the exact supplied repos 3. While the baseline runs, read `threat-model.md` once and obtain its independent architecture review within the available worker allowance. Verify its resource rows against their actual consumers, use the returned canonical `threatModel` as the generated model, and build source-backed investigation packets from it. Immediately retain the resulting model through the caller's existing semantic checkpoint mechanism with `complete: false`, partial coverage, and `findings: []` when findings are not yet available. This model-only checkpoint does not claim completed source review. Carry that object and its evidence into the final result instead of reconstructing a shorter summary. Preserve any user-supplied threat model unchanged as the authoritative security assumptions; map its real surfaces and controls without replacing it. 4. Group related source-backed security questions into investigation packets. Each group shares its plausible attacker, protected asset, entry points, expected controls, sensitive operations, component relationships, and actual repository-relative source anchors. Keep each question concrete, preserve distinct attacker boundaries and security mechanisms, and let investigators establish the detailed dataflow. 5. Launch focused investigator subagents with `fork_turns: "none"` as soon as useful packet groups exist. Choose their number and assignments from the amount, complexity, and independence of source-backed work, bounded by the supplied available subagent allowance; use fewer for related packets and more only when distinct surfaces justify them. Keep mapping other surfaces while they run. Send each only its focused-investigator prompt below, assigned packets, investigator perspective, repository path, authorized scope, any supplied scoped-source inventory, exact user context, supplied threat model, applicable packet-specific security guidance and its resolver command, the optional authoritative knowledge-base location, and verified search command. Do not include this reference or another worker's prompt. Supporting code may be outside a requested path, but an affected entry point, control, or operation must be in scope. -6. Before combining or revalidating any returned baseline or investigator result, persist it through the caller's bound `record_codex_security_scan_draft` tool when available, using `complete: false` and partial coverage. Give each candidate a stable `candidateId`; put candidates awaiting parent validation in `coverage.deferred` with a meaningful reason and their original finding payload under `candidate`. Preserve returned counterevidence and unresolved questions too. Checkpoint again after each validation decision, without waiting for other workers or the final report. Put source-validated findings in `findings` with the same `provenance.candidateId`; for a rejection, retain the candidate ID, original evidence, and source-backed counterevidence on a `rejected` coverage surface. An unfinished scan must retain its saved findings and pending candidates without presenting pending work as validated. Reconcile source coverage before combining findings. Union only the baseline and focused investigators' `fully_reviewed_files` with files the parent fully security-audited, then intersect that set with the supplied authorized inventory or an inventory of the selected current scope. Architecture mapping alone and supporting files outside that inventory do not count toward completed audit coverage. Finish the remaining in-scope files in coherent groups, reusing available investigators within the same allowance. Inspect implementation-owning generated or compressed code as data. Do not add overlapping worker counts or claim that a search hit completed a file. Keep this one transient set; do not create a separate progress ledger or receipt format. If a user limit or unavailable source prevents completion, identify the actual remaining paths and report partial coverage. Then combine baseline and investigator findings once. Group observations only when they share the same broken security control and effective remediation; preserve every affected route, operation, sink, and supporting source location. Never merge different security failures solely because they share a CWE. +6. Before combining or revalidating any returned baseline or investigator result, persist it through the caller's bound `record_codex_security_scan_draft` tool when available, using `complete: false` and partial coverage. Give each candidate a stable `candidateId`; put candidates awaiting parent validation in `coverage.deferred` with a meaningful reason and their original finding payload under `candidate`. Preserve returned counterevidence and unresolved questions too. Checkpoint again after each validation decision, without waiting for other workers or the final report. Reuse the saved candidate's `candidateId` (or its existing deferred `id`) when recording a validation decision. Put source-validated findings in `findings` with that `provenance.candidateId`; for a rejection, set the same `candidateId` on a `rejected` coverage surface and retain original evidence and source-backed counterevidence. A prose surface summary alone does not resolve a different candidate. Later drafts preserve omitted findings and unresolved candidates. An unfinished scan must retain its saved findings and pending candidates without presenting pending work as validated. Reconcile source coverage before combining findings. Union only the baseline and focused investigators' `fully_reviewed_files` with files the parent fully security-audited, then intersect that set with the supplied authorized inventory or an inventory of the selected current scope. Architecture mapping alone and supporting files outside that inventory do not count toward completed audit coverage. Finish the remaining in-scope files in coherent groups, reusing available investigators within the same allowance. Inspect implementation-owning generated or compressed code as data. Do not add overlapping worker counts or claim that a search hit completed a file. Keep this one transient set; do not create a separate progress ledger or receipt format. If a user limit or unavailable source prevents completion, identify the actual remaining paths and report partial coverage. Then combine baseline and investigator findings once. Group observations only when they share the same broken security control and effective remediation; preserve every affected route, operation, sink, and supporting source location. Never merge different security failures solely because they share a CWE. 7. Independently validate each unique finding against local source once. Establish its attacker, entry point, trust boundary, attacker-controlled dataflow, transformations, broken control, sensitive operation, prerequisites, effective mitigations, strongest counterevidence, and concrete impact. For secrets exposed in source, apply the disclosure path below instead of requiring attacker-controlled application input or execution of the containing code. Record concise, source-backed `rootCause.summary`, `validation.summary`, `attackPath.dataflow.summary`, and `attackPath.reachability.summary` alongside their supporting facts; determine impact, likelihood, and severity from those established facts. State optional configuration, dependency-version, or deployment prerequisites; do not require proof of a real deployment or runtime reproduction. A public library or parser boundary is sufficient when callers control the input. Reject only with source-backed counterevidence, preserve valid baseline findings, record material unresolved proof gaps, and apply the severity rules below. 8. Assemble complete semantic `scope`, `threatModel`, `findings`, and `coverage` following `final-report.md`; use `schemas/tools/scan-draft.schema.json` for draft tool arguments. Use the canonical field mapping and scenario reconciliation in `threat-model.md`, preserving supplied models unchanged and retaining source-backed architecture, capability, deployment, and uncertainty facts. Give each finding a stable lowercase vulnerability-family `ruleId`, its precise `taxonomy.category` and `taxonomy.cwe` values, genuine `provenance.source`, an instance when separately reported findings would otherwise collide, a `root_control` location when identifiable, all materially affected locations, calibrated severity and rationale, confidence and rationale, verified nonempty source evidence, attacker-to-sink reachability, and practical remediation. Write source evidence as `codeEvidence` entries with required `id`, `label`, `path`, `startLine`, `code`, and `explanation` fields; `endLine`, `language`, and `role` are optional. Use `code`, never `snippet`, and write new root-cause details as `rootCause`, never `root_cause`. Follow `finding-detail-fields.md` when constructing rich finding details. Use actual coverage surface labels and dispositions; report reviewed surfaces, explicit exclusions, deferred work, and unresolved questions honestly, and mark coverage `complete` only when the requested source scope was actually reviewed. Preserve every genuine finding, evidence item, user-supplied assumption, and unresolved proof gap in the caller's complete semantic result. diff --git a/plugins/codex-security/references/scan-contract.md b/plugins/codex-security/references/scan-contract.md index a244221963..771b6b0dfa 100644 --- a/plugins/codex-security/references/scan-contract.md +++ b/plugins/codex-security/references/scan-contract.md @@ -106,7 +106,7 @@ Use CWE taxonomy separately. Do not include file names, line numbers, scan IDs, `coverage.json` records scan scope and completion information. Standard and diff summaries also describe reviewed surfaces and outstanding work. -For a Deep parent scan, the host copies the configured paths into `includePaths` and `excludePaths` and sets `completeness` from the coordinator's outcome. A successful aggregate uses `complete`; `surfaces`, `explicitExclusions`, and `deferred` are empty arrays, and `openQuestions` is omitted. If the configured time limit expires before any review completes, the coordinator writes `partial` and records the explanation in `deferred`. Stopped outcomes follow the [stopped-result recovery rules](#stopped-result-recovery). +For a Deep parent scan, the host copies the configured paths into `includePaths` and `excludePaths`. Deep v2 aggregates preserve accepted source coverage in `surfaces`, `explicitExclusions`, `deferred`, `openQuestions`, and `reviews`, including recorded source provenance and receipt references. `completeness` is `partial` when an accepted source is partial, work is deferred, or a surface has disposition `needs_follow_up`; otherwise it is `unknown` when source coverage is unknown, and `complete` when neither condition applies. A successful aggregate or a sealed `completed` scan can therefore retain partial or unknown coverage. Report canonical coverage separately from lifecycle completion. If the configured time limit expires before any review completes, the coordinator writes `partial` and records the explanation in `deferred`. Stopped outcomes follow the [stopped-result recovery rules](#stopped-result-recovery). Each Deep worker writes an ordinary Standard result, including its own coverage. A reducer submits `record_codex_security_deep_reduction({ scanId, findings, scope?, threatModel? })`; its saved results and checkpoints contain the accepted findings and optional scope and threat-model context. diff --git a/plugins/codex-security/scripts/deep_scan_workbench.py b/plugins/codex-security/scripts/deep_scan_workbench.py index 3e06b0a656..039ce2a680 100644 --- a/plugins/codex-security/scripts/deep_scan_workbench.py +++ b/plugins/codex-security/scripts/deep_scan_workbench.py @@ -17,11 +17,29 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from deep_scan_config import resolve_deep_scan_config -from finalize_scan_contract import _read_scan_local_json +from finalize_scan_contract import ( + _read_scan_local_json, + open_scan_local_file_descriptor, + write_scan_local_bytes, +) from workbench.handoff import require_current_continuation from workbench.storage import create_private_directory -from workbench_target import directory_snapshot_regular_file_count -from workbench_validation import optional_text, require_uuid, user_context_argument +from workbench_saved_results import ( + _restore_published_outputs, + _selected_publication_digest_for_documents, + _snapshot_published_outputs, + _worker_checkpoint_head, +) +from workbench_scan_usage import capture_scan_usage_owner +from workbench_target import ( + directory_snapshot_regular_file_count, +) +from workbench_validation import ( + optional_text, + reject_non_finite_json, + require_uuid, + user_context_argument, +) DEEP_SCAN_WORKER_KINDS = ("setup", "discovery", "dedup") DEEP_SCAN_WORKER_STATUSES = ("queued", "running", "succeeded", "failed", "canceled") @@ -31,7 +49,12 @@ "invalid_discovery_artifacts", ) DEEP_SCAN_TERMINAL_REASONS = ("saturated", "capped") -DEEP_SCAN_WORKFLOW_VERSION = "deep-security-scan/v1" +DEEP_SCAN_WORKFLOW_VERSION = "deep-security-scan/v2" +SUPPORTED_DEEP_SCAN_WORKFLOWS = { + DEEP_SCAN_WORKFLOW_VERSION, + "deep-scan-mcp/v1", + "deep-security-scan/v1", +} DEEP_SCAN_COORDINATOR_LEASE_SECONDS = 30 DEEP_SCAN_LEGACY_COORDINATOR_GRACE_SECONDS = 120 DEEP_SCAN_MAX_ERROR_LENGTH = 2400 @@ -188,20 +211,50 @@ def require_deep_scan_run(connection: sqlite3.Connection, scan_id: str) -> sqlit return row +def require_supported_deep_scan(run: sqlite3.Row) -> None: + if run["schema_version"] != 1 or run["workflow_version"] not in SUPPORTED_DEEP_SCAN_WORKFLOWS: + raise SystemExit( + "This Deep Scan uses an unsupported workflow or schema version. " + "Resume it with a compatible Codex Security release." + ) + + finalization = deep_scan_finalization_input(run) + if finalization is not None and ( + run["workflow_version"] != "deep-security-scan/v2" + or not isinstance(finalization, dict) + or finalization.get("version") != 1 + ): + raise SystemExit("This Deep Scan uses an unsupported finalization input version.") + + +def deep_scan_finalization_input(run: sqlite3.Row) -> dict[str, Any] | None: + if "finalization_input_json" not in run.keys() or run["finalization_input_json"] is None: + return None + return json.loads(run["finalization_input_json"]) + + def deep_scan_deadline_reached(run: sqlite3.Row) -> bool: elapsed = _parse_timestamp(dependencies().now()) - _parse_timestamp(str(run["created_at"])) return elapsed.total_seconds() / 3600 >= run["max_time_hours"] +def find_supported_deep_scan_run( + connection: sqlite3.Connection, scan_id: str +) -> sqlite3.Row | None: + run = connection.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) + ).fetchone() + if run is not None: + require_supported_deep_scan(run) + return run + + def require_deep_scan_ready_for_parent_completion( connection: sqlite3.Connection, scan: sqlite3.Row ) -> None: if scan["mode"] != "deep": return - run = connection.execute( - "SELECT status, manifest_path FROM deep_scan_runs WHERE scan_id = ?", - (scan["id"],), - ).fetchone() + run = find_supported_deep_scan_run(connection, scan["id"]) if run is None or run["status"] != "succeeded" or run["manifest_path"] is None: raise SystemExit( "Deep Scan discovery orchestration must finish and persist its manifest before " @@ -329,20 +382,35 @@ def canonical_discovery_artifacts(scan: sqlite3.Row) -> dict[str, str]: def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, Any]: + if connection.in_transaction: + return _deep_scan_state(connection, scan_id) + connection.execute("BEGIN") + try: + state = _deep_scan_state(connection, scan_id) + connection.commit() + return state + except BaseException: + connection.rollback() + raise + + +def _deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, Any]: run = require_deep_scan_run(connection, scan_id) scan = dependencies().require_scan(connection, run["scan_id"]) worker_rows = connection.execute( """ - SELECT * - FROM deep_scan_workers - WHERE scan_id = ? - ORDER BY created_at, id + SELECT workers.*, attempts.accepted_result_path + FROM deep_scan_workers AS workers + LEFT JOIN deep_scan_attempts AS attempts + ON attempts.worker_id = workers.id AND attempts.attempt = workers.attempt + WHERE workers.scan_id = ? + ORDER BY workers.created_at, workers.id """, (run["scan_id"],), ) input_rows = connection.execute( """ - SELECT dedup_worker_id, discovery_worker_id, input_order + SELECT * FROM deep_scan_dedup_inputs WHERE scan_id = ? ORDER BY dedup_worker_id, input_order @@ -370,12 +438,26 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A ) return { "scanId": run["scan_id"], + "ownerThreadId": scan["deep_scan_owner_thread_id"] + or dependencies().require_workspace(connection, scan["workspace_id"])["thread_id"], "targetPath": scan["target_path"], "scope": scan["scope"], - "userContext": scan["user_context"], + "model": scan["model"], + "reasoningEffort": scan["reasoning_effort"], + "userContext": ( + run["discovery_user_context"] + if "discovery_user_context" in run.keys() + else scan["user_context"] + ), "scanDir": scan["scan_dir"], "schemaVersion": run["schema_version"], "workflowVersion": run["workflow_version"], + "finalizationInput": deep_scan_finalization_input(run), + "usageOwner": ( + json.loads(run["usage_owner_json"]) + if "usage_owner_json" in run.keys() and run["usage_owner_json"] + else None + ), "coordinatorGeneration": run["coordinator_generation"], "status": run["status"], "phase": run["phase"], @@ -400,11 +482,55 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A "updatedAt": run["updated_at"], "completedAt": run["completed_at"], "workers": [deep_scan_worker_state(row) for row in worker_rows], + "attempts": [ + { + "workerId": row["worker_id"], + "attempt": row["attempt"], + "status": row["status"], + "startedAt": row["started_at"], + "completedAt": row["completed_at"], + "endReason": row["end_reason"], + "error": row["error_message"], + "acceptedResultPath": row["accepted_result_path"], + "acceptedResultSha256": row["accepted_result_sha256"], + } + for row in connection.execute( + "SELECT * FROM deep_scan_attempts WHERE scan_id = ? ORDER BY worker_id, attempt", + (scan_id,), + ) + ], + "attemptSessions": [ + { + "workerId": row["worker_id"], + "attempt": row["attempt"], + "sdkThreadId": row["sdk_thread_id"], + "observedAt": row["observed_at"], + } + for row in connection.execute( + "SELECT * FROM deep_scan_attempt_sessions WHERE scan_id = ? " + "ORDER BY observed_at, worker_id, attempt, sdk_thread_id", + (scan_id,), + ) + ], + "mergeClaims": [ + { + "workerId": row["worker_id"], + "previousWorkerId": row["previous_worker_id"], + "previousResultPath": row["previous_result_path"], + "previousResultSha256": row["previous_result_sha256"], + } + for row in connection.execute( + "SELECT * FROM deep_scan_merge_claims WHERE scan_id = ? ORDER BY rowid", (scan_id,) + ) + ], "dedupInputs": [ { "dedupWorkerId": row["dedup_worker_id"], "discoveryWorkerId": row["discovery_worker_id"], "inputOrder": row["input_order"], + "resultManifestPath": row["result_manifest_path"], + "resultManifestSha256": row["result_manifest_sha256"], + "attempt": row["attempt"], } for row in input_rows ], @@ -452,6 +578,9 @@ def deep_scan_worker_state(row: sqlite3.Row) -> dict[str, Any]: "promptPath": row["prompt_path"], "artifactDir": row["artifact_dir"], "resultManifestPath": row["result_manifest_path"], + "acceptedResultPath": row["accepted_result_path"] + if "accepted_result_path" in row.keys() + else None, "attempt": row["attempt"], "sdkThreadId": row["sdk_thread_id"], "completionSequence": row["completion_sequence"], @@ -480,30 +609,73 @@ def effective_deep_scan_config(args: argparse.Namespace) -> dict[str, int | floa return resolve_deep_scan_config(available_parallelism) +def recorded_deep_scan_execution_settings(run: sqlite3.Row) -> dict[str, Any] | None: + saved = run["execution_settings_json"] if "execution_settings_json" in run.keys() else None + return json.loads(saved) if saved else None + + +def include_execution_settings(connection: sqlite3.Connection, result: dict[str, Any]) -> None: + if "deepScan" in result: + run = require_deep_scan_run(connection, result["deepScan"]["scanId"]) + result["deepScan"]["executionSettings"] = recorded_deep_scan_execution_settings(run) + + +def validate_deep_scan_execution_settings(saved: dict[str, Any]) -> dict[str, Any]: + if saved.get("version") != 1: + raise SystemExit("This Deep Scan uses an unsupported execution settings version.") + settings = saved.get("settings") + if not isinstance(settings, dict) or not all( + isinstance(settings.get(key), str) for key in ("codexPath", "codexHome") + ): + raise SystemExit( + "Deep Scan execution settings are missing the recorded executable or Codex home." + ) + return settings + + def ensure_deep_scan_run( connection: sqlite3.Connection, scan: sqlite3.Row, config: dict[str, int | float], workflow_version: str, timestamp: str, + args: argparse.Namespace | None = None, ) -> sqlite3.Row: existing = connection.execute( "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan["id"],) ).fetchone() if existing is not None: + require_supported_deep_scan(existing) return existing if scan["mode"] != "deep": raise SystemExit("Deep Scan orchestration requires a scan in deep mode.") if scan["status"] != "running": raise SystemExit("Only a running Deep Scan can start orchestration.") + saved_settings = None + if args is not None and hasattr(args, "execution_settings"): + if args.execution_settings is None: + raise SystemExit("The original Deep Scan execution settings could not be captured.") + settings = dict(args.execution_settings) + for key, column in (("model", "model"), ("reasoningEffort", "reasoning_effort")): + if scan[column] is not None: + settings[key] = scan[column] + saved_settings = {"version": 1, "settings": settings} + validate_deep_scan_execution_settings(saved_settings) + # The workbench transaction binds original settings. This artifact is a + # projection and cannot choose a fresh or resumed executable or home. + write_scan_local_bytes( + Path(scan["scan_dir"]), + "artifacts/deep_discovery/execution-settings.json", + (json.dumps(saved_settings, indent=2) + "\n").encode(), + ) connection.execute( """ INSERT INTO deep_scan_runs ( scan_id, schema_version, workflow_version, status, phase, workers, subagents, stop_after_no_new, stop_after_consecutive_errors, - max_discovery_runs, max_time_hours, - created_at, updated_at - ) VALUES (?, 1, ?, 'running', 'setup', ?, ?, ?, ?, ?, ?, ?, ?) + max_discovery_runs, max_time_hours, discovery_user_context, + created_at, updated_at, execution_settings_json + ) VALUES (?, 1, ?, 'running', 'setup', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( scan["id"], @@ -514,11 +686,20 @@ def ensure_deep_scan_run( config["stopAfterConsecutiveErrors"], config["maxDiscoveryRuns"], config["maxTimeHours"], + scan["user_context"], timestamp, timestamp, + json.dumps(saved_settings) if saved_settings is not None else None, ), ) - return require_deep_scan_run(connection, scan["id"]) + run = require_deep_scan_run(connection, scan["id"]) + if "usage_owner_json" in run.keys(): + connection.execute( + "UPDATE deep_scan_runs SET usage_owner_json = ? WHERE scan_id = ?", + (json.dumps(capture_scan_usage_owner(connection, scan)), scan["id"]), + ) + run = require_deep_scan_run(connection, scan["id"]) + return run def existing_deep_scan_for_target( @@ -606,6 +787,11 @@ def begin_deep_scan_for_scan( ) -> dict[str, Any]: scan_id = require_uuid(scan_id, "scan-id") candidate = dependencies().require_scan(connection, scan_id) + existing = connection.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) + ).fetchone() + if existing is not None: + require_supported_deep_scan(existing) workspace = dependencies().require_workspace(connection, candidate["workspace_id"]) if ( candidate["mode"] == "deep" @@ -643,23 +829,10 @@ def begin_deep_scan_for_scan( ) if scan["mode"] != "deep": raise SystemExit("Deep Scan orchestration requires a scan in deep mode.") - model = optional_text(args.model, maximum=200) - reasoning_effort = optional_text(args.reasoning_effort, maximum=32) - if model is not None or reasoning_effort is not None: - connection.execute( - """ - UPDATE scans - SET model = COALESCE(?, model), reasoning_effort = COALESCE(?, reasoning_effort) - WHERE id = ? - """, - (model, reasoning_effort, scan_id), - ) - connection.commit() - existing = connection.execute( - "SELECT scan_id FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) - ).fetchone() if existing is not None: return deep_scan_result(connection, scan_id, start_disposition="joined") + model = optional_text(args.model, maximum=200) + reasoning_effort = optional_text(args.reasoning_effort, maximum=32) config = effective_deep_scan_config(args) workflow_version = optional_text(args.workflow_version, maximum=256) if workflow_version is None: @@ -672,7 +845,25 @@ def begin_deep_scan_for_scan( args.claim_token, error_message="Deep Scan orchestration is owned by another continuation.", ) - ensure_deep_scan_run(connection, scan, config, workflow_version, dependencies().now()) + existing = connection.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) + ).fetchone() + if existing is not None: + require_supported_deep_scan(existing) + connection.commit() + return deep_scan_result(connection, scan_id, start_disposition="joined") + if model is not None or reasoning_effort is not None: + connection.execute( + """ + UPDATE scans + SET model = COALESCE(?, model), reasoning_effort = COALESCE(?, reasoning_effort) + WHERE id = ? + """, + (model, reasoning_effort, scan_id), + ) + scan = dependencies().require_scan(connection, scan_id) + ensure_deep_scan_run(connection, scan, config, workflow_version, dependencies().now(), args) + connection.commit() return deep_scan_result(connection, scan_id, start_disposition="created") @@ -697,15 +888,17 @@ def begin_deep_scan_for_target( existing = existing_deep_scan_for_target(connection, thread_id, target_path, scope) if existing is not None: existing_run = connection.execute( - "SELECT 1 FROM deep_scan_runs WHERE scan_id = ?", (existing["id"],) + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (existing["id"],) ).fetchone() + if existing_run is not None: + require_supported_deep_scan(existing_run) if existing_run is None: config = effective_deep_scan_config(args) workflow_version = optional_text(args.workflow_version, maximum=256) if workflow_version is None: raise SystemExit("workflow-version is required.") ensure_deep_scan_run( - connection, existing, config, workflow_version, dependencies().now() + connection, existing, config, workflow_version, dependencies().now(), args ) connection.commit() return deep_scan_result( @@ -801,11 +994,15 @@ def begin_deep_scan_for_target( scan_dir=scan_dir, ) scan = dependencies().require_scan(connection, scan_id) - ensure_deep_scan_run(connection, scan, config, workflow_version, timestamp) + ensure_deep_scan_run(connection, scan, config, workflow_version, timestamp, args) + connection.commit() return deep_scan_result(connection, scan_id, start_disposition="created") def begin_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]: + workflow_version = optional_text(args.workflow_version, maximum=256) + if workflow_version not in SUPPORTED_DEEP_SCAN_WORKFLOWS: + raise SystemExit("This Deep Scan uses an unsupported workflow version.") thread_id = optional_text(args.thread_id, maximum=512) if thread_id is None: raise SystemExit("thread-id is required.") @@ -853,6 +1050,8 @@ def coordinator_lease_is_live( try: heartbeat = json.loads(heartbeat_path.read_text(encoding="utf-8")) if heartbeat["coordinatorGeneration"] == run["coordinator_generation"]: + if heartbeat.get("released") is True: + return False heartbeat_time = max(heartbeat_time, _parse_timestamp(heartbeat["updatedAt"])) except (OSError, KeyError, TypeError, ValueError): pass @@ -861,6 +1060,7 @@ def coordinator_lease_is_live( def require_current_coordinator(run: sqlite3.Row, args: argparse.Namespace) -> None: + require_supported_deep_scan(run) generation = getattr(args, "coordinator_generation", None) if run["coordinator_generation"] == 1: if generation is not None: @@ -891,7 +1091,25 @@ def claim_deep_scan_coordinator_locked( args.claim_token, error_message="Deep Scan orchestration is owned by another continuation.", ) - run, _ = require_running_deep_scan(connection, scan_id) + run = require_deep_scan_run(connection, scan_id) + require_supported_deep_scan(run) + selected_parent = ( + run["status"] == "succeeded" and deep_scan_finalization_input(run) is not None + ) + if selected_parent and scan["status"] == "complete": + connection.commit() + return { + **deep_scan_result(connection, scan_id), + "coordinatorDisposition": "observing", + } + if not selected_parent: + run, _ = require_running_deep_scan(connection, scan_id) + elif ( + run["cancel_requested"] + or scan["status"] != "running" + or scan["canceled_at"] is not None + ): + raise SystemExit("Only a running scan can complete its selected Deep Scan result.") timestamp = dependencies().now() if args.coordinator_generation is not None: require_current_coordinator(run, args) @@ -904,15 +1122,27 @@ def claim_deep_scan_coordinator_locked( } else: adopted = run["coordinator_generation"] > 1 or run["phase"] != "setup" - if adopted: - recover_expired_coordinator(connection, run, timestamp) disposition = "adopted" if adopted else "claimed" + if ( + getattr(args, "require_execution_settings", False) + and deep_scan_finalization_input(run) is None + ): + saved = recorded_deep_scan_execution_settings(run) + if saved is not None: + validate_deep_scan_execution_settings(saved) + elif run["workflow_version"] == DEEP_SCAN_WORKFLOW_VERSION: + raise SystemExit( + "This Deep Scan has no recorded original execution settings; " + "its executable and Codex home cannot be recovered." + ) + if disposition == "adopted": + recover_expired_coordinator(connection, run, timestamp) connection.execute( """ UPDATE deep_scan_runs SET coordinator_generation = coordinator_generation + ?, updated_at = ? - WHERE scan_id = ? AND status = 'running' + WHERE scan_id = ? AND status IN ('running', 'succeeded') """, (int(args.coordinator_generation != run["coordinator_generation"]), timestamp, scan_id), ) @@ -925,6 +1155,9 @@ def claim_deep_scan_coordinator_locked( def recover_expired_coordinator( connection: sqlite3.Connection, run: sqlite3.Row, timestamp: str ) -> None: + require_supported_deep_scan(run) + if deep_scan_finalization_input(run) is not None: + return scan_id = run["scan_id"] recover_candidate_ledger_publication(connection, scan_id) legacy_generation = int(run["coordinator_generation"] == 1) @@ -1051,6 +1284,7 @@ def require_running_deep_scan( connection: sqlite3.Connection, scan_id: str ) -> tuple[sqlite3.Row, sqlite3.Row]: run = require_deep_scan_run(connection, scan_id) + require_supported_deep_scan(run) scan = dependencies().require_scan(connection, run["scan_id"]) if run["status"] != "running" or run["cancel_requested"]: raise SystemExit("Only a running Deep Scan can update orchestration state.") @@ -1071,6 +1305,132 @@ def require_worker_transition(current: str, requested: str) -> None: raise SystemExit(f"Deep Scan worker cannot transition from {current} to {requested}.") +def snapshot_accepted_result(scan: sqlite3.Row, worker: sqlite3.Row) -> tuple[str, str]: + source = deep_scan_path( + scan, worker["result_manifest_path"], "Accepted worker result", kind="file" + ) + scan_dir = Path(scan["scan_dir"]) + contents = Path(source).read_bytes() + semantic = json.loads(contents) + if isinstance(semantic, dict): + semantic.pop("handoffClaimToken", None) + directory = Path(worker["artifact_dir"]) / "checkpoints" + head = ( + _worker_checkpoint_head( + scan_dir, Path(worker["artifact_dir"]).relative_to(scan_dir).as_posix(), scan["id"] + ) + if worker["kind"] == "discovery" + else None + ) + candidates = [scan_dir / head] if head else sorted(directory.glob("*.json")) + for checkpoint in candidates: + safe = deep_scan_path(scan, str(checkpoint), "Accepted worker checkpoint", kind="file") + checkpoint_bytes = Path(safe).read_bytes() + if json.loads(checkpoint_bytes) == semantic: + return safe, hashlib.sha256(checkpoint_bytes).hexdigest() + if head: + raise SystemExit( + "The accepted worker result does not match its current checkpoint head." + ) + # Legacy/direct file producers may have no checkpoint. Use the existing native + # checkpoint store; typed artifact writers already supplied the matching copy. + digest = hashlib.sha256(contents).hexdigest() + destination = directory / f"{digest}.json" + if destination.exists(): + raise SystemExit("An existing worker checkpoint does not match its accepted content.") + write_scan_local_bytes(scan_dir, destination.relative_to(scan_dir).as_posix(), contents) + return str(destination), digest + + +def record_worker_attempt( + connection: sqlite3.Connection, + scan: sqlite3.Row, + worker: sqlite3.Row, + timestamp: str, + *, + observed_thread_id: str | None = None, + error: str | None = None, + end_reason: str | None = None, +) -> None: + if worker["status"] == "queued" or worker["attempt"] < 1: + return + connection.execute( + """ + UPDATE deep_scan_attempts + SET status = 'replaced', completed_at = ?, end_reason = 'replacement_attempt' + WHERE worker_id = ? AND attempt < ? AND completed_at IS NULL + """, + (timestamp, worker["id"], worker["attempt"]), + ) + status = worker["status"] + if end_reason in DEEP_SCAN_REPLACEABLE_FAILURE_KINDS: + status = "failed" + if status != "running": + error = worker["error_message"] + if status == "running" and error: + status = "failed" + completed = timestamp if status != "running" else None + reason = end_reason or ( + "execution_or_artifact_error" if status == "failed" else status if completed else None + ) + accepted_path = accepted_sha = None + if status == "succeeded" and worker["result_manifest_path"]: + accepted_path, accepted_sha = snapshot_accepted_result(scan, worker) + connection.execute( + """ + INSERT INTO deep_scan_attempts ( + scan_id, worker_id, attempt, status, started_at, completed_at, + end_reason, error_message, accepted_result_path, accepted_result_sha256 + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(worker_id, attempt) DO UPDATE SET + status = CASE WHEN deep_scan_attempts.completed_at IS NULL THEN excluded.status + ELSE deep_scan_attempts.status END, + completed_at = COALESCE(deep_scan_attempts.completed_at, excluded.completed_at), + end_reason = COALESCE(deep_scan_attempts.end_reason, excluded.end_reason), + error_message = COALESCE(excluded.error_message, deep_scan_attempts.error_message), + accepted_result_path = COALESCE(excluded.accepted_result_path, + deep_scan_attempts.accepted_result_path), + accepted_result_sha256 = COALESCE(excluded.accepted_result_sha256, + deep_scan_attempts.accepted_result_sha256) + """, + ( + scan["id"], + worker["id"], + worker["attempt"], + status, + timestamp, + completed, + reason, + error, + accepted_path, + accepted_sha, + ), + ) + if observed_thread_id: + connection.execute( + """ + INSERT OR IGNORE INTO deep_scan_attempt_sessions ( + scan_id, worker_id, attempt, sdk_thread_id, observed_at + ) VALUES (?, ?, ?, ?, ?) + """, + (scan["id"], worker["id"], worker["attempt"], observed_thread_id, timestamp), + ) + + +def worker_result_reference( + connection: sqlite3.Connection, scan: sqlite3.Row, worker: sqlite3.Row +) -> tuple[str, str]: + accepted = connection.execute( + "SELECT accepted_result_path, accepted_result_sha256 FROM deep_scan_attempts " + "WHERE worker_id = ? AND attempt = ?", + (worker["id"], worker["attempt"]), + ).fetchone() + if accepted is not None and accepted["accepted_result_path"]: + return accepted["accepted_result_path"], accepted["accepted_result_sha256"] + # Old accepted workers have no attempt history; freeze their current accepted result on claim. + return snapshot_accepted_result(scan, worker) + + def upsert_deep_scan_worker( connection: sqlite3.Connection, args: argparse.Namespace ) -> dict[str, Any]: @@ -1114,11 +1474,14 @@ def upsert_deep_scan_worker( scan, args.artifact_dir, "Worker artifact directory", kind="directory" ) result_manifest_path = ( - deep_scan_path( - scan, - args.result_manifest_path, - "Worker result manifest path", - kind="file", + ( + deep_scan_output_path( + scan, args.result_manifest_path, "Worker result manifest path" + ) + if terminal_repeat + else deep_scan_path( + scan, args.result_manifest_path, "Worker result manifest path", kind="file" + ) ) if args.result_manifest_path else None @@ -1177,8 +1540,18 @@ def upsert_deep_scan_worker( timestamp, ), ) + record_worker_attempt( + connection, + scan, + require_deep_scan_worker(connection, worker_id), + timestamp, + observed_thread_id=optional_text(args.sdk_thread_id, maximum=512), + error=optional_text(args.error_message, maximum=2400), + end_reason=args.replaceable_failure_kind, + ) + result = deep_scan_result(connection, scan_id) connection.commit() - return deep_scan_result(connection, scan_id) + return result if existing["scan_id"] != scan_id or existing["kind"] != args.kind: raise SystemExit("Deep Scan worker identity does not match its persisted run and kind.") @@ -1201,8 +1574,15 @@ def upsert_deep_scan_worker( and repeated_error != existing["error_message"] ): raise SystemExit("Deep Scan worker terminal state is immutable.") + receipt = connection.execute( + "SELECT receipt_json FROM deep_scan_attempts WHERE worker_id = ? AND attempt = ?", + (worker_id, existing["attempt"]), + ).fetchone() + result = deep_scan_result(connection, scan_id) + if receipt is not None and receipt["receipt_json"]: + result["deepScan"]["workerReceipt"] = json.loads(receipt["receipt_json"]) connection.commit() - return deep_scan_result(connection, scan_id) + return result attempt = args.attempt if args.attempt is not None else existing["attempt"] if attempt < existing["attempt"]: raise SystemExit("Deep Scan worker attempt cannot decrease.") @@ -1298,7 +1678,27 @@ def upsert_deep_scan_worker( worker_id, ), ) - return deep_scan_result(connection, scan_id) + record_worker_attempt( + connection, + scan, + require_deep_scan_worker(connection, worker_id), + timestamp, + observed_thread_id=optional_text(args.sdk_thread_id, maximum=512), + error=optional_text(args.error_message, maximum=2400), + end_reason=args.replaceable_failure_kind, + ) + result = deep_scan_result(connection, scan_id) + if args.status in {"succeeded", "failed", "canceled"}: + receipt = next( + worker for worker in result["deepScan"]["workers"] if worker["id"] == worker_id + ) + result["deepScan"]["workerReceipt"] = receipt + connection.execute( + "UPDATE deep_scan_attempts SET receipt_json = ? WHERE worker_id = ? AND attempt = ?", + (json.dumps(receipt), worker_id, attempt), + ) + connection.commit() + return result def claim_deep_scan_dedup( @@ -1311,7 +1711,8 @@ def claim_deep_scan_dedup( raise SystemExit("Dedup input worker IDs must be unique.") connection.execute("BEGIN IMMEDIATE") with connection: - run, scan = require_running_deep_scan(connection, scan_id) + run = require_deep_scan_run(connection, scan_id) + scan = dependencies().require_scan(connection, scan_id) require_current_coordinator(run, args) prompt_path = deep_scan_path(scan, args.prompt_path, "Dedup prompt path", kind="file") artifact_dir = deep_scan_path( @@ -1340,9 +1741,11 @@ def claim_deep_scan_dedup( and existing["artifact_dir"] == artifact_dir and persisted_inputs == input_ids ): + result = deep_scan_result(connection, scan_id) connection.commit() - return deep_scan_result(connection, scan_id) + return result raise SystemExit("Dedup worker ID is already used by a different reducer claim.") + require_running_deep_scan(connection, scan_id) active_reducer = connection.execute( """ SELECT 1 FROM deep_scan_workers @@ -1410,14 +1813,41 @@ def claim_deep_scan_dedup( """, (worker_id, scan_id, prompt_path, artifact_dir, timestamp, timestamp), ) + previous = connection.execute( + "SELECT * FROM deep_scan_workers WHERE scan_id = ? AND kind = 'dedup' " + "AND status = 'succeeded' ORDER BY completed_at DESC, rowid DESC LIMIT 1", + (scan_id,), + ).fetchone() + previous_path, previous_sha = ( + worker_result_reference(connection, scan, previous) if previous else (None, None) + ) + connection.execute( + """ + INSERT INTO deep_scan_merge_claims ( + worker_id, scan_id, previous_worker_id, previous_result_path, previous_result_sha256 + ) VALUES (?, ?, ?, ?, ?) + """, + (worker_id, scan_id, previous["id"] if previous else None, previous_path, previous_sha), + ) for input_order, input_id in enumerate(input_ids): + discovery = require_deep_scan_worker(connection, input_id) + accepted_path, accepted_sha = worker_result_reference(connection, scan, discovery) connection.execute( """ INSERT INTO deep_scan_dedup_inputs ( - scan_id, dedup_worker_id, discovery_worker_id, input_order - ) VALUES (?, ?, ?, ?) + scan_id, dedup_worker_id, discovery_worker_id, input_order, + result_manifest_path, result_manifest_sha256, attempt + ) VALUES (?, ?, ?, ?, ?, ?, ?) """, - (scan_id, worker_id, input_id, input_order), + ( + scan_id, + worker_id, + input_id, + input_order, + accepted_path, + accepted_sha, + discovery["attempt"], + ), ) connection.execute( f""" @@ -1439,7 +1869,8 @@ def claim_deep_scan_dedup( """, (timestamp, scan_id), ) - return deep_scan_result(connection, scan_id) + result = deep_scan_result(connection, scan_id) + return result def commit_deep_scan_dedup( @@ -1465,8 +1896,14 @@ def commit_deep_scan_dedup_locked( if worker["scan_id"] != scan_id or worker["kind"] != "dedup": raise SystemExit("Dedup worker does not belong to this Deep Scan.") if worker["status"] == "succeeded": + receipt = connection.execute( + "SELECT receipt_json FROM deep_scan_merge_claims WHERE worker_id = ?", (worker_id,) + ).fetchone() + result = deep_scan_result(connection, scan_id) + if receipt is not None and receipt["receipt_json"]: + result["deepScan"]["committedMerge"] = json.loads(receipt["receipt_json"]) connection.commit() - return deep_scan_result(connection, scan_id) + return result require_running_deep_scan(connection, scan_id) if worker["status"] not in {"queued", "running"}: raise SystemExit("Only an active dedup worker can commit a result.") @@ -1512,6 +1949,23 @@ def commit_deep_scan_dedup_locked( ) if not inputs or any(row["merge_state"] != "merging" for row in inputs): raise SystemExit("Dedup inputs are not in the claimed merging state.") + claim = connection.execute( + "SELECT * FROM deep_scan_merge_claims WHERE worker_id = ?", (worker_id,) + ).fetchone() + references = [ + (row["result_manifest_path"], row["result_manifest_sha256"]) + for row in connection.execute( + "SELECT * FROM deep_scan_dedup_inputs WHERE dedup_worker_id = ? ORDER BY input_order", + (worker_id,), + ) + ] + if claim is not None and claim["previous_result_path"]: + references.append((claim["previous_result_path"], claim["previous_result_sha256"])) + for path, digest in references: + if path is not None and digest is not None: + safe_path = deep_scan_path(scan, path, "Claimed reducer input", kind="file") + if hashlib.sha256(Path(safe_path).read_bytes()).hexdigest() != digest: + raise SystemExit("A claimed Deep Scan reducer input changed after acceptance.") if candidate_ledger_path and canonical_candidate_ledger_path: canonical_path = Path(canonical_candidate_ledger_path) publication_copy = canonical_path.with_name( @@ -1555,6 +2009,27 @@ def commit_deep_scan_dedup_locked( """, (no_new_streak, timestamp, scan_id), ) + committed_worker = require_deep_scan_worker(connection, worker_id) + record_worker_attempt( + connection, + scan, + committed_worker, + timestamp, + observed_thread_id=committed_worker["sdk_thread_id"], + ) + accepted_path, accepted_sha = worker_result_reference(connection, scan, committed_worker) + result = deep_scan_result(connection, scan_id) + result["deepScan"]["committedMerge"] = { + "workerId": worker_id, + "resultManifestPath": accepted_path, + "resultManifestSha256": accepted_sha, + "newFindings": args.new_findings_count, + } + connection.execute( + "INSERT INTO deep_scan_merge_claims (worker_id, scan_id, receipt_json) VALUES (?, ?, ?) " + "ON CONFLICT(worker_id) DO UPDATE SET receipt_json = excluded.receipt_json", + (worker_id, scan_id, json.dumps(result["deepScan"]["committedMerge"])), + ) connection.commit() except BaseException: connection.rollback() @@ -1567,10 +2042,20 @@ def commit_deep_scan_dedup_locked( finish_staged_file(promotion) if publication_copy is not None: publication_copy.unlink(missing_ok=True) - return deep_scan_result(connection, scan_id) + return result -def finish_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]: +def finish_deep_scan( + connection: sqlite3.Connection, args: argparse.Namespace, select_finalization: bool = False +) -> dict[str, Any]: + if select_finalization: + import sys + + args = argparse.Namespace( + **vars(args), + select_finalization=True, + finalization_result_path=json.load(sys.stdin)["resultPath"], + ) scan_id = require_uuid(args.scan_id, "scan-id") with dependencies().scan_completion_lock(scan_id): return finish_deep_scan_locked(connection, args, scan_id) @@ -1584,15 +2069,26 @@ def finish_deep_scan_locked( ] if len(set(omitted_worker_ids)) != len(omitted_worker_ids): raise SystemExit("Omitted Deep Scan worker IDs must be unique.") + selecting = getattr(args, "select_finalization", False) promotion: tuple[Path, Path, Path | None] | None = None connection.execute("BEGIN IMMEDIATE") try: run = require_deep_scan_run(connection, scan_id) require_current_coordinator(run, args) scan = dependencies().require_scan(connection, scan_id) + if selecting and run["workflow_version"] != "deep-security-scan/v2": + raise SystemExit("Selected finalization requires the supported v2 workflow.") + finalization = deep_scan_finalization_input(run) + if finalization is not None and ( + args.terminal_reason != finalization["terminalReason"] + or omitted_worker_ids != finalization["omittedWorkerIds"] + ): + raise SystemExit( + "Deep Scan finalization must retain its selected reason and omissions." + ) manifest_path = ( deep_scan_output_path(scan, args.manifest_path, "Deep Scan coordinator manifest path") - if args.staged_manifest_path + if args.staged_manifest_path or selecting else deep_scan_path( scan, args.manifest_path, "Deep Scan coordinator manifest path", kind="file" ) @@ -1602,6 +2098,7 @@ def finish_deep_scan_locked( failure_capped = False if ( standard_scan_manifest + and not selecting and args.terminal_reason == "capped" and (run["status"] == "running" or omitted_worker_ids) ): @@ -1708,7 +2205,10 @@ def finish_deep_scan_locked( "Deep Scan cannot finish capped before reaching its configured maximum." ) canonical_artifacts = None - if standard_scan_manifest: + if selecting: + if not standard_scan_manifest: + raise SystemExit("Selected Deep Scan finalization requires the parent manifest.") + elif standard_scan_manifest: for artifact_name in ("scan-manifest.json", "findings.json", "coverage.json"): deep_scan_path( scan, @@ -1812,6 +2312,17 @@ def finish_deep_scan_locked( f"Deep Scan {args.terminal_reason} completion must exactly identify all buffered discovery " "workers with --omitted-worker-id." ) + if selecting: + selection = selected_deep_scan_finalization( + connection, run, scan, args, omitted_worker_ids, zero_discovery_deadline + ) + connection.execute( + "UPDATE deep_scan_runs SET finalization_input_json = ?, terminal_reason = ?, " + "phase = 'terminal', updated_at = ? WHERE scan_id = ?", + (json.dumps(selection), selection["terminalReason"], dependencies().now(), scan_id), + ) + connection.commit() + return deep_scan_result(connection, scan_id) if args.staged_manifest_path: staged_manifest_path = deep_scan_path( scan, @@ -1842,6 +2353,165 @@ def finish_deep_scan_locked( return deep_scan_result(connection, scan_id) +def selected_deep_scan_finalization( + connection: sqlite3.Connection, + run: sqlite3.Row, + scan: sqlite3.Row, + args: argparse.Namespace, + omitted_worker_ids: list[str], + zero_discovery_deadline: bool, +) -> dict[str, Any]: + """Select the committed attempt's immutable aggregate before publication.""" + if run["finalization_input_json"] is not None: + return json.loads(run["finalization_input_json"]) + result_path = getattr(args, "finalization_result_path", None) + relative: str | None = None + digest: str | None = None + if result_path is None: + if not zero_discovery_deadline: + raise SystemExit("Deep Scan finalization requires its accepted reducer result.") + else: + accepted = connection.execute( + "SELECT attempts.accepted_result_path, attempts.accepted_result_sha256 " + "FROM deep_scan_workers AS workers LEFT JOIN deep_scan_attempts AS attempts " + "ON attempts.worker_id = workers.id AND attempts.attempt = workers.attempt " + "WHERE workers.id = (SELECT id FROM deep_scan_workers WHERE scan_id = ? " + "AND kind = 'dedup' AND status = 'succeeded' ORDER BY completed_at DESC, id DESC LIMIT 1) " + "AND (workers.result_manifest_path = ? OR attempts.accepted_result_path = ?)", + (scan["id"], result_path, result_path), + ).fetchone() + if ( + accepted is None + or not accepted["accepted_result_path"] + or not accepted["accepted_result_sha256"] + ): + raise SystemExit( + "Deep Scan finalization requires its committed accepted reducer reference." + ) + scan_dir = Path(scan["scan_dir"]) + source = Path( + deep_scan_path( + scan, accepted["accepted_result_path"], "Selected Deep Scan result", kind="file" + ) + ) + relative = source.relative_to(scan_dir).as_posix() + digest = accepted["accepted_result_sha256"] + selection = { + "version": 1, + "resultPath": relative, + "resultSha256": digest, + "terminalReason": args.terminal_reason, + "omittedWorkerIds": omitted_worker_ids, + "selectedAt": dependencies().now(), + } + return selection + + +def budget_unmerged_workers(connection: sqlite3.Connection, scan_id: str) -> list[sqlite3.Row]: + return connection.execute( + "SELECT workers.*, attempts.accepted_result_path, attempts.accepted_result_sha256 " + "FROM deep_scan_workers AS workers JOIN deep_scan_attempts AS attempts " + "ON attempts.worker_id = workers.id AND attempts.attempt = workers.attempt " + "WHERE workers.scan_id = ? AND workers.kind = 'discovery' " + "AND workers.status = 'succeeded' AND workers.merge_state IN ('buffered', 'merging') " + "ORDER BY workers.completion_sequence, workers.id", + (scan_id,), + ).fetchall() + + +def prepare_budget_exhausted_deep_scan( + connection: sqlite3.Connection, + scan: sqlite3.Row, + scan_dir: Path, + warning: str, + write_draft: Callable[[dict[str, Any] | None, list[sqlite3.Row]], None], +) -> None: + """Finish local budget publication without starting or promoting scan work.""" + snapshots = _snapshot_published_outputs(scan_dir) + connection.execute("BEGIN IMMEDIATE") + try: + # The completion lock and this transaction preserve any committed selector. + run, scan = require_running_deep_scan(connection, scan["id"]) + unmerged = budget_unmerged_workers(connection, scan["id"]) + if run["finalization_input_json"] is not None: + selection = json.loads(run["finalization_input_json"]) + else: + reducer = connection.execute( + "SELECT attempts.accepted_result_path FROM deep_scan_workers AS workers " + "LEFT JOIN deep_scan_attempts AS attempts ON attempts.worker_id = workers.id " + "AND attempts.attempt = workers.attempt WHERE workers.scan_id = ? " + "AND workers.kind = 'dedup' AND workers.status = 'succeeded' " + "ORDER BY workers.completed_at DESC, workers.id DESC LIMIT 1", + (scan["id"],), + ).fetchone() + if reducer is not None and not reducer["accepted_result_path"]: + raise SystemExit("Budget completion requires the committed reducer reference.") + selection = selected_deep_scan_finalization( + connection, + run, + scan, + argparse.Namespace( + finalization_result_path=reducer["accepted_result_path"] if reducer else None, + terminal_reason="capped", + ), + [row["id"] for row in unmerged], + reducer is None, + ) + accepted = None + if selection["resultPath"] is not None: + descriptor = open_scan_local_file_descriptor( + scan_dir, selection["resultPath"], "Accepted reducer" + ) + with os.fdopen(descriptor, "rb") as source: + contents = source.read() + if hashlib.sha256(contents).hexdigest() != selection["resultSha256"]: + raise SystemExit("The accepted reducer changed before budget completion.") + accepted = json.loads(contents, parse_constant=reject_non_finite_json) + if ( + not isinstance(accepted, dict) + or accepted.get("scanId") != scan["id"] + or accepted.get("complete", True) is not True + or not isinstance(accepted.get("sourceCoverage"), dict) + ): + raise SystemExit("Budget completion requires the accepted complete reducer.") + write_draft(accepted, unmerged) + documents = tuple( + _read_scan_local_json(scan_dir, name, name) + for name in ("scan-manifest.json", "findings.json", "coverage.json") + ) + selection["publicationSha256"] = _selected_publication_digest_for_documents( + dependencies(), scan, documents + ) + timestamp = dependencies().now() + cancel_active_workers(connection, scan["id"], timestamp) + connection.execute( + "UPDATE deep_scan_runs SET status = 'succeeded', phase = 'terminal', " + "terminal_reason = ?, cancel_requested = 1, error_message = ?, " + "manifest_path = ?, finalization_input_json = ?, completed_at = ?, updated_at = ? " + "WHERE scan_id = ?", + ( + selection["terminalReason"], + warning, + str(scan_dir / "scan-manifest.json"), + json.dumps(selection), + timestamp, + timestamp, + scan["id"], + ), + ) + warnings = json.loads(scan["completion_warnings_json"]) + if warning not in warnings: + connection.execute( + "UPDATE scans SET completion_warnings_json = ? WHERE id = ?", + (json.dumps([*warnings, warning]), scan["id"]), + ) + connection.commit() + except BaseException: + connection.rollback() + _restore_published_outputs(scan_dir, snapshots) + raise + + def fail_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]: scan_id = require_uuid(args.scan_id, "scan-id") with dependencies().scan_completion_lock(scan_id): @@ -2020,6 +2690,11 @@ def cancel_from_parent_scan(connection: sqlite3.Connection, scan_id: str, timest def cancel_active_workers(connection: sqlite3.Connection, scan_id: str, timestamp: str) -> None: + connection.execute( + "UPDATE deep_scan_attempts SET status = 'canceled', completed_at = ?, " + "end_reason = 'scan_stopped' WHERE scan_id = ? AND completed_at IS NULL", + (timestamp, scan_id), + ) connection.execute( """ UPDATE deep_scan_workers diff --git a/plugins/codex-security/scripts/report_projection.py b/plugins/codex-security/scripts/report_projection.py index 196e4cb3c0..8589b04007 100644 --- a/plugins/codex-security/scripts/report_projection.py +++ b/plugins/codex-security/scripts/report_projection.py @@ -490,7 +490,14 @@ def _target_scope_lines(target: dict[str, Any]) -> list[str]: def _surface_notes(surface: dict[str, Any]) -> str: - notes = surface.get("notes", "No additional canonical notes were recorded.") + notes = surface.get( + "notes", surface.get("reason", "No additional canonical notes were recorded.") + ) + if "notes" in surface and "reason" in surface and surface["reason"] != notes: + notes += f" {surface['reason']}" + source = _coverage_source(surface) + if source: + notes = f"{source}. {notes}" receipt_refs = surface.get("receiptRefs", []) if not isinstance(receipt_refs, list) or not receipt_refs: return _cell(notes) @@ -500,6 +507,51 @@ def _surface_notes(surface: dict[str, Any]) -> str: return _cell(f"{notes} Evidence: {evidence}") +def _coverage_source(item: dict[str, Any]) -> str: + provenance = item.get("provenance", {}) + if not isinstance(provenance, dict) or not provenance.get("workerId"): + return "" + source = f"Review {provenance['workerId']}" + if provenance.get("attempt") is not None: + source += f", attempt {provenance['attempt']}" + return source + + +def _remediation_section(finding: dict[str, Any]) -> list[str]: + remediation = _text(finding.get("remediation"), "No canonical remediation was recorded.") + lines = ["", "#### Remediation", "", remediation] + seen = {remediation} + sources = finding.get("provenance", {}).get("sourceFindings", []) + originals = ( + [ + source + for source in sources + if isinstance(source, dict) and isinstance(source.get("finding"), dict) + ] + if isinstance(sources, list) + else [] + ) + for source in originals: + text = _text(source["finding"].get("remediation"), "") + if text and text not in seen: + seen.add(text) + lines.extend(["", f"Source {_text(source.get('id'), 'finding')}: {text}"]) + for field, label in ( + ("remediationTests", "Tests"), + ("preventiveControls", "Preventive controls"), + ): + values = list( + dict.fromkeys( + value + for original in [finding, *(source["finding"] for source in originals)] + for value in _strings(original.get(field)) + ) + ) + if values: + lines.extend(["", f"{label}:", *_bullets(values, "None recorded.")]) + return lines + + def _finding_section(number: int, finding: dict[str, Any]) -> list[str]: validation = finding.get("validation") if isinstance(finding.get("validation"), dict) else {} _, raw_root_cause = merged_root_cause(finding) @@ -593,8 +645,6 @@ def _finding_section(number: int, finding: dict[str, Any]) -> list[str]: severity.get("changeConditions"), "Additional runtime or deployment evidence could raise or lower this severity.", ) - remediation_tests = _strings(finding.get("remediationTests")) - preventive_controls = _strings(finding.get("preventiveControls")) attack_steps = _strings(attack_path.get("steps")) cwes = ", ".join(finding["taxonomy"]["cwe"]) or "none" title = _text(finding["title"], "Untitled finding") @@ -713,18 +763,7 @@ def _finding_section(number: int, finding: dict[str, Any]) -> list[str]: lines.extend( ["", f"{label} assessment:", *(f"- **{name}:** {value}" for name, value in details)] ) - lines.extend( - [ - "", - "#### Remediation", - "", - _text(finding["remediation"], "No canonical remediation was recorded."), - ] - ) - if remediation_tests: - lines.extend(["", "Tests:", *_bullets(remediation_tests, "No tests recorded.")]) - if preventive_controls: - lines.extend(["", "Preventive controls:", *_bullets(preventive_controls, "None recorded.")]) + lines.extend(_remediation_section(finding)) return lines @@ -746,8 +785,12 @@ def _linked_finding_section(number: int, finding: dict[str, Any], report_path: s f"| CWE | {_cell(cwes)} |", f"| Affected lines | {_cell(_locations(finding))} |", ] - for heading in ("Summary", "Validation", "Dataflow", "Reachability", "Severity", "Remediation"): + for heading in ("Summary", "Validation", "Dataflow", "Reachability", "Severity"): lines.extend(["", f"#### {heading}", "", f"See the {link}."]) + if finding.get("provenance", {}).get("sourceFindings"): + lines.extend(_remediation_section(finding)) + else: + lines.extend(["", "#### Remediation", "", f"See the {link}."]) return lines @@ -968,8 +1011,8 @@ def build_report_markdown( "No vulnerability conclusion can be drawn." if no_source_review else ( - "No findings were validated before the scan reached its cost limit. " - "Review the deferred candidates in Open Questions And Follow Up." + "No findings are included in this partial report. " + "Review the unresolved work in Open Questions And Follow Up." if budget_exhausted else "No reportable findings survived the canonical discovery, validation, " "and reportability gates." @@ -989,6 +1032,22 @@ def build_report_markdown( f"[Open the structural hardening portfolio]({hardening_portfolio_path})", ] ) + reviews = coverage.get("reviews", []) + if reviews: + lines.extend( + [ + "", + "## Source Review Coverage", + "", + "| Review | Attempt | Coverage |", + "| --- | --- | --- |", + ] + ) + for review in reviews: + if isinstance(review, dict): + lines.append( + f"| {_cell(review.get('workerId'))} | {_cell(str(review.get('attempt', 'unknown')))} | {_cell(review.get('completeness'))} |" + ) surfaces = coverage.get("surfaces", []) if surfaces: lines.extend( @@ -1026,6 +1085,7 @@ def build_report_markdown( questions.extend( { "question": item.get("reason", "Deferred review requires follow-up."), + "provenance": item.get("provenance", {}), "followUpPrompt": " ".join( ( f"Review deferred unit {item.get('id', 'unknown')} and close its stated proof gap.", @@ -1047,6 +1107,9 @@ def build_report_markdown( if not isinstance(question, dict): continue lines.append(f"- {_text(question.get('question'), 'Unspecified open question.')}") + source = _coverage_source(question) + if source: + lines.append(f" - {_text(source, '')}.") prompt = _text(question.get("followUpPrompt"), "") if prompt: lines.append(f" - Follow-up prompt: {prompt}") diff --git a/plugins/codex-security/scripts/workbench_cli.py b/plugins/codex-security/scripts/workbench_cli.py index ae1db666d3..d991391405 100644 --- a/plugins/codex-security/scripts/workbench_cli.py +++ b/plugins/codex-security/scripts/workbench_cli.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import json import sys from pathlib import Path @@ -31,7 +32,7 @@ def add_user_context(parser: argparse.ArgumentParser, *, required: bool = False) context.add_argument("--user-context-stdin", action="store_true") -def parse_args(description: str) -> argparse.Namespace: +def parse_args(description: str, *, execution_settings: bool = False) -> argparse.Namespace: parser = argparse.ArgumentParser(description=description) subparsers = parser.add_subparsers(dest="command", required=True) @@ -400,7 +401,16 @@ def parse_args(description: str) -> argparse.Namespace: parser.error("pass exactly one user-context transport") index = arguments.index("--user-context-stdin") arguments[index] = "--user-context=" + sys.stdin.buffer.read().decode("utf-8") - return parser.parse_args(arguments) + args = parser.parse_args(arguments) + if execution_settings: + args.require_execution_settings = True + if args.command == "begin-deep-scan": + # Private MCP creation input shares stdin with the user's unchanged context. + payload = json.load(sys.stdin) + args.execution_settings = payload["executionSettings"] + args.user_context = payload.get("userContext") + args.user_context_stdin = False + return args if __name__ == "__main__": diff --git a/plugins/codex-security/scripts/workbench_db.py b/plugins/codex-security/scripts/workbench_db.py index 4a28a4e3c1..5d6a03ce0a 100644 --- a/plugins/codex-security/scripts/workbench_db.py +++ b/plugins/codex-security/scripts/workbench_db.py @@ -55,9 +55,7 @@ _prepare_scan_finalization, _write_prepared_scan_finalization, finalize_scan, - finding_candidate_id, open_scan_local_file_descriptor, - write_scan_local_bytes, ) from finding_preview import bounded_finding_details from workbench import handoff @@ -1134,18 +1132,22 @@ def complete_budget_exhausted_scan( connection: sqlite3.Connection, args: argparse.Namespace ) -> dict[str, Any]: scan_id = require_uuid(args.scan_id, "scan-id") - cost_json = parse_scan_cost(args.cost_json) + cost_json = parse_scan_cost(args.cost_json, allow_lower_bound=True) if cost_json is None: raise SystemExit("Budget-exhausted scan completion requires the measured scan cost.") with scan_completion_lock(scan_id): scan = require_scan(connection, scan_id) if scan["status"] != "running" or scan["mode"] != "deep" or scan["recipe_json"] is None: raise SystemExit("Only a running CLI Deep Scan can complete after its cost limit.") + handoff.require_current_continuation( + scan, None, error_message="Scan completion is owned by another continuation." + ) recipe = json.loads(scan["recipe_json"], parse_constant=reject_non_finite_json) if not isinstance(recipe, dict) or recipe.get("mode") != "deep": raise SystemExit("Budget-exhausted scan completion requires a Deep Scan launch recipe.") cost = json.loads(cost_json) - measured = cost.get("cost", cost) + lower_bound = set(cost) == {"lowerBound"} + measured = cost["lowerBound"] if lower_bound else cost.get("cost", cost) limit = recipe.get("maxCostUsd") if ( not isinstance(limit, (int, float)) @@ -1154,11 +1156,13 @@ def complete_budget_exhausted_scan( or measured.get("estimatedUsd", 0) <= limit ): raise SystemExit("Deep Scan has not exceeded its configured cost limit.") - run = connection.execute( - "SELECT status, terminal_reason, manifest_path FROM deep_scan_runs WHERE scan_id = ?", - (scan_id,), - ).fetchone() - if ( + run = deep_scan.find_supported_deep_scan_run(connection, scan_id) + before_selection = ( + run is not None + and run["status"] == "running" + and run["workflow_version"] == "deep-security-scan/v2" + ) + if not before_selection and ( run is None or run["status"] != "succeeded" or run["terminal_reason"] not in {"saturated", "capped"} @@ -1171,16 +1175,77 @@ def complete_budget_exhausted_scan( scan_dir = require_canonical_scan_directory(Path(scan["scan_dir"])) candidates = ( [] - if run["manifest_path"] == str(scan_dir / "scan-manifest.json") + if before_selection or run["manifest_path"] == str(scan_dir / "scan-manifest.json") else budget_exhausted_candidates(scan, scan_dir) ) warning = optional_text(args.message, maximum=2400) if warning is None: + retained = ( + "saved work was retained" + if before_selection + else "completed discovery was preserved" + ) warning = ( f"Deep Scan reached its cost limit after an estimated " - f"${measured['estimatedUsd']:.6g}; completed discovery was preserved." + f"${measured['estimatedUsd']:.6g}; {retained}." + ) + selection = deep_scan.deep_scan_finalization_input(run) + if selection is not None: + try: + saved_results._require_selected_result(scan, selection) + if "publicationSha256" in selection: + prepared = _prepare_scan_finalization( + scan_dir, + expected_coverage_mode=expected_coverage_mode(scan), + completion_binding=workbench_completion_binding( + scan, now(), read_json_object(scan_dir / ARTIFACTS["manifest"]) + ), + ) + saved_results.require_selected_publication( + _WORKBENCH_DB_CONTEXT, connection, scan, prepared + ) + except ContractError as exc: + raise SystemExit(str(exc)) from exc + if before_selection: + deep_scan.prepare_budget_exhausted_deep_scan( + connection, + scan, + scan_dir, + warning, + lambda accepted, unmerged: budget_exhausted_draft( + scan, + scan_dir, + [], + warning, + before_selection=True, + accepted_result=accepted, + unmerged_workers=unmerged, + ), + ) + else: + budget_exhausted_draft( + scan, + scan_dir, + candidates, + warning, + unmerged_workers=deep_scan.budget_unmerged_workers(connection, scan_id), + ) + if ( + deep_scan.deep_scan_finalization_input( + deep_scan.require_deep_scan_run(connection, scan_id) + ) + is not None + ): + documents = tuple( + saved_results._read_scan_local_json(scan_dir, name, name) + for name in ("scan-manifest.json", "findings.json", "coverage.json") ) - budget_exhausted_draft(scan, scan_dir, candidates, warning) + try: + saved_results.record_selected_publication( + _WORKBENCH_DB_CONTEXT, connection, scan, documents + ) + except ContractError as exc: + raise SystemExit(str(exc)) from exc warnings = json.loads(scan["completion_warnings_json"]) if warning not in warnings: connection.execute( @@ -1188,7 +1253,8 @@ def complete_budget_exhausted_scan( (json.dumps([*warnings, warning]), scan_id), ) connection.commit() - return complete_scan_locked(connection, scan_id, None, cost_json) + # A priced subtotal proves the stop but is not the saved total estimate. + return complete_scan_locked(connection, scan_id, None, None if lower_bound else cost_json) def budget_exhausted_candidates(scan: sqlite3.Row, scan_dir: Path) -> list[dict[str, Any]]: @@ -1268,148 +1334,50 @@ def budget_exhausted_draft( scan_dir: Path, candidates: list[dict[str, Any]], warning: str, + *, + before_selection: bool = False, + accepted_result: dict[str, Any] | None = None, + unmerged_workers: list[sqlite3.Row] = (), ) -> None: - documents: dict[str, dict[str, Any]] = {} - for name in ("scan-manifest.json", "findings.json", "coverage.json"): - path = artifact_path(scan_dir, name, required=False) - if path is not None: - documents[name] = read_json_object(path) - if documents and len(documents) != 3: - raise SystemExit("Budget-exhausted scan contains an incomplete canonical scan draft.") - - if documents: - manifest = documents["scan-manifest.json"] - findings = documents["findings.json"] - coverage = documents["coverage.json"] - if not isinstance(manifest.get("scan"), dict) or not isinstance( - findings.get("findings"), list - ): - raise SystemExit("Budget-exhausted scan contains an invalid canonical scan draft.") - for key in ("surfaces", "explicitExclusions", "deferred"): - if not isinstance(coverage.get(key), list): - raise SystemExit("Budget-exhausted scan contains invalid canonical coverage.") - if manifest["scan"].get("sealedAt") is not None or manifest["scan"].get("artifacts"): - raise SystemExit("Budget-exhausted scan cannot replace an already sealed scan draft.") - else: - contract = scan_contract(scan) - target_contract = contract["target"] - target: dict[str, Any] = { - "kind": target_contract["allowedKinds"][0], - "targetId": target_contract["targetId"], - "displayName": target_contract["displayName"], - } - if scan["target_revision"] != "unversioned": - target["revision"] = scan["target_revision"] - if "requiredSnapshotDigest" in target_contract: - target["snapshotDigest"] = target_contract["requiredSnapshotDigest"] - manifest = { - "scan": { - "target": target, - "scope": {"limitations": [warning], "validationMode": "incomplete"}, - } - } - findings = {"findings": []} - coverage = { - "completeness": "partial", - "inventoryStrategy": ( - "scoped_path" if expected_coverage_mode(scan) == "scoped_path" else "repository" - ), - "surfaces": [], - "explicitExclusions": [], - "deferred": [], - } + saved_results.budget_exhausted_draft( + _WORKBENCH_DB_CONTEXT, + scan, + scan_dir, + candidates, + warning, + scan_contract(scan), + before_selection=before_selection, + accepted_result=accepted_result, + unmerged_workers=unmerged_workers, + ) - findings_by_candidate = { - candidate_id - for finding in findings["findings"] - if isinstance(finding, dict) - and isinstance(candidate_id := finding_candidate_id(finding), str) - } - existing_deferred = { - item.get("candidateId", item.get("id")) - for item in coverage["deferred"] - if isinstance(item, dict) and isinstance(item.get("candidateId", item.get("id")), str) - } - existing_surfaces = { - item.get("id") - for item in coverage["surfaces"] - if isinstance(item, dict) and isinstance(item.get("id"), str) + +def completed_scan_context( + connection: sqlite3.Connection, scan_id: str, coverage: dict[str, Any] +) -> dict[str, Any]: + context = scan_context(connection, scan_id) + context["coverageSummary"] = { + "completeness": coverage["completeness"], + "surfaceCount": len(coverage["surfaces"]), + "deferredCount": len(coverage.get("deferred", [])), + "explicitExclusionCount": len(coverage.get("explicitExclusions", [])), } - for candidate in candidates: - candidate_id = candidate["candidate_id"] - if candidate_id in findings_by_candidate or candidate_id in existing_deferred: - continue - paths = list(dict.fromkeys(location["path"] for location in candidate["locations"])) - surface_id = f"candidate-{candidate_id}" - validation = candidate.get("validation") - validation = validation.get("disposition") if isinstance(validation, dict) else None - attack = candidate.get("attack_path") - attack = attack.get("decision") if isinstance(attack, dict) else None - disposition = ( - "needs_follow_up" - if validation == "deferred" or attack == "deferred" - else "not_applicable" - if validation == "not_applicable" - else "rejected" - if validation == "suppressed" or attack == "ignore" - else "needs_follow_up" - ) - if surface_id not in existing_surfaces: - coverage["surfaces"].append( - { - "id": surface_id, - "label": candidate["summary"], - "disposition": disposition, - "notes": candidate["evidence"], - "receiptRefs": [], - } - ) - existing_surfaces.add(surface_id) - if disposition != "needs_follow_up": - continue - coverage["deferred"].append( - { - "id": candidate_id, - "candidateId": candidate_id, - "reason": ( - "Validation was deferred because the scan reached its cost limit: " - f"{candidate['summary']}. Evidence: {candidate['evidence']}" - ), - "paths": paths, - "surfaceIds": [surface_id], - } - ) - if not any( - isinstance(item, dict) - and isinstance(reason := item.get("reason"), str) - and ( - reason == "Validation was deferred because the scan reached its cost limit." - or reason.startswith( - "Validation was deferred because the scan reached its cost limit: " - ) - ) - for item in coverage["deferred"] - ): - coverage["deferred"].append( - { - "id": "scan-cost-limit", - "reason": "Validation was deferred because the scan reached its cost limit.", - } - ) - coverage["completeness"] = "partial" - for name, payload in ( - ("findings.json", findings), - ("coverage.json", coverage), - ("scan-manifest.json", manifest), - ): - try: - write_scan_local_bytes( - scan_dir, - name, - (json.dumps(payload, allow_nan=False, indent=2, sort_keys=True) + "\n").encode(), - ) - except (ContractError, OSError, TypeError, ValueError) as exc: - raise SystemExit(f"Budget-exhausted scan draft could not be saved: {exc}") from exc + return context + + +def validate_completed_scan_artifacts(scan: sqlite3.Row) -> None: + scan_dir = require_canonical_scan_directory(Path(scan["scan_dir"])) + require_recorded_manifest_digest(scan, scan_dir) + verify_manifest_binding(scan, read_json_object(scan_dir / ARTIFACTS["manifest"])) + try: + manifest = _prepare_scan_finalization( + scan_dir, + expected_coverage_mode=expected_coverage_mode(scan), + )[2] + except ContractError as exc: + raise SystemExit(str(exc)) from exc + verify_manifest_binding(scan, manifest) + published_manifest_digest(scan_dir, manifest) def complete_scan_locked( @@ -1423,6 +1391,7 @@ def complete_scan_locked( ) -> dict[str, Any]: scan = require_scan(connection, scan_id) if scan["status"] == "complete": + deep_scan.find_supported_deep_scan_run(connection, scan_id) return saved_results.refresh_completed_scan( _WORKBENCH_DB_CONTEXT, connection, scan, cost_json ) @@ -1520,14 +1489,19 @@ def add_warning() -> None: completion_warnings=warnings if scan["mode"] != "deep" else None, draft_documents=draft_documents, ) + saved_results.require_selected_publication( + _WORKBENCH_DB_CONTEXT, connection, scan, prepared + ) add_warning() wrote = True - manifest, findings, _ = _write_prepared_scan_finalization( + manifest, findings, coverage = _write_prepared_scan_finalization( prepared, projection_warnings=warnings ) except ContractError as exc: - if wrote or ( + # Replay a validated Deep aggregate after an output write fails. + if (wrote and scan["mode"] != "deep") or ( scan["mode"] == "deep" + and not wrote and not already_sealed and not isinstance(exc, RecoverableContractError) ): @@ -1567,7 +1541,7 @@ def add_warning() -> None: scan = require_scan(connection, scan["id"]) if scan["status"] == "complete": connection.commit() - return scan_context(connection, scan["id"]) + return completed_scan_context(connection, scan["id"], coverage) if scan["status"] != "running": raise SystemExit("Only a running scan can be completed.") deep_scan.require_deep_scan_ready_for_parent_completion(connection, scan) @@ -1614,7 +1588,7 @@ def add_warning() -> None: ) if updated.rowcount != 1: raise SystemExit("Only a running scan can be completed.") - context = scan_context(connection, scan["id"]) + context = completed_scan_context(connection, scan["id"], coverage) context["targetWarnings"] = target_warnings return context @@ -2417,7 +2391,7 @@ def require_reviewed_patch_applied( checkout = checkout_root copy_directory_excluding(target, checkout, excluded) else: - copy_git_worktree_files(target, checkout_root, excluded) + checkout = copy_git_worktree_files(target, checkout_root, excluded) arguments = ["apply", "--reverse", "--whitespace=nowarn"] if unversioned: arguments.append("--no-index") @@ -2774,8 +2748,7 @@ def scan_result( **scan_usage.stored_scan_cost_fields(scan["cost_json"]), "contract": scan_contract(scan), "continuationThreadId": scan["continuation_thread_id"], - "threadIds": scan_usage._scan_root_thread_ids(connection, scan, None), - "executionThreadIds": scan_usage._scan_execution_thread_ids(connection, scan), + **scan_usage.scan_execution_fields(connection, scan), "failureMessage": scan["failure_message"], "findings": [ finding_result(connection, scan, row, related=relations.get(row["id"], [])) @@ -3288,10 +3261,10 @@ def read_json_object(path: Path) -> dict[str, Any]: _WORKBENCH_DB_CONTEXT = SimpleNamespace(**globals()) -def main() -> None: +def main(*, select_finalization: bool = False, with_execution_settings: bool = False) -> None: # Workbench callers send UTF-8 even when Windows uses a legacy code page. sys.stdin.reconfigure(encoding="utf-8") - args = parse_args(__doc__) + args = parse_args(__doc__, execution_settings=with_execution_settings) command_context = SimpleNamespace( **globals(), preserve_stopped_results=preserve_stopped_results_after_transition, @@ -3349,6 +3322,13 @@ def main() -> None: result = _start_prompt_driven_scan(connection, args, headless_standard=True) elif args.command == "begin-deep-scan": result = deep_scan.begin_deep_scan(connection, args) + if ( + result["deepScan"]["status"] == "succeeded" + and result["deepScan"]["finalizationInput"] is not None + ): + scan = require_scan(connection, result["deepScan"]["scanId"]) + if scan["status"] == "complete": + validate_completed_scan_artifacts(scan) elif args.command == "get-deep-scan": result = deep_scan.get_deep_scan(connection, args) elif args.command == "claim-deep-scan-coordinator": @@ -3360,7 +3340,7 @@ def main() -> None: elif args.command == "commit-deep-scan-dedup": result = deep_scan.commit_deep_scan_dedup(connection, args) elif args.command == "finish-deep-scan": - result = deep_scan.finish_deep_scan(connection, args) + result = deep_scan.finish_deep_scan(connection, args, select_finalization) elif args.command == "fail-deep-scan": result = deep_scan.fail_deep_scan(connection, args) elif args.command == "record-deep-scan-publication-failure": @@ -3505,6 +3485,8 @@ def main() -> None: result = list_stored_findings(connection, limit=args.limit, offset=args.offset) else: raise SystemExit(f"Unknown command: {args.command}") + if with_execution_settings: + deep_scan.include_execution_settings(connection, result) print(json.dumps(result, allow_nan=False, sort_keys=True)) diff --git a/plugins/codex-security/scripts/workbench_saved_results.py b/plugins/codex-security/scripts/workbench_saved_results.py index 9de4b04a2a..bf76a0a4dc 100644 --- a/plugins/codex-security/scripts/workbench_saved_results.py +++ b/plugins/codex-security/scripts/workbench_saved_results.py @@ -20,6 +20,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from finalize_scan_contract import ( ContractError, + RecoverableContractError, _finding_strength, _populate_unsealed_artifact_envelope, _populate_unsealed_manifest_envelope, @@ -96,7 +97,7 @@ def refresh_completed_scan( db.require_recorded_manifest_digest(scan, scan_dir) db.verify_manifest_binding(scan, db.read_json_object(scan_dir / db.ARTIFACTS["manifest"])) try: - manifest, _, _ = finalize_scan( + manifest, _, coverage = finalize_scan( scan_dir, expected_coverage_mode=db.expected_coverage_mode(scan), projection_warnings=warnings, @@ -114,7 +115,22 @@ def refresh_completed_scan( "UPDATE scans SET completion_warnings_json = ? WHERE id = ?", (json.dumps(warnings), scan["id"]), ) - return db.scan_context(connection, scan["id"]) + return db.completed_scan_context(connection, scan["id"], coverage) + + +def validate_sealed_budget_draft( + db: Any, scan: sqlite3.Row, scan_dir: Path, manifest: dict[str, Any] +) -> None: + # The seal can reach disk before parent completion commits. Validate + # it without changing bytes; the existing finalizer commits replay. + try: + _prepare_scan_finalization( + scan_dir, + expected_coverage_mode=db.expected_coverage_mode(scan), + completion_binding=db.workbench_completion_binding(scan, scan["started_at"], manifest), + ) + except ContractError as exc: + raise SystemExit(str(exc)) from exc def _encoded(value: Any) -> bytes: @@ -141,6 +157,35 @@ def _children(scan_dir: Path, relative: str) -> list[str]: return sorted(child.name for child in cursor.iterdir()) +def _saved_workers(connection: Any, scan_id: str) -> list[dict[str, Any]]: + rows = connection.execute( + "SELECT worker.*, attempt.accepted_result_path FROM deep_scan_workers AS worker " + "LEFT JOIN deep_scan_attempts AS attempt " + "ON attempt.worker_id = worker.id AND attempt.attempt = worker.attempt " + "WHERE worker.scan_id = ? ORDER BY worker.created_at, worker.id", + (scan_id,), + ).fetchall() + return [ + { + **dict(row), + "result_manifest_path": row["accepted_result_path"] or row["result_manifest_path"], + } + for row in rows + ] + + +def _accepted_source_digests(connection: Any, scan_id: str) -> dict[str, str]: + return { + row["accepted_result_path"]: row["accepted_result_sha256"] + for row in connection.execute( + "SELECT accepted_result_path, accepted_result_sha256 FROM deep_scan_attempts " + "WHERE scan_id = ? AND accepted_result_path IS NOT NULL " + "AND accepted_result_sha256 IS NOT NULL", + (scan_id,), + ) + } + + def _latest_successful_reducer(workers: list[Any]) -> Any | None: return max( ( @@ -230,15 +275,22 @@ def _capture_saved_source( kind: str | None = None, snapshot_head: bool = True, write: bool = True, + accepted_source_digests: dict[str, str] | None = None, ) -> dict[str, tuple[str, int]]: if not snapshot_head or Path(relative).name != "checkpoint-head.json": - _, digest, observed = _read_saved_result(scan_dir, relative, scan_id, kind=kind) + _, digest, observed = _read_saved_result( + scan_dir, relative, scan_id, kind=kind, accepted_source_digests=accepted_source_digests + ) return {relative: (digest, observed)} - head, _, observed = _read_saved_result(scan_dir, relative, scan_id) + head, _, observed = _read_saved_result( + scan_dir, relative, scan_id, accepted_source_digests=accepted_source_digests + ) observation = {"checkpoint": head["checkpoint"], "observedAtNs": str(observed)} directory = Path(relative).parent selected = (directory / "checkpoints" / observation["checkpoint"]).as_posix() - _, selected_digest, selected_time = _read_saved_result(scan_dir, selected, scan_id) + _, selected_digest, selected_time = _read_saved_result( + scan_dir, selected, scan_id, accepted_source_digests=accepted_source_digests + ) digest = _digest(observation) snapshot = (directory / "checkpoint-heads" / f"{digest}.json").as_posix() # Capture the selected file even if the worker created it after directory enumeration. @@ -258,17 +310,30 @@ def _is_source_order_snapshot(relative: str) -> bool: def _read_saved_result( - scan_dir: Path, relative: str, scan_id: str, *, kind: str | None = None + scan_dir: Path, + relative: str, + scan_id: str, + *, + kind: str | None = None, + accepted_source_digests: dict[str, str] | None = None, ) -> tuple[dict[str, Any], str, int]: - draft, _, metadata = _read_scan_local_json_with_metadata( + draft, contents, metadata = _read_scan_local_json_with_metadata( scan_dir, relative, "Saved scan checkpoint" ) + expected = (accepted_source_digests or {}).get(str(scan_dir / relative)) + if expected is not None and hashlib.sha256(contents).hexdigest() != expected: + raise ContractError("checkpoint changed after acceptance") directory = _checkpoint_head_directory(relative) if directory is not None: checkpoint = draft.get("checkpoint") if not isinstance(checkpoint, str) or not re.fullmatch(r"[0-9a-f]{64}\.json", checkpoint): raise ContractError("checkpoint head does not name a saved checkpoint") - _read_saved_result(scan_dir, (directory / "checkpoints" / checkpoint).as_posix(), scan_id) + _read_saved_result( + scan_dir, + (directory / "checkpoints" / checkpoint).as_posix(), + scan_id, + accepted_source_digests=accepted_source_digests, + ) if Path(relative).name == "checkpoint-head.json": return draft, _digest([draft, metadata.st_mtime_ns]), metadata.st_mtime_ns observed = draft.get("observedAtNs") @@ -279,7 +344,12 @@ def _read_saved_result( raise ContractError("checkpoint belongs to a different scan") if not _is_source_order_snapshot(relative) and ( not isinstance(draft.get("findings"), list) - or not isinstance(draft.get("coverage", {} if kind == "dedup" else None), dict) + or not isinstance( + draft.get("sourceCoverage", draft.get("coverage", {})) + if kind == "dedup" + else draft.get("coverage"), + dict, + ) ): raise ContractError("checkpoint has no semantic findings or coverage") return draft, _digest(draft), metadata.st_mtime_ns @@ -352,6 +422,58 @@ def _parent_scan_draft( return parent +def _worker_checkpoint_head( + scan_dir: Path, + directory: str, + scan_id: str, + accepted_source_digests: dict[str, str] | None = None, +) -> str | None: + relative = f"{directory}/checkpoint-head.json" + try: + (scan_dir / relative).lstat() + except FileNotFoundError: + return None + head = _read_scan_local_json(scan_dir, relative, "Saved worker checkpoint head") + name = head.get("checkpoint") + if not isinstance(name, str) or not re.fullmatch(r"[0-9a-f]{64}\.json", name): + raise ContractError("Saved worker checkpoint head is invalid.") + checkpoint = f"{directory}/checkpoints/{name}" + # A committed head precedes replacement of result.json. Do not fall back to + # that older result if the selected checkpoint cannot be read. + _read_saved_result( + scan_dir, checkpoint, scan_id, accepted_source_digests=accepted_source_digests + ) + return checkpoint + + +def _worker_checkpoint_heads( + scan_dir: Path, + workers: list[Any], + scan_id: str, + accepted_source_digests: dict[str, str] | None = None, +) -> dict[str, str]: + heads: dict[str, str] = {} + for worker in workers: + if worker["kind"] != "discovery": + continue + try: + output = Path(worker["artifact_dir"]).relative_to(scan_dir) + except (TypeError, ValueError): + continue + attempts = (output.parent if output.name == "output" else output) / "attempts" + directories = [output] + [ + attempts / name + for name in _children(scan_dir, attempts.as_posix()) + if re.fullmatch(r"attempt-\d+", name) + ] + for directory in directories: + relative = directory.as_posix() + head = _worker_checkpoint_head(scan_dir, relative, scan_id, accepted_source_digests) + if head is not None: + heads[relative] = head + return heads + + def _read_saved_parent_result( scan_dir: Path, scan_id: str ) -> tuple[dict[str, Any], dict[str, Any]]: @@ -399,18 +521,21 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool: try: scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) manifest_path = db.artifact_path(scan_dir, db.ARTIFACTS["manifest"], required=False) - workers = connection.execute( - "SELECT id, kind, status, completed_at, artifact_dir, result_manifest_path " - "FROM deep_scan_workers WHERE scan_id = ?", - (scan["id"],), - ).fetchall() + workers = _saved_workers(connection, scan["id"]) + accepted_digests = _accepted_source_digests(connection, scan["id"]) paths = dict(_saved_result_paths(scan_dir, workers)) frozen_sources = scan["retained_source_digests_json"] def has_saved_source() -> bool: for path in paths: try: - _read_saved_result(scan_dir, path, scan["id"], kind=paths[path]) + _read_saved_result( + scan_dir, + path, + scan["id"], + kind=paths[path], + accepted_source_digests=accepted_digests, + ) return True except (ContractError, OSError, ValueError): continue @@ -439,6 +564,10 @@ def has_saved_source() -> bool: manifest_scan.get("preservedSources", {}), "Published scan source digests are malformed.", ) + if _worker_checkpoint_heads( + scan_dir, workers, scan["id"], accepted_digests + ) != manifest_scan.get("preservedCheckpointHeads", {}): + return True current_sources = dict(published_sources) paths.update({path: None for path in published_sources if _is_source_order_snapshot(path)}) for path in paths: @@ -448,6 +577,7 @@ def has_saved_source() -> bool: path, scan["id"], kind=paths[path], + accepted_source_digests=accepted_digests, snapshot_head=path not in published_sources, write=False, ) @@ -459,7 +589,9 @@ def has_saved_source() -> bool: return False -def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[str, str], bool]: +def _recovery_source_digests( + db: Any, connection: Any, scan: Any +) -> tuple[dict[str, str], bool, dict[str, str]]: scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) frozen_sources: dict[str, str] | None = None include_parent = True @@ -498,18 +630,20 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[ else: include_parent = True - workers = connection.execute( - "SELECT id, kind, status, completed_at, artifact_dir, result_manifest_path " - "FROM deep_scan_workers WHERE scan_id = ?", - (scan["id"],), - ).fetchall() + workers = _saved_workers(connection, scan["id"]) + accepted_digests = _accepted_source_digests(connection, scan["id"]) + checkpoint_heads = _worker_checkpoint_heads(scan_dir, workers, scan["id"], accepted_digests) paths = dict(_saved_result_paths(scan_dir, workers)) recovery_sources = dict(frozen_sources or {}) source_times = _frozen_source_times(scan_dir, scan["id"], recovery_sources) for relative, expected_digest in recovery_sources.items(): try: _, digest, observed = _read_saved_result( - scan_dir, relative, scan["id"], kind=paths.get(relative) + scan_dir, + relative, + scan["id"], + kind=paths.get(relative), + accepted_source_digests=accepted_digests, ) except (ContractError, OSError, ValueError) as exc: raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc @@ -520,7 +654,13 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[ for relative in paths.keys() - recovery_sources.keys(): try: - captured = _capture_saved_source(scan_dir, relative, scan["id"], kind=paths[relative]) + captured = _capture_saved_source( + scan_dir, + relative, + scan["id"], + kind=paths[relative], + accepted_source_digests=accepted_digests, + ) except (ContractError, OSError, ValueError): continue for path, (digest, observed) in captured.items(): @@ -529,7 +669,7 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[ recovery_sources[path] = digest source_times.setdefault(path, observed) _freeze_source_times(scan_dir, scan["id"], recovery_sources, source_times) - return recovery_sources, include_parent + return recovery_sources, include_parent, checkpoint_heads def scan_results_recovery_needed(db: Any, connection: Any, scan: Any) -> bool: @@ -630,6 +770,30 @@ def _ensure_finding_identity(finding: Any, *, candidate_only: bool = False) -> N finding["identity"] = {"anchor": anchor} +def _ensure_finding_identities(findings: list[dict[str, Any]]) -> None: + def key(finding: dict[str, Any]) -> bytes: + identity = finding["identity"] + return _encoded( + [finding.get("ruleId"), identity.get("anchor"), identity.get("instance", "")] + ) + + # Reserve authored identities even when they follow a generated sibling. + reserved = { + key(finding) + for finding in findings + if isinstance(finding, dict) and isinstance(finding.get("identity"), dict) + } + for finding in findings: + if not isinstance(finding, dict) or "identity" in finding: + continue + _ensure_finding_identity(finding) + suffix = 2 + while key(finding) in reserved: + finding["identity"]["instance"] = f"saved-{suffix}" + suffix += 1 + reserved.add(key(finding)) + + def _retained_findings(finding: dict[str, Any]) -> Iterator[dict[str, Any]]: """Yield canonical and historical findings without trusting candidate IDs.""" pending = [finding] @@ -917,7 +1081,9 @@ def merge_saved_results( stopped: bool, reason: str, frozen_source_digests: dict[str, str] | None = None, + checkpoint_heads: dict[str, str] | None = None, allow_frozen_legacy_parent: bool = False, + accepted_source_digests: dict[str, str] | None = None, frozen_model_source: str | None = None, selected_model_source: list[str] | None = None, ) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]] | None: @@ -927,6 +1093,10 @@ def merge_saved_results( source_times = _frozen_source_times(scan_dir, scan_id, frozen_source_digests or {}) except (ContractError, OSError, ValueError) as exc: raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc + if checkpoint_heads is None: + checkpoint_heads = _worker_checkpoint_heads( + scan_dir, workers, scan_id, accepted_source_digests + ) parent: dict[str, Any] | None = None parent_manifest: dict[str, Any] | None = None parent_is_canonical = False @@ -1019,6 +1189,7 @@ def merge_saved_results( reducer_paths.add(latest_reducer) except ValueError: warnings.append("Skipped a reducer result outside the scan directory.") + accepted_reducer = latest_reducer def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None: if worker_id is not None: @@ -1097,7 +1268,11 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None for relative, worker_id in paths.items(): try: draft, digest, observed = _read_saved_result( - scan_dir, relative, scan_id, kind="dedup" if relative in reducer_paths else None + scan_dir, + relative, + scan_id, + kind="dedup" if relative in reducer_paths else None, + accepted_source_digests=accepted_source_digests, ) if frozen_source_digests is not None and frozen_source_digests[relative] != digest: raise ContractError("checkpoint changed after the scan stopped") @@ -1107,8 +1282,13 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None if _checkpoint_head_directory(relative) is not None: saved_heads[relative] = draft["checkpoint"] continue - # Recovery expects coverage, but reducer results only contain findings - # and context. Add an empty value after hashing the original result. + # The host supplies reducer coverage separately from model output. + # Preserve the digest of the original accepted document. + if relative in reducer_paths: + draft = { + **draft, + "coverage": draft.get("sourceCoverage", draft.get("coverage", {})), + } sources.append((relative, {"coverage": {}, **draft}, worker_id)) except (ContractError, OSError, ValueError) as exc: if (scan_dir / relative).exists(): @@ -1149,6 +1329,15 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None source_order.update(selected_observations) drafts_by_path = {relative: draft for relative, draft, _ in sources} + if "sourceCoverage" not in drafts_by_path.get(accepted_reducer, {}): + accepted_reducer = None + accepted_coverage = drafts_by_path.get(accepted_reducer, {}).get("coverage", {}) + reviewed_attempts = { + (review.get("workerId"), review.get("attempt")) + for review in accepted_coverage.get("reviews", []) + if isinstance(review, dict) + } + workers_by_id = {worker["id"]: worker for worker in workers} latest_reducer_key = ( (reducer["completed_at"] or "", reducer["id"], int(reducer["attempt"] or 0)) if reducer is not None and latest_reducer in drafts_by_path @@ -1237,6 +1426,7 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None and parent_manifest["scan"].get("sealedAt") and parent_manifest["scan"].get("status") == binding["status"] and parent_manifest["scan"].get("preservedSources") == source_digests + and parent_manifest["scan"].get("preservedCheckpointHeads", {}) == checkpoint_heads and all(warning in initial_warnings for warning in warnings) ): return None @@ -1269,6 +1459,7 @@ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None for key in ("sealedAt", "artifacts"): manifest["scan"].pop(key, None) manifest["scan"]["preservedSources"] = source_digests + manifest["scan"]["preservedCheckpointHeads"] = checkpoint_heads # Completion follows the selected parent, including legacy terminal drafts # that omit the optional marker, rather than an older canonical manifest. if parent is not None: @@ -1347,6 +1538,18 @@ def valid_finding(value: Any) -> bool: ) return bool(document["findings"]) + all_sources = ([("parent", parent, None)] if parent else []) + sources + + def coverage_candidate( + owner: str | None, item: dict[str, Any], identity: Any = None + ) -> tuple[str | None, Any]: + provenance = item.get("provenance") + if owner is None and isinstance(provenance, dict): + return provenance.get("workerId"), provenance.get( + "candidateId", identity if identity is not None else item.get("candidateId") + ) + return owner, identity if identity is not None else item.get("candidateId") + source_order["parent"] = (0, parent_modified) deferred_rows = { relative: _deferred_rows(draft["coverage"]) for relative, draft, _ in all_sources @@ -1373,7 +1576,10 @@ def valid_finding(value: Any) -> bool: ambiguous_deferred.add((owner, identity)) by_id[identity] = row current_drafts = ([("parent", parent, None)] if parent else []) + [ - source for source in sources if source[0] in current_results | selected_observations.keys() + source + for source in sources + if source[0] in current_results | selected_observations.keys() + or source[0] == accepted_reducer ] # Generic closures belong to one logical scan or worker, just like candidates. # Keep them when recovering a terminal checkpoint without its canonical write. @@ -1493,7 +1699,8 @@ def valid_finding(value: Any) -> bool: and isinstance(item.get("candidateId"), str) and item.get("disposition") in {"rejected", "not_applicable"} ): - outcomes.append((relative, owner, item["candidateId"], item["disposition"])) + candidate_owner, candidate_id = coverage_candidate(owner, item) + outcomes.append((relative, candidate_owner, candidate_id, item["disposition"])) ordered_candidates.update( (owner, candidate_id) for relative, owner, candidate_id, _ in outcomes @@ -1893,10 +2100,27 @@ def valid_finding(value: Any) -> bool: continue finding_positions[key] = len(findings) findings.append(finding) - if superseded and not selected_candidates and not retain_pending: + worker = workers_by_id.get(worker_id) + reviewed = ( + worker is not None + and {"status", "merge_state", "attempt"} <= worker.keys() + and worker["status"] == "succeeded" + and worker["merge_state"] == "merged" + and (worker_id, worker["attempt"]) in reviewed_attempts + ) + if reviewed or ( + superseded + and relative != accepted_reducer + and not selected_candidates + and not retain_pending + ): continue - for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"): - if superseded and field not in {"surfaces", "explicitExclusions", "deferred"}: + for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions", "reviews"): + if ( + superseded + and relative != accepted_reducer + and field not in {"surfaces", "explicitExclusions", "deferred"} + ): continue items = draft["coverage"].get(field, []) if field == "deferred": @@ -1921,15 +2145,19 @@ def valid_finding(value: Any) -> bool: ): continue # A selected outcome must retain its evidence even if its result write failed. - if superseded and not ( - isinstance(item, dict) - and ( - (field == "deferred" and retain_pending) - or ( - isinstance(item.get("candidateId"), str) - and item["candidateId"] in selected_candidates - and item.get("disposition") - == resolved.get((worker_id, item["candidateId"])) + if ( + superseded + and relative != accepted_reducer + and not ( + isinstance(item, dict) + and ( + (field == "deferred" and retain_pending) + or ( + isinstance(item.get("candidateId"), str) + and item["candidateId"] in selected_candidates + and item.get("disposition") + == resolved.get((worker_id, item["candidateId"])) + ) ) ) ): @@ -1972,7 +2200,7 @@ def valid_finding(value: Any) -> bool: else item.get("candidateId"), str, ) - and (worker_id, identity) in resolved + and coverage_candidate(worker_id, item, identity) in resolved and (field == "deferred" or item.get("disposition") == "needs_follow_up") ): continue @@ -2033,6 +2261,306 @@ def valid_finding(value: Any) -> bool: return manifest, {"findings": findings}, coverage +def budget_exhausted_draft( + db: Any, + scan: sqlite3.Row, + scan_dir: Path, + candidates: list[dict[str, Any]], + warning: str, + contract: dict[str, Any], + *, + before_selection: bool = False, + accepted_result: dict[str, Any] | None = None, + unmerged_workers: list[sqlite3.Row] = (), +) -> None: + documents: dict[str, dict[str, Any]] = {} + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + path = db.artifact_path(scan_dir, name, required=False) + if path is not None: + documents[name] = db.read_json_object(path) + interrupted_documents = {} + if documents and len(documents) != 3: + if not before_selection or set(documents) not in ( + {"findings.json"}, + {"findings.json", "coverage.json"}, + ): + raise SystemExit("Budget-exhausted scan contains an incomplete canonical scan draft.") + interrupted_documents = documents + documents = {} + + if documents: + manifest = documents["scan-manifest.json"] + findings = documents["findings.json"] + coverage = documents["coverage.json"] + if not isinstance(manifest.get("scan"), dict) or not isinstance( + findings.get("findings"), list + ): + raise SystemExit("Budget-exhausted scan contains an invalid canonical scan draft.") + for key in ("surfaces", "explicitExclusions", "deferred"): + if not isinstance(coverage.get(key), list): + raise SystemExit("Budget-exhausted scan contains invalid canonical coverage.") + if manifest["scan"].get("sealedAt") is not None or manifest["scan"].get("artifacts"): + if before_selection: + raise SystemExit( + "Budget-exhausted scan cannot replace an already sealed scan draft." + ) + # The seal can reach disk before parent completion commits. Validate + # it without changing bytes; the existing finalizer commits replay. + try: + _prepare_scan_finalization( + scan_dir, + expected_coverage_mode=db.expected_coverage_mode(scan), + completion_binding=db.workbench_completion_binding( + scan, scan["started_at"], manifest + ), + ) + except ContractError as exc: + raise SystemExit(str(exc)) from exc + return + else: + target_contract = contract["target"] + target: dict[str, Any] = { + "kind": target_contract["allowedKinds"][0], + "targetId": target_contract["targetId"], + "displayName": target_contract["displayName"], + } + if scan["target_revision"] != "unversioned": + target["revision"] = scan["target_revision"] + if "requiredSnapshotDigest" in target_contract: + target["snapshotDigest"] = target_contract["requiredSnapshotDigest"] + manifest = { + "scan": { + "target": target, + "scope": {"limitations": [warning], "validationMode": "incomplete"}, + } + } + findings = {"findings": []} + coverage = { + "completeness": "partial", + "inventoryStrategy": ( + "scoped_path" if db.expected_coverage_mode(scan) == "scoped_path" else "repository" + ), + "surfaces": [], + "explicitExclusions": [], + "deferred": [], + } + + if before_selection: + # Unmerged discoveries remain in their accepted artifacts. Only the + # committed reducer may contribute findings to this partial result. + findings = { + "findings": copy.deepcopy(accepted_result["findings"]) if accepted_result else [] + } + _ensure_finding_identities(findings["findings"]) + if accepted_result is not None: + coverage = accepted_result["sourceCoverage"] + if "threatModel" in accepted_result: + manifest["scan"]["threatModel"] = accepted_result["threatModel"] + if "scope" in accepted_result: + manifest["scan"]["scope"].update(copy.deepcopy(accepted_result["scope"])) + limitations = manifest["scan"]["scope"].setdefault("limitations", []) + if warning not in limitations: + limitations.append(warning) + + for worker in unmerged_workers: + retain_unmerged_budget_coverage(scan, scan_dir, coverage, worker) + + findings_by_candidate = { + candidate_id + for finding in findings["findings"] + if isinstance(finding, dict) + and isinstance(candidate_id := finding_candidate_id(finding), str) + } + existing_deferred = { + item.get("candidateId", item.get("id")) + for item in coverage["deferred"] + if isinstance(item, dict) and isinstance(item.get("candidateId", item.get("id")), str) + } + existing_surfaces = { + item.get("id") + for item in coverage["surfaces"] + if isinstance(item, dict) and isinstance(item.get("id"), str) + } + for candidate in candidates: + candidate_id = candidate["candidate_id"] + if candidate_id in findings_by_candidate or candidate_id in existing_deferred: + continue + paths = list(dict.fromkeys(location["path"] for location in candidate["locations"])) + surface_id = f"candidate-{candidate_id}" + validation = candidate.get("validation") + validation = validation.get("disposition") if isinstance(validation, dict) else None + attack = candidate.get("attack_path") + attack = attack.get("decision") if isinstance(attack, dict) else None + disposition = ( + "needs_follow_up" + if validation == "deferred" or attack == "deferred" + else "not_applicable" + if validation == "not_applicable" + else "rejected" + if validation == "suppressed" or attack == "ignore" + else "needs_follow_up" + ) + if surface_id not in existing_surfaces: + coverage["surfaces"].append( + { + "id": surface_id, + "label": candidate["summary"], + "disposition": disposition, + "notes": candidate["evidence"], + "receiptRefs": [], + } + ) + existing_surfaces.add(surface_id) + if disposition != "needs_follow_up": + continue + coverage["deferred"].append( + { + "id": candidate_id, + "candidateId": candidate_id, + "reason": ( + "Validation was deferred because the scan reached its cost limit: " + f"{candidate['summary']}. Evidence: {candidate['evidence']}" + ), + "paths": paths, + "surfaceIds": [surface_id], + } + ) + if not any( + isinstance(item, dict) + and isinstance(reason := item.get("reason"), str) + and ( + reason == "Validation was deferred because the scan reached its cost limit." + or reason.startswith( + "Validation was deferred because the scan reached its cost limit: " + ) + ) + for item in coverage["deferred"] + ): + coverage["deferred"].append( + { + "id": "scan-cost-limit", + "reason": "Validation was deferred because the scan reached its cost limit.", + } + ) + coverage["completeness"] = "partial" + outputs = ( + ("findings.json", findings), + ("coverage.json", coverage), + ("scan-manifest.json", manifest), + ) + # Process death can leave a prefix of these derived writes before SQLite + # commits. Recover only bytes whose documents match the accepted inputs. + if any( + interrupted_documents[name] != payload + for name, payload in outputs + if name in interrupted_documents + ): + raise SystemExit("Budget-exhausted scan contains an incomplete canonical scan draft.") + for name, payload in outputs: + try: + write_scan_local_bytes( + scan_dir, + name, + (json.dumps(payload, allow_nan=False, indent=2, sort_keys=True) + "\n").encode(), + ) + except (ContractError, OSError, TypeError, ValueError) as exc: + raise SystemExit(f"Budget-exhausted scan draft could not be saved: {exc}") from exc + + +def retain_unmerged_budget_coverage( + scan: sqlite3.Row, scan_dir: Path, coverage: dict[str, Any], worker: sqlite3.Row +) -> None: + """Keep each unmerged review's obligations; its findings remain evidence only.""" + relative = Path(worker["accepted_result_path"]).relative_to(scan_dir).as_posix() + draft, _, _ = _read_saved_result( + scan_dir, + relative, + scan["id"], + accepted_source_digests={worker["accepted_result_path"]: worker["accepted_result_sha256"]}, + ) + source = draft["coverage"] + provenance = {"workerId": worker["id"], "attempt": worker["attempt"]} + prefix = f"{worker['id']}-attempt-{worker['attempt']}" + artifact_prefix = Path(worker["artifact_dir"]).relative_to(scan_dir).as_posix() + surfaces = { + item.get("id"): f"{prefix}-surface-{index + 1}" + for index, item in enumerate(source.get("surfaces", [])) + } + + def retain(field: str, item: dict[str, Any]) -> None: + # A committed budget draft can be replayed before the scan is sealed. + # These IDs and provenance identify the same immutable accepted review. + items = coverage.setdefault(field, []) + if "id" in item: + matches = [] + for index, existing in enumerate(items): + existing_provenance = ( + existing.get("provenance") if isinstance(existing, dict) else None + ) + if ( + isinstance(existing_provenance, dict) + and existing.get("id") == item["id"] + and all( + existing_provenance.get(key) == value for key, value in provenance.items() + ) + ): + matches.append(index) + if matches: + # Refresh older projections from the same accepted bytes, even + # if an interrupted writer saved more than one copy. + items[matches[0]] = item + for index in reversed(matches[1:]): + del items[index] + return + if item not in items: + items.append(item) + + for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"): + for index, original in enumerate(source.get(field, [])): + item = copy.deepcopy(original if isinstance(original, dict) else {"question": original}) + source_provenance = item.get("provenance") + if not isinstance(source_provenance, dict): + source_provenance = {} + # Keep source descriptions; the accepted owner supplies identity. + for key in ("workerId", "attempt", "sourceId", "candidateId"): + source_provenance.pop(key, None) + item["provenance"] = { + **source_provenance, + **provenance, + **({"sourceId": item["id"]} if "id" in item else {}), + **({"candidateId": item["candidateId"]} if "candidateId" in item else {}), + } + item["id"] = f"{prefix}-{field}-{index + 1}" + if field == "surfaces": + item["id"] = f"{prefix}-surface-{index + 1}" + item["receiptRefs"] = [ + f"{artifact_prefix}/{ref}" for ref in item.get("receiptRefs", []) + ] + if field == "deferred" and "candidateId" in item: + item["candidateId"] = f"{prefix}-candidate-{index + 1}" + if "surfaceIds" in item: + item["surfaceIds"] = [surfaces.get(value, value) for value in item["surfaceIds"]] + retain(field, item) + retain("reviews", {**provenance, "completeness": source["completeness"]}) + for index, limitation in enumerate(draft.get("scope", {}).get("limitations", [])): + retain( + "deferred", + { + "id": f"{prefix}-scope-{index + 1}", + "reason": limitation, + "provenance": provenance, + }, + ) + retain( + "deferred", + { + "id": f"{prefix}-unmerged", + "provenance": provenance, + "reason": "This accepted discovery was not merged before the scan reached its cost limit.", + }, + ) + + def coverage_for_comparison(db: Any, scan: Any) -> dict[str, Any]: if scan["seal_manifest_digest"] is None: raise SystemExit("Only sealed scans can be compared.") @@ -2088,6 +2616,7 @@ def preserve_scan_results_locked( scan_id: str, *, recovery_source_digests: dict[str, str] | None = None, + recovery_checkpoint_heads: dict[str, str] | None = None, include_parent_with_recovery: bool = False, ) -> bool: """Publish or verify retained terminal results through the workbench host.""" @@ -2095,6 +2624,7 @@ def preserve_scan_results_locked( if scan["status"] != "failed": return False frozen_source_digests: dict[str, str] | None = None + checkpoint_heads = recovery_checkpoint_heads model_source: list[str] = [] saved_model_source: str | None = None raw_frozen_sources = scan["retained_source_digests_json"] @@ -2106,10 +2636,17 @@ def preserve_scan_results_locked( model_source.append(saved_model_source) if recovery_source_digests is not None: frozen_source_digests = recovery_source_digests + elif raw_frozen_sources is not None: + raw_checkpoint_heads = scan["retained_checkpoint_heads_json"] + checkpoint_heads = ( + json.loads(raw_checkpoint_heads) if raw_checkpoint_heads is not None else {} + ) scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) deep_run = connection.execute( - "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) ).fetchone() + if deep_run is not None: + db.deep_scan.require_supported_deep_scan(deep_run) outcome = ( "canceled" if scan["canceled_at"] @@ -2157,11 +2694,15 @@ def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> No db.index_findings(connection, scan_id, findings, scan["completed_at"]) connection.execute( "UPDATE scans SET seal_manifest_digest = ?, retained_source_digests_json = ?, " + "retained_checkpoint_heads_json = ?, " "completion_warnings_json = ?, " "updated_at = ? WHERE id = ? AND status = 'failed'", ( digest, _encode_retained_sources(retained_sources, model_source), + json.dumps( + manifest["scan"].get("preservedCheckpointHeads", {}), sort_keys=True + ), json.dumps(list(dict.fromkeys(warnings))), timestamp, scan_id, @@ -2191,11 +2732,13 @@ def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> No db.verify_manifest_binding(scan, existing) if existing_scan.get("status") == outcome: existing_sources = existing_scan.get("preservedSources") + existing_heads = existing_scan.get("preservedCheckpointHeads", {}) if frozen_source_digests is None: frozen_source_digests = _source_digests( existing_sources, "Stopped scan source digests could not be frozen." ) - if existing_sources == frozen_source_digests: + checkpoint_heads = existing_heads + if existing_sources == frozen_source_digests and existing_heads == checkpoint_heads: if ( raw_frozen_sources is not None and scan["seal_manifest_digest"] is not None @@ -2217,10 +2760,7 @@ def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> No scan_dir, scan_id, binding, - connection.execute( - "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id", - (scan_id,), - ).fetchall(), + _saved_workers(connection, scan_id), warnings, stopped=True, reason=( @@ -2228,6 +2768,8 @@ def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> No f"{scan['failure_message'] or ''}" ).strip(), frozen_source_digests=frozen_source_digests, + checkpoint_heads=checkpoint_heads, + accepted_source_digests=_accepted_source_digests(connection, scan_id), frozen_model_source=model_source[0] if model_source else None, selected_model_source=model_source, allow_frozen_legacy_parent=( @@ -2260,10 +2802,13 @@ def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> No ) with connection: connection.execute( - "UPDATE scans SET retained_source_digests_json = ? " + "UPDATE scans SET retained_source_digests_json = ?, retained_checkpoint_heads_json = ? " "WHERE id = ? AND retained_source_digests_json IS ?", ( _encode_retained_sources(retained_sources, model_source), + json.dumps( + documents[0]["scan"].get("preservedCheckpointHeads", {}), sort_keys=True + ), scan_id, raw_frozen_sources, ), @@ -2296,12 +2841,15 @@ def recover_scan_results(db: Any, connection: Any, args: Any) -> dict[str, Any]: raise SystemExit("Only a stopped scan can recover terminal results.") if scan["canceled_at"] is not None: raise SystemExit("Canceled scans cannot recover terminal results.") - recovery_source_digests, include_parent = _recovery_source_digests(db, connection, scan) + recovery_source_digests, include_parent, checkpoint_heads = _recovery_source_digests( + db, connection, scan + ) if not preserve_scan_results_locked( db, connection, scan_id, recovery_source_digests=recovery_source_digests, + recovery_checkpoint_heads=checkpoint_heads, include_parent_with_recovery=include_parent, ): raise SystemExit("No saved stopped-scan results were available to recover.") @@ -2386,6 +2934,122 @@ def save_scan_artifact(db: Any, connection: Any, args: Any) -> dict[str, Any]: return {"scanId": scan_id, "path": str(scan_dir / output)} +def _read_staged_scan_draft(scan_dir: Path, draft_path: str) -> dict[str, Any]: + try: + relative = Path(draft_path).relative_to(scan_dir).as_posix() + except ValueError as exc: + raise SystemExit("Scan draft must be inside the registered scan drafts directory.") from exc + if not re.fullmatch(r"drafts/[0-9a-fA-F-]+\.json", relative): + raise SystemExit("Scan draft must be inside the registered scan drafts directory.") + return _read_scan_local_json(scan_dir, relative, "Staged scan draft") + + +def _selected_publication_digest(prepared: Any) -> str: + # Completion time is chosen at sealing, after publication. Everything else + # must remain the host projection of the same accepted aggregate. + manifest = copy.deepcopy(prepared[2]) + for field in ("completedAt", "sealedAt"): + manifest["scan"].pop(field, None) + return _digest([manifest, prepared[3], prepared[4]]) + + +def _require_selected_result(scan: Any, selection: dict[str, Any]) -> None: + relative = selection["resultPath"] + if relative is not None: + _read_saved_result( + Path(scan["scan_dir"]), + relative, + scan["id"], + kind="dedup", + accepted_source_digests={ + str(Path(scan["scan_dir"]) / relative): selection["resultSha256"] + }, + ) + + +def _selected_publication_digest_for_documents(db: Any, scan: Any, documents: Any) -> str: + prepared = _prepare_scan_finalization( + Path(scan["scan_dir"]), + expected_coverage_mode=db.expected_coverage_mode(scan), + completion_binding=db.workbench_completion_binding(scan, db.now(), documents[0]), + draft_documents=documents, + ) + return _selected_publication_digest(prepared) + + +def record_selected_publication(db: Any, connection: Any, scan: Any, documents: Any) -> None: + run = db.deep_scan.require_deep_scan_run(connection, scan["id"]) + selection = db.deep_scan.deep_scan_finalization_input(run) + if selection is None: + return + _require_selected_result(scan, selection) + selection["publicationSha256"] = _selected_publication_digest_for_documents(db, scan, documents) + with connection: + connection.execute( + "UPDATE deep_scan_runs SET finalization_input_json = ? WHERE scan_id = ?", + (json.dumps(selection), scan["id"]), + ) + + +def require_selected_publication(db: Any, connection: Any, scan: Any, prepared: Any) -> None: + if scan["mode"] != "deep": + return + run = db.deep_scan.require_deep_scan_run(connection, scan["id"]) + selection = db.deep_scan.deep_scan_finalization_input(run) + if selection is None or "publicationSha256" not in selection: + return + try: + _require_selected_result(scan, selection) + except ContractError as exc: + raise RecoverableContractError(str(exc)) from exc + if selection.get("publicationSha256") != _selected_publication_digest(prepared): + raise RecoverableContractError( + "The selected Deep Scan publication changed or is missing; " + "republish its accepted result before completing the scan." + ) + + +def _require_current_deep_publication( + db: Any, connection: Any, scan_id: str, draft: dict[str, Any] +) -> None: + publication = draft.get("deepScanPublication") + run = db.deep_scan.require_deep_scan_run(connection, scan_id) + db.deep_scan.require_current_coordinator( + run, + argparse.Namespace( + coordinator_generation=publication.get("coordinatorGeneration") if publication else None + ), + ) + selection = db.deep_scan.deep_scan_finalization_input(run) + if selection is not None: + if publication is None: + raise SystemExit("Deep Scan publication requires its committed selection.") + scan = db.require_scan(connection, scan_id) + _require_selected_result(scan, selection) + if "publicationSha256" in selection: + prepared = _prepare_scan_finalization( + Path(scan["scan_dir"]), + expected_coverage_mode=db.expected_coverage_mode(scan), + completion_binding=db.workbench_completion_binding(scan, db.now()), + draft_documents=(draft["manifest"], draft["findings"], draft["coverage"]), + ) + require_selected_publication(db, connection, scan, prepared) + selected_result = ( + str(Path(scan["scan_dir"]) / selection["resultPath"]) + if selection["resultPath"] is not None + else None + ) + else: + # Generation-one runs predate host publication metadata. Keep their existing + # draft path; adopted coordinators must carry their generation and selection. + if publication is None: + return + reducer = _latest_successful_reducer(_saved_workers(connection, scan_id)) + selected_result = reducer["result_manifest_path"] if reducer is not None else None + if publication["resultPath"] != selected_result: + raise SystemExit("Deep Scan aggregate belongs to a superseded publication selection.") + + def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]: scan_id = db.require_uuid(args.scan_id, "scan-id") with db.scan_completion_lock(scan_id): @@ -2398,6 +3062,10 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]: "The scan stopped; its saved checkpoint was retained without replacing sealed results." ) scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) + draft = None + if scan["mode"] == "deep": + draft = _read_staged_scan_draft(scan_dir, args.draft_path) + _require_current_deep_publication(db, connection, scan_id, draft) if ( args.expected_draft_digest is not None and args.expected_draft_digest != _scan_draft_digest(scan_dir) @@ -2462,6 +3130,8 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]: (json.dumps(document, allow_nan=False, indent=2) + "\n").encode(), ) model_warning = write_threat_model_projection_if_possible(scan_dir, manifest) + if scan["mode"] == "deep": + record_selected_publication(db, connection, scan, (manifest, findings, coverage)) # Accepted Standard drafts are evidence of review or report assembly, # even when the parent omitted its explicit progress call. model_only_checkpoint = ( diff --git a/plugins/codex-security/scripts/workbench_scan_history.py b/plugins/codex-security/scripts/workbench_scan_history.py index 4dde1e2c4e..6534a318bb 100644 --- a/plugins/codex-security/scripts/workbench_scan_history.py +++ b/plugins/codex-security/scripts/workbench_scan_history.py @@ -69,10 +69,16 @@ def cli_scan_resume( ): raise SystemExit("Resume requires the original owning CLI session.") run = connection.execute( - "SELECT status, cancel_requested FROM deep_scan_runs WHERE scan_id = ?", (scan["id"],) + "SELECT status, cancel_requested, finalization_input_json FROM deep_scan_runs " + "WHERE scan_id = ?", + (scan["id"],), ).fetchone() if run is not None and ( - run["status"] not in {"running", "succeeded"} or run["cancel_requested"] + run["status"] not in {"running", "succeeded"} + or ( + run["cancel_requested"] + and not (run["status"] == "succeeded" and run["finalization_input_json"] is not None) + ) ): raise SystemExit("This Deep Scan has stopped and cannot resume.") try: diff --git a/plugins/codex-security/scripts/workbench_scan_usage.py b/plugins/codex-security/scripts/workbench_scan_usage.py index 63fa3b45ac..cdfbc3da90 100644 --- a/plugins/codex-security/scripts/workbench_scan_usage.py +++ b/plugins/codex-security/scripts/workbench_scan_usage.py @@ -9,6 +9,7 @@ import sqlite3 import sys import uuid +from collections import deque from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path @@ -86,12 +87,61 @@ def collect_scan_usage( ) -> dict[str, Any]: """Count only complete, attributable rollout events inside this scan's window.""" - roots = _scan_root_thread_ids(connection, scan, thread_id) + attribution = scan_execution_attribution(connection, scan) + if attribution and attribution.get("legacy"): + attribution = None + roots = ( + list( + dict.fromkeys( + [ + *( + [attribution["owner"]["threadId"]] + if attribution["owner"].get("threadId") + else [] + ), + *attribution["executionThreadIds"], + ] + ) + ) + if attribution + else _scan_root_thread_ids(connection, scan, thread_id) + ) if not roots: return _unavailable_usage("scan_thread_unavailable") - state_database = _codex_state_database() - if state_database is None: + warnings: set[str] = set() + current_database = _codex_state_database() + worker_codex_home = None + if scan["mode"] == "deep": + # Only the workbench-bound original settings select a worker log home. + from deep_scan_workbench import ( + recorded_deep_scan_execution_settings, + validate_deep_scan_execution_settings, + ) + + run = connection.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan["id"],) + ).fetchone() + saved = recorded_deep_scan_execution_settings(run) if run is not None else None + if saved is not None: + try: + settings = validate_deep_scan_execution_settings(saved) + worker_codex_home = Path(settings["codexHome"]) + except SystemExit: + # Legacy scans may have no recorded home. Keep usage best effort. + pass + groups = [(current_database, roots)] + worker_roots: set[str] = set() + if worker_codex_home is not None: + worker_roots = set( + _scan_root_thread_ids(connection, scan, None, include_owner_threads=False) + ) + # Workers retain their Codex home, but inherit an explicit current + # SQLite home. Their earlier and resumed sessions can be in either index. + worker_database = _codex_state_database(worker_codex_home) + if worker_database != current_database: + groups.append((worker_database, [root for root in roots if root in worker_roots])) + if not any(database is not None for database, _ in groups) and not worker_roots: return _unavailable_usage("codex_state_unavailable") started_at = _timestamp(scan["started_at"]) @@ -99,23 +149,92 @@ def collect_scan_usage( if started_at is None: return _unavailable_usage("scan_window_unavailable") - warnings: set[str] = set() - try: - sessions, missing_thread_ids = _discover_rollout_sessions( - state_database, - roots, - warnings, + sessions: dict[str, list[RolloutSession]] = {} + missing_thread_ids: set[str] = set() + seen_thread_ids: set[str] = set() + for state_database, group_roots in groups: + if not group_roots: + continue + try: + if state_database is None: + raise FileNotFoundError("Codex state is unavailable") + discovered, missing = _discover_rollout_sessions( + state_database, + group_roots, + warnings, + descendant_roots=set(attribution["executionThreadIds"]) if attribution else None, + ) + except (OSError, sqlite3.Error, ValueError): + warnings.add("codex_state_unavailable") + missing_thread_ids.update(group_roots) + continue + missing_thread_ids.update(missing) + for session in discovered: + copies = sessions.setdefault(session.thread_id, []) + if session not in copies: + copies.append(session) + seen_thread_ids.add(session.thread_id) + + if worker_codex_home is not None and worker_roots: + # An external SQLite location can change on recovery. Native rollouts + # still live in the recorded worker home; they retain their lineage. + for session in _discover_recorded_worker_sessions(worker_codex_home, worker_roots): + copies = sessions.setdefault(session.thread_id, []) + if session not in copies: + copies.append(session) + seen_thread_ids.add(session.thread_id) + + # Absence from one known index is not missing usage when another has it. + missing_thread_ids.difference_update(seen_thread_ids) + if not missing_thread_ids: + warnings.difference_update( + {"scan_root_unavailable", "codex_state_unavailable", "rollout_unavailable"} ) - except (OSError, sqlite3.Error, ValueError): - return _unavailable_usage("codex_state_unavailable") if not sessions: - return _unavailable_usage("scan_thread_unavailable", warnings=warnings) + return _unavailable_usage( + "codex_state_unavailable" + if "codex_state_unavailable" in warnings + else "scan_thread_unavailable", + warnings=warnings, + ) total = _empty_token_usage() + observed_thread_count = 0 accepted_thread_ids: set[str] = set() excluded_thread_ids: set[str] = set() - for session in sessions: + model_usage: dict[str | None, dict[str, int]] = {} + children: dict[str | None, list[str]] = {} + for thread_id, copies in sessions.items(): + children.setdefault(copies[0].parent_thread_id, []).append(thread_id) + pending = deque( + thread_id + for thread_id, copies in sessions.items() + if copies[0].parent_thread_id not in sessions + ) + ordered_thread_ids: list[str] = [] + while pending: + thread_id = pending.popleft() + ordered_thread_ids.append(thread_id) + pending.extend(children.get(thread_id, [])) + # Keep unresolved lineage in the existing accounting checks as well. + ordered = set(ordered_thread_ids) + ordered_thread_ids.extend(thread_id for thread_id in sessions if thread_id not in ordered) + for thread_id in ordered_thread_ids: + copies = sessions[thread_id] + session = copies[0] + owner_turn_id = None + if ( + attribution + and session.thread_id not in attribution["executionThreadIds"] + and session.parent_thread_id is None + ): + owner = attribution["owner"] + if session.thread_id != owner.get("threadId") or not owner.get("turnId"): + missing_thread_ids.add(session.thread_id) + warnings.add("scan_owner_turn_unavailable") + continue + owner_turn_id = owner["turnId"] if session.parent_thread_id in excluded_thread_ids: excluded_thread_ids.add(session.thread_id) continue @@ -127,10 +246,12 @@ def collect_scan_usage( warnings.add("thread_lineage_incomplete") continue try: - session_usage, session_warnings = _read_rollout_usage( - session, + session_usage, session_warnings = _read_rollout_copies_usage( + copies, started_at=started_at, completed_at=stopped_at, + owner_turn_id=owner_turn_id, + model_usage=model_usage, ) except (OSError, UnicodeError, ValueError): missing_thread_ids.add(session.thread_id) @@ -146,24 +267,80 @@ def collect_scan_usage( missing_thread_ids.add(session.thread_id) continue accepted_thread_ids.add(session.thread_id) + if "token_usage_unavailable" in session_warnings: + missing_thread_ids.add(session.thread_id) + continue + observed_thread_count += 1 _add_token_usage(total, session_usage) - if not accepted_thread_ids: + if not observed_thread_count: return _unavailable_usage("scan_thread_unavailable", warnings=warnings) result: dict[str, Any] = { "coverage": "partial" if missing_thread_ids or warnings else "complete", "source": "codex_rollout", **total, - "threadCount": len(accepted_thread_ids), + "threadCount": observed_thread_count, } if missing_thread_ids: result["missingThreadCount"] = len(missing_thread_ids) if warnings: result["warnings"] = sorted(warnings) + if attribution or any(model is not None for model in model_usage): + result["modelUsage"] = [{"model": model, **usage} for model, usage in model_usage.items()] return result +def _read_rollout_copies_usage( + copies: list[RolloutSession], + *, + started_at: datetime, + completed_at: datetime | None, + owner_turn_id: str | None, + model_usage: dict[str | None, dict[str, int]], +) -> tuple[dict[str, int], set[str]]: + readings = [] + for session in copies: + local_models: dict[str | None, dict[str, int]] = {} + try: + usage, warnings = _read_rollout_usage( + session, + started_at=started_at, + completed_at=completed_at, + owner_turn_id=owner_turn_id, + model_usage=local_models, + ) + except (OSError, UnicodeError, ValueError): + continue + readings.append((usage, warnings, local_models)) + if not readings: + raise ValueError("No readable rollout copy.") + attributable = [ + reading + for reading in readings + if not reading[1].intersection( + { + "thread_identity_mismatch", + "thread_ownership_unavailable", + "thread_outside_scan_window", + "token_usage_unavailable", + } + ) + ] + # Restored indexes can reference a prefix and its complete continuation. + # Keep totals and model attribution from the same copy, counting it once. + usage, warnings, selected_models = max( + attributable or readings, + key=lambda reading: ( + reading[0]["totalTokens"], + "rollout_record_incomplete" not in reading[1], + ), + ) + for model, tokens in selected_models.items(): + _add_token_usage(model_usage.setdefault(model, _empty_token_usage()), tokens) + return usage, warnings + + def _scan_root_thread_ids( connection: sqlite3.Connection, scan: sqlite3.Row, @@ -188,12 +365,13 @@ def _scan_root_thread_ids( row["sdk_thread_id"] for row in connection.execute( """ - SELECT DISTINCT sdk_thread_id - FROM deep_scan_workers + SELECT sdk_thread_id FROM deep_scan_attempt_sessions WHERE scan_id = ? + UNION + SELECT sdk_thread_id FROM deep_scan_workers WHERE scan_id = ? AND sdk_thread_id IS NOT NULL ORDER BY sdk_thread_id """, - (scan["id"],), + (scan["id"], scan["id"]), ) ) roots: dict[str, None] = {} @@ -213,15 +391,120 @@ def _scan_execution_thread_ids(connection: sqlite3.Connection, scan: sqlite3.Row ) -def _codex_state_database() -> Path | None: - configured_database = os.environ.get("CODEX_STATE_DB", "").strip() +def capture_scan_usage_owner(connection: sqlite3.Connection, scan: sqlite3.Row) -> dict[str, Any]: + """Bind the active native turn once; joining a scan does not bind later conversation work.""" + roots = _scan_root_thread_ids(connection, scan, None) + owner = roots[0] if roots else None + result = { + "threadId": owner, + "turnId": None, + "startedAt": scan["started_at"], + "dedicated": scan["recipe_json"] is not None, + } + database = _codex_state_database() + if owner is None or database is None: + return result + try: + sessions, _ = _discover_rollout_sessions(database, [owner], set(), descendant_roots=set()) + if not sessions: + return result + with sessions[0].path.open("rb") as source: + for line in source: + if not line.endswith(b"\n"): + continue + event = json.loads(line) + payload = event.get("payload") + if not isinstance(payload, dict): + continue + if event.get("type") == "turn_context" or ( + event.get("type") == "event_msg" and payload.get("type") == "task_started" + ): + turn_id = payload.get("turn_id") + if isinstance(turn_id, str): + result["turnId"] = turn_id + elif event.get("type") == "event_msg" and payload.get("type") == "task_complete": + result["turnId"] = None + except (OSError, ValueError, sqlite3.Error): + # Accounting availability must not prevent a scan from starting. + pass + return result + + +def scan_execution_attribution( + connection: sqlite3.Connection, scan: sqlite3.Row +) -> dict[str, Any] | None: + if scan["mode"] != "deep": + return None + run = connection.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan["id"],) + ).fetchone() + if run is None: + return None + owner_json = run["usage_owner_json"] if "usage_owner_json" in run.keys() else None + legacy = False + if owner_json is None: + legacy = ( + connection.execute( + "SELECT 1 FROM deep_scan_attempts WHERE scan_id = ? LIMIT 1", (scan["id"],) + ).fetchone() + is None + ) + roots = _scan_root_thread_ids(connection, scan, None) + owner = { + "threadId": roots[0] if roots else None, + "turnId": None, + "startedAt": scan["started_at"], + "dedicated": scan["recipe_json"] is not None, + } + else: + owner = json.loads(owner_json) + executions = _scan_execution_thread_ids(connection, scan) + if owner.get("dedicated") and owner.get("threadId") not in executions: + executions.append(owner["threadId"]) + from deep_scan_workbench import ( + DEEP_SCAN_WORKFLOW_VERSION, + recorded_deep_scan_execution_settings, + validate_deep_scan_execution_settings, + ) + + saved_settings = recorded_deep_scan_execution_settings(run) + worker_home = {} + if saved_settings is not None: + worker_home["workerCodexHome"] = validate_deep_scan_execution_settings(saved_settings)[ + "codexHome" + ] + elif run["workflow_version"] == DEEP_SCAN_WORKFLOW_VERSION: + worker_home["workerCodexHome"] = None + return { + "formatVersion": 1, + **({"legacy": True} if legacy else {}), + "executionThreadIds": executions, + "owner": owner, + "startedAt": scan["started_at"], + "completedAt": scan["completed_at"], + **worker_home, + } + + +def scan_execution_fields(connection: sqlite3.Connection, scan: sqlite3.Row) -> dict[str, Any]: + return { + "threadIds": _scan_root_thread_ids(connection, scan, None), + "executionThreadIds": _scan_execution_thread_ids(connection, scan), + "executionAttribution": scan_execution_attribution(connection, scan), + } + + +def _codex_state_database(worker_codex_home: Path | None = None) -> Path | None: + configured_home = os.environ.get("CODEX_HOME", "").strip() + current_home = Path(configured_home).expanduser() if configured_home else Path.home() / ".codex" + codex_home = worker_codex_home if worker_codex_home is not None else current_home + same_home = worker_codex_home is None or codex_home.resolve() == current_home.resolve() + configured_database = os.environ.get("CODEX_STATE_DB", "").strip() if same_home else "" if configured_database: path = Path(configured_database).expanduser() return path.resolve() if path.is_file() and os.access(path, os.R_OK) else None - configured_home = os.environ.get("CODEX_HOME", "").strip() - codex_home = Path(configured_home).expanduser() if configured_home else Path.home() / ".codex" - configured_sqlite_home = os.environ.get("CODEX_SQLITE_HOME", "").strip() + configured_sqlite_home = os.environ.get("CODEX_SQLITE_HOME", "").strip() if same_home else "" search_roots = [ *([Path(configured_sqlite_home).expanduser()] if configured_sqlite_home else []), codex_home, @@ -252,6 +535,8 @@ def _discover_rollout_sessions( state_database: Path, roots: list[str], warnings: set[str], + *, + descendant_roots: set[str] | None = None, ) -> tuple[list[RolloutSession], set[str]]: database = sqlite3.connect( state_database.as_uri() + "?mode=ro", @@ -285,6 +570,8 @@ def _discover_rollout_sessions( warnings.add("rollout_unavailable") continue sessions[root] = RolloutSession(root, None, path) + if descendant_roots is not None and root not in descendant_roots: + continue descendants = database.execute( """ WITH RECURSIVE descendants( @@ -346,6 +633,50 @@ def _discover_rollout_sessions( database.close() +def _discover_recorded_worker_sessions(codex_home: Path, roots: set[str]) -> list[RolloutSession]: + recorded: dict[str, list[RolloutSession]] = {} + children: dict[str, set[str]] = {} + for candidate in sorted((codex_home / "sessions").rglob("*.jsonl")): + path = _rollout_path(str(candidate)) + if path is None: + continue + try: + with path.open("rb") as stream: + metadata = json.loads(stream.readline()) + except (OSError, UnicodeError, ValueError): + continue + if not isinstance(metadata, dict) or metadata.get("type") != "session_meta": + continue + payload = metadata.get("payload") + if not isinstance(payload, dict): + continue + thread_id = payload.get("id") or payload.get("session_id") + if not isinstance(thread_id, str): + continue + parent_id = _session_parent_thread_id(payload) + recorded.setdefault(thread_id, []).append(RolloutSession(thread_id, parent_id, path)) + if parent_id is not None: + children.setdefault(parent_id, set()).add(thread_id) + + sessions: list[RolloutSession] = [] + included = set(roots) + pending = deque(sorted(roots)) + while pending: + thread_id = pending.popleft() + for session in recorded.get(thread_id, []): + sessions.append( + RolloutSession( + thread_id, + None if thread_id in roots else session.parent_thread_id, + session.path, + ) + ) + for child_id in sorted(children.get(thread_id, set()) - included): + included.add(child_id) + pending.append(child_id) + return sessions + + def _require_state_columns( connection: sqlite3.Connection, table: str, @@ -388,11 +719,22 @@ def _read_rollout_usage( *, started_at: datetime, completed_at: datetime | None, + owner_turn_id: str | None = None, + model_usage: dict[str | None, dict[str, int]] | None = None, ) -> tuple[dict[str, int], set[str]]: total = _empty_token_usage() + counter_total = _empty_token_usage() warnings: set[str] = set() previous = _empty_token_usage() boundary_reached = False + usage_observed = False + current_turn_id: str | None = None + current_model: str | None = None + response_ids: set[str] = set() + response_usage_observed = False + response_tokens = 0 + expected_response_tokens = 0 + local_models: dict[str | None, dict[str, int]] = {} with session.path.open("rb") as source: for line_number, raw_line in enumerate(source, start=1): @@ -412,6 +754,10 @@ def _read_rollout_usage( warnings.add("rollout_record_invalid") continue payload = event.get("payload") + if event.get("type") in {"session_meta", "turn_context"} and isinstance(payload, dict): + model = payload.get("model") + if isinstance(model, str) and model: + current_model = model if line_number == 1: if event.get("type") != "session_meta" or not isinstance(payload, dict): warnings.add("thread_identity_mismatch") @@ -434,6 +780,10 @@ def _read_rollout_usage( if not isinstance(payload, dict): continue + if event.get("type") == "turn_context" or ( + event.get("type") == "event_msg" and payload.get("type") == "task_started" + ): + current_turn_id = payload.get("turn_id") if not boundary_reached: if _is_owned_task_start(session.thread_id, event, payload): task_started_at = _timestamp(event.get("timestamp")) @@ -451,28 +801,108 @@ def _read_rollout_usage( if inherited_usage is not None: previous = inherited_usage continue + if event.get("type") == "token_usage_record": + response_id = payload.get("response_id") + usage = _token_snapshot({"info": {"total_token_usage": payload.get("usage")}}) + if ( + not isinstance(response_id, str) + or usage is None + or payload.get("thread_id", session.thread_id) != session.thread_id + or response_id in response_ids + ): + continue + response_ids.add(response_id) + cumulative = _token_snapshot( + {"info": {"total_token_usage": payload.get("thread_token_usage")}} + ) + if cumulative is not None: + expected_response_tokens = max( + expected_response_tokens, cumulative["totalTokens"] + ) + response_tokens += usage["totalTokens"] + timestamp = _timestamp(event.get("timestamp")) + if timestamp is None: + warnings.add("token_record_invalid") + continue + if timestamp < started_at or ( + completed_at is not None and timestamp > completed_at + ): + continue + if ( + owner_turn_id is not None + and payload.get("turn_id", current_turn_id) != owner_turn_id + ): + continue + if not response_usage_observed: + response_usage_observed = True + total = _empty_token_usage() + local_models = {} + usage_observed = True + model = payload.get("model", current_model) + if not isinstance(model, str): + model = None + _add_token_usage(total, usage) + _add_token_usage(local_models.setdefault(model, _empty_token_usage()), usage) + continue if event.get("type") != "event_msg" or payload.get("type") != "token_count": continue + # Native rate-limit updates can carry no token usage. + if "info" in payload and payload["info"] is None: + continue timestamp = _timestamp(event.get("timestamp")) snapshot = _token_snapshot(payload) if timestamp is None or snapshot is None: warnings.add("token_record_invalid") continue - delta = { - key: value - previous[key] if value >= previous[key] else value - for key, value in snapshot.items() - } + if snapshot["totalTokens"] < previous["totalTokens"]: + warnings.add("token_counter_regressed") + continue + delta = {key: max(0, value - previous[key]) for key, value in snapshot.items()} previous = snapshot if timestamp < started_at: continue if completed_at is not None and timestamp > completed_at: continue + if owner_turn_id is not None and current_turn_id != owner_turn_id: + continue + usage_observed = True + if not response_usage_observed: + local_models.setdefault(current_model, _empty_token_usage()) if delta["totalTokens"] <= 0: continue - _add_token_usage(total, delta) + _add_token_usage(counter_total, delta) + if not response_usage_observed: + _add_token_usage(total, delta) + _add_token_usage( + local_models.setdefault(current_model, _empty_token_usage()), delta + ) + remainder = {key: max(0, value - total[key]) for key, value in counter_total.items()} + remainder["cachedInputTokens"] = min(remainder["cachedInputTokens"], remainder["inputTokens"]) + remainder["cacheWriteInputTokens"] = min( + remainder["cacheWriteInputTokens"], + remainder["inputTokens"] - remainder["cachedInputTokens"], + ) + remainder["reasoningOutputTokens"] = min( + remainder["reasoningOutputTokens"], remainder["outputTokens"] + ) + remainder["totalTokens"] = remainder["inputTokens"] + remainder["outputTokens"] + if remainder["totalTokens"] > 0: + if response_usage_observed: + warnings.add("token_receipts_incomplete") + _add_token_usage(total, remainder) + _add_token_usage(local_models.setdefault(None, _empty_token_usage()), remainder) + if response_usage_observed: + warnings.discard("token_counter_regressed") + if expected_response_tokens > response_tokens: + warnings.add("token_receipts_incomplete") + if model_usage is not None: + for model, usage in local_models.items(): + _add_token_usage(model_usage.setdefault(model, _empty_token_usage()), usage) if not boundary_reached: warnings.add("thread_ownership_unavailable") + elif not usage_observed: + warnings.add("token_usage_unavailable") return total, warnings diff --git a/plugins/codex-security/scripts/workbench_schema.py b/plugins/codex-security/scripts/workbench_schema.py index cbf54bf839..fee3f5f731 100644 --- a/plugins/codex-security/scripts/workbench_schema.py +++ b/plugins/codex-security/scripts/workbench_schema.py @@ -867,6 +867,92 @@ ); """, ), + ( + 44, + "preserve original deep scan discovery context", + """ + ALTER TABLE deep_scan_runs ADD COLUMN discovery_user_context TEXT; + UPDATE deep_scan_runs + SET discovery_user_context = ( + SELECT user_context FROM scans WHERE scans.id = deep_scan_runs.scan_id + ) + WHERE workflow_version IN ( + 'deep-security-scan/v1', 'deep-scan-mcp/v1' + ); + """, + ), + ( + 45, + "retain deep scan attempts and exact merge inputs", + """ + CREATE TABLE deep_scan_attempts ( + scan_id TEXT NOT NULL REFERENCES deep_scan_runs(scan_id) ON DELETE CASCADE, + worker_id TEXT NOT NULL REFERENCES deep_scan_workers(id) ON DELETE CASCADE, + attempt INTEGER NOT NULL, + status TEXT NOT NULL, + started_at TEXT NOT NULL, + completed_at TEXT, + end_reason TEXT, + error_message TEXT, + accepted_result_path TEXT, + accepted_result_sha256 TEXT, + receipt_json TEXT, + PRIMARY KEY (worker_id, attempt) + ); + + CREATE TABLE deep_scan_attempt_sessions ( + scan_id TEXT NOT NULL REFERENCES deep_scan_runs(scan_id) ON DELETE CASCADE, + worker_id TEXT NOT NULL, + attempt INTEGER NOT NULL, + sdk_thread_id TEXT NOT NULL, + observed_at TEXT NOT NULL, + PRIMARY KEY (worker_id, attempt, sdk_thread_id), + FOREIGN KEY (worker_id, attempt) + REFERENCES deep_scan_attempts(worker_id, attempt) ON DELETE CASCADE + ); + + CREATE TABLE deep_scan_merge_claims ( + worker_id TEXT PRIMARY KEY REFERENCES deep_scan_workers(id) ON DELETE CASCADE, + scan_id TEXT NOT NULL REFERENCES deep_scan_runs(scan_id) ON DELETE CASCADE, + previous_worker_id TEXT, + previous_result_path TEXT, + previous_result_sha256 TEXT, + receipt_json TEXT + ); + + ALTER TABLE deep_scan_dedup_inputs ADD COLUMN result_manifest_path TEXT; + ALTER TABLE deep_scan_dedup_inputs ADD COLUMN result_manifest_sha256 TEXT; + ALTER TABLE deep_scan_dedup_inputs ADD COLUMN attempt INTEGER; + """, + ), + ( + 46, + "persist selected deep scan finalization input", + """ + ALTER TABLE deep_scan_runs ADD COLUMN finalization_input_json TEXT; + """, + ), + ( + 47, + "freeze stopped scan checkpoint selections", + """ + ALTER TABLE scans ADD COLUMN retained_checkpoint_heads_json TEXT; + """, + ), + ( + 48, + "bind original deep scan parent usage turn", + """ + ALTER TABLE deep_scan_runs ADD COLUMN usage_owner_json TEXT; + """, + ), + ( + 51, + "bind original deep scan execution settings", + """ + ALTER TABLE deep_scan_runs ADD COLUMN execution_settings_json TEXT; + """, + ), ) diff --git a/plugins/codex-security/scripts/workbench_validation.py b/plugins/codex-security/scripts/workbench_validation.py index 9fd6988eda..3d6ecf0096 100644 --- a/plugins/codex-security/scripts/workbench_validation.py +++ b/plugins/codex-security/scripts/workbench_validation.py @@ -118,6 +118,7 @@ def _valid_measured_scan_usage(usage: object) -> bool: "threadCount", "missingThreadCount", "warnings", + "modelUsage", *SCAN_USAGE_TOKEN_KEYS, } if thread_count == 0 or not set(usage).issubset(allowed_keys): @@ -127,6 +128,22 @@ def _valid_measured_scan_usage(usage: object) -> bool: or usage["cachedInputTokens"] + usage["cacheWriteInputTokens"] > usage["inputTokens"] ): return False + if "modelUsage" in usage: + parts = usage["modelUsage"] + if not isinstance(parts, list) or not parts: + return False + for part in parts: + if not isinstance(part, dict) or set(part) != {"model", *SCAN_USAGE_TOKEN_KEYS}: + return False + if part["model"] is not None and not isinstance(part["model"], str): + return False + if ( + not all(type(part[key]) is int and part[key] >= 0 for key in SCAN_USAGE_TOKEN_KEYS) + or part["cachedInputTokens"] + part["cacheWriteInputTokens"] > part["inputTokens"] + ): + return False + if any(sum(part[key] for part in parts) != usage[key] for key in SCAN_USAGE_TOKEN_KEYS): + return False missing = usage.get("missingThreadCount", 0) if type(missing) is not int or missing < 0: return False @@ -137,7 +154,7 @@ def _valid_measured_scan_usage(usage: object) -> bool: return True -def parse_scan_cost(value: str | None) -> str | None: +def parse_scan_cost(value: str | None, *, allow_lower_bound: bool = False) -> str | None: if value is None: return None if len(value.encode("utf-8")) > 8192: @@ -146,7 +163,10 @@ def parse_scan_cost(value: str | None) -> str | None: cost = json.loads(value, parse_constant=reject_nonstandard_json_number) except (TypeError, UnicodeError, ValueError) as exc: raise SystemExit("Scan cost must be a valid JSON object.") from exc - if isinstance(cost, dict) and "usage" in cost: + if allow_lower_bound and isinstance(cost, dict) and set(cost) == {"lowerBound"}: + if not _valid_legacy_scan_cost(cost["lowerBound"]): + raise SystemExit("Scan cost lower bound must be a valid measured cost.") + elif isinstance(cost, dict) and "usage" in cost: if ( not set(cost).issubset({"usage", "cost"}) or not _valid_measured_scan_usage(cost["usage"]) diff --git a/plugins/codex-security/tests/test_accepted_publication_references.py b/plugins/codex-security/tests/test_accepted_publication_references.py new file mode 100644 index 0000000000..2f7580525f --- /dev/null +++ b/plugins/codex-security/tests/test_accepted_publication_references.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import hashlib +import json +from argparse import Namespace + +import pytest +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +def accept_reducer(connection, scan): + result = add_worker(connection, scan) + worker_id = result.parent.name + coverage = { + **scan.coverage, + "completeness": "partial", + "deferred": [{"id": "accepted-follow-up", "reason": "Accepted unresolved review."}], + "reviews": [{"workerId": worker_id, "attempt": 1, "completeness": "partial"}], + } + contents = json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": scan.findings, + "sourceCoverage": coverage, + } + ).encode() + digest = hashlib.sha256(contents).hexdigest() + result.write_bytes(contents) + accepted = result.parent / "accepted" / f"{digest}.json" + accepted.parent.mkdir() + accepted.write_bytes(contents) + result.unlink() + with connection: + connection.execute( + "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none' WHERE id = ?", + (worker_id,), + ) + connection.execute( + "INSERT INTO deep_scan_attempts (scan_id, worker_id, attempt, status, started_at, " + "completed_at, accepted_result_path, accepted_result_sha256) " + "VALUES (?, ?, 1, 'succeeded', ?, ?, ?, ?)", + (scan.scan_id, worker_id, scan.timestamp, scan.timestamp, str(accepted), digest), + ) + connection.execute( + "UPDATE deep_scan_runs SET coordinator_generation = 3 WHERE scan_id = ?", + (scan.scan_id,), + ) + return result, accepted, coverage + + +@pytest.mark.parametrize("selected", [True, False], ids=["accepted", "replaceable-output"]) +def test_legacy_publication_compares_registered_accepted_reference( + workbench_api, workbench_db, publication_scan, selected +): + scan = publication_scan() + result, accepted, _ = accept_reducer(workbench_db, scan) + staged = stage_publication( + scan, generation=3, result_path=accepted if selected else result, title="Accepted aggregate" + ) + before = {path: path.read_bytes() for path in scan.scan_dir.rglob("*.json")} + + if selected: + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, staged + ) + else: + with pytest.raises(SystemExit, match="aggregate"): + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, staged + ) + assert {path: path.read_bytes() for path in scan.scan_dir.rglob("*.json")} == before + + +def test_stopped_recovery_uses_accepted_bytes_after_replaceable_output_disappears( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + _, accepted, coverage = accept_reducer(workbench_db, scan) + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + contents = accepted.read_bytes() + + stopped = workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."), + )["scan"] + + assert stopped["findingCount"] == 1 + published = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert published["reviews"] == coverage["reviews"] + assert coverage["deferred"][0] in published["deferred"] + manifest = (scan.scan_dir / "scan-manifest.json").read_bytes() + workbench_api["saved_results"].preserve_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None + ), + ) + assert (scan.scan_dir / "scan-manifest.json").read_bytes() == manifest + assert accepted.read_bytes() == contents diff --git a/plugins/codex-security/tests/test_budget_coverage_provenance.py b/plugins/codex-security/tests/test_budget_coverage_provenance.py new file mode 100644 index 0000000000..458f873a45 --- /dev/null +++ b/plugins/codex-security/tests/test_budget_coverage_provenance.py @@ -0,0 +1,250 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import sqlite3 +from argparse import Namespace + +import pytest +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_publication_stop_interleavings import published_bytes +from test_workbench_db import BUDGET_COST + + +@pytest.mark.parametrize("explicit_ids", [True, False], ids=["named-surfaces", "omitted-ids"]) +@pytest.mark.parametrize("replay", ["fresh", "legacy", "duplicated"]) +def test_cost_completion_retains_independent_unmerged_surfaces_and_receipts( + workbench_api, + workbench_db, + publication_scan, + explicit_ids, + replay, + tmp_path, + monkeypatch, +): + scan = publication_scan() + worker_id = add_worker(workbench_db, scan).parent.name + output = scan.scan_dir / "artifacts/deep_discovery/workers/discovery-0001/output" + output.mkdir(parents=True) + descriptions = { + "source": "source-review", + "worker": "focused-reviewer", + "task": "Check the filesystem boundary.", + } + source_provenance = { + **descriptions, + "workerId": "worker-local-label", + "attempt": 99, + "sourceId": "worker-local-source", + "candidateId": "worker-local-candidate", + } + surfaces = [ + { + "label": "Filesystem boundary", + "disposition": "needs_follow_up", + "notes": "Filesystem race remains untested.", + "reason": "The caller's filesystem policy is unknown.", + }, + { + "label": "Template boundary", + "disposition": "needs_follow_up", + "notes": "Template caller policy remains untested.", + }, + { + "label": "Configuration boundary", + "disposition": "rejected", + "reason": "The input comes only from trusted application configuration.", + }, + ] + receipts = {} + for index, surface in enumerate(surfaces, start=1): + surface["provenance"] = source_provenance + if explicit_ids: + surface["id"] = f"source-surface-{index}" + relative = f"artifacts/review-{index}.md" + surface["receiptRefs"] = [relative] + receipt = output / relative + receipt.parent.mkdir(exist_ok=True) + receipts[receipt] = f"Independent review receipt {index}.\n".encode() + receipt.write_bytes(receipts[receipt]) + deferred = { + "reason": "Both independent validation tasks remain unfinished.", + "provenance": source_provenance, + } + if explicit_ids: + deferred["surfaceIds"] = [surface["id"] for surface in surfaces] + contents = json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": scan.findings, + "coverage": { + **scan.coverage, + "completeness": "partial", + "surfaces": surfaces, + "deferred": [deferred], + "explicitExclusions": [ + { + "pattern": "vendor/", + "reason": "External dependencies were excluded.", + "provenance": source_provenance, + } + ], + "openQuestions": [ + { + "question": "Which deployment controls apply?", + "provenance": source_provenance, + } + ], + }, + } + ).encode() + digest = hashlib.sha256(contents).hexdigest() + accepted = output / "checkpoints" / f"{digest}.json" + accepted.parent.mkdir() + accepted.write_bytes(contents) + # The mutable result may advance after acceptance. Publication must read the + # accepted checkpoint, while receipt paths remain relative to its owner. + result = output / "result.json" + result.write_text("A later, unaccepted result.") + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET artifact_dir = ?, result_manifest_path = ?, " + "prompt_path = ?, merge_state = 'buffered' WHERE id = ?", + (str(output), str(result), str(output.parent / "prompt.md"), worker_id), + ) + workbench_db.execute( + "INSERT INTO deep_scan_attempts (scan_id, worker_id, attempt, status, started_at, " + "completed_at, accepted_result_path, accepted_result_sha256) " + "VALUES (?, ?, 1, 'succeeded', ?, ?, ?, ?)", + (scan.scan_id, worker_id, scan.timestamp, scan.timestamp, str(accepted), digest), + ) + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'discovery', " + "workflow_version = 'deep-security-scan/v2', manifest_path = NULL, " + "terminal_reason = NULL, completed_at = NULL, max_discovery_runs = 100" + ) + counter_query = ( + "SELECT discovery_runs_dispatched, completion_sequence, consecutive_no_new " + "FROM deep_scan_runs" + ) + counters = tuple(workbench_db.execute(counter_query).fetchone()) + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + + budget = workbench_api["complete_budget_exhausted_scan"] + args = Namespace( + scan_id=scan.scan_id, + cost_json=json.dumps(BUDGET_COST), + message="Scan reached its original cost limit.", + ) + if replay != "fresh": + saved = budget.__globals__["saved_results"] + retain = saved.retain_unmerged_budget_coverage + + def legacy_projection(*args): + retain(*args) + # Older writers kept host identity but omitted source descriptions. + for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"): + items = args[2].get(field, []) + for item in list(items): + if item.get("provenance", {}).get("workerId") == worker_id: + if replay == "duplicated" and any( + key in item["provenance"] for key in descriptions + ): + # A second interrupted writer appended the described + # projection beside the old item before sealing. + items.append(copy.deepcopy(item)) + for key in descriptions: + item["provenance"].pop(key, None) + + def interrupt_before_seal(*args): + raise RuntimeError("Publication interrupted after the budget draft committed.") + + database = tmp_path / "interrupted-budget.sqlite3" + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + workbench_db.backup(connection) + with monkeypatch.context() as patch: + patch.setattr(saved, "retain_unmerged_budget_coverage", legacy_projection) + # The legacy writer predates the selected-publication digest. + patch.setattr(saved, "record_selected_publication", lambda *args: None) + patch.setitem(budget.__globals__, "complete_scan_locked", interrupt_before_seal) + with pytest.raises(RuntimeError, match="budget draft committed"): + budget(connection, args) + # Reconnect to the committed old draft and run the real public completion. + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + assert ( + connection.execute("SELECT status FROM deep_scan_runs").fetchone()[0] == "succeeded" + ) + assert connection.execute("SELECT status FROM scans").fetchone()[0] == "running" + assert ( + connection.execute("SELECT seal_manifest_digest FROM scans").fetchone()[0] is None + ) + budget(connection, args) + connection.backup(workbench_db) + else: + budget(workbench_db, args) + + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "complete" + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert coverage["completeness"] == "partial" + actual = coverage["surfaces"] + assert [(item["label"], item.get("notes"), item.get("reason")) for item in actual] == [ + (item["label"], item.get("notes"), item.get("reason")) for item in surfaces + ] + assert len({item["id"] for item in actual}) == len(surfaces) + for original, retained in zip(surfaces, actual, strict=True): + assert retained["provenance"] == { + **descriptions, + "workerId": worker_id, + "attempt": 1, + **({"sourceId": original["id"]} if explicit_ids else {}), + } + assert retained["receiptRefs"] == [ + (output / ref).relative_to(scan.scan_dir).as_posix() for ref in original["receiptRefs"] + ] + for path, receipt_contents in receipts.items(): + assert path.read_bytes() == receipt_contents + obligation = next(item for item in coverage["deferred"] if item["reason"] == deferred["reason"]) + assert obligation["provenance"] == {**descriptions, "workerId": worker_id, "attempt": 1} + assert sum(item["reason"] == deferred["reason"] for item in coverage["deferred"]) == 1 + for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"): + assert len({item["id"] for item in coverage[field]}) == len(coverage[field]) + for field in ("explicitExclusions", "openQuestions"): + assert len(coverage[field]) == 1 + item = coverage[field][0] + assert item["id"] == f"{worker_id}-attempt-1-{field}-1" + assert item["provenance"] == {**descriptions, "workerId": worker_id, "attempt": 1} + if explicit_ids: + assert obligation["surfaceIds"] == [item["id"] for item in actual] + assert {"workerId": worker_id, "attempt": 1, "completeness": "partial"} in coverage["reviews"] + report = (scan.scan_dir / "report.md").read_text() + for surface in surfaces: + for field in ("notes", "reason"): + if field in surface: + assert surface[field] in report + assert deferred["reason"] in report + assert json.loads((scan.scan_dir / "findings.json").read_text())["findings"] == [] + assert tuple(workbench_db.execute(counter_query).fetchone()) == counters + attempt = workbench_db.execute( + "SELECT accepted_result_path, accepted_result_sha256 FROM deep_scan_attempts " + "WHERE worker_id = ?", + (worker_id,), + ).fetchone() + assert tuple(attempt) == (str(accepted), digest) + assert accepted.read_bytes() == contents + before = published_bytes(scan) + workbench_api["complete_scan"]( + workbench_db, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + assert published_bytes(scan) == before + assert accepted.read_bytes() == contents diff --git a/plugins/codex-security/tests/test_budget_finding_identity.py b/plugins/codex-security/tests/test_budget_finding_identity.py new file mode 100644 index 0000000000..44bcaec826 --- /dev/null +++ b/plugins/codex-security/tests/test_budget_finding_identity.py @@ -0,0 +1,251 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import sqlite3 +from argparse import Namespace + +import pytest +from test_accepted_publication_references import accept_reducer +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_publication_stop_interleavings import published_bytes +from test_workbench_db import BUDGET_COST +from workbench_test_support import run_workbench + + +def prepare_budget_publication(connection, scan): + _, accepted, _ = accept_reducer(connection, scan) + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + with connection: + recipe = json.loads(connection.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + connection.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + connection.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'discovery', " + "workflow_version = 'deep-security-scan/v2', manifest_path = NULL, " + "terminal_reason = NULL, completed_at = NULL" + ) + return accepted, Namespace( + scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=None + ) + + +@pytest.mark.parametrize("identity_kind", ["omitted", "candidate", "authored", "anchor-only"]) +def test_budget_publication_identifies_accepted_semantic_findings( + workbench_api, workbench_db, publication_scan, identity_kind +): + scan = publication_scan() + finding = scan.findings[0] + finding["title"] = "Archive extraction crosses output boundary" + finding.pop("identity", None) + finding.pop("extensions", None) + expected = {"anchor": "archive-extraction-crosses-output-boundary"} + if identity_kind == "candidate": + finding["extensions"] = {"candidateId": "archive-candidate"} + expected = {"anchor": "archive-candidate"} + elif identity_kind == "authored": + finding["identity"] = {"anchor": "authored-anchor", "instance": "first-route"} + expected = finding["identity"].copy() + elif identity_kind == "anchor-only": + finding["identity"] = expected = {"anchor": "authored-anchor"} + accepted, args = prepare_budget_publication(workbench_db, scan) + original = accepted.read_bytes() + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + published = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(published) == 1 + assert published[0]["identity"] == expected + assert published[0]["codeEvidence"] == finding["codeEvidence"] + assert published[0]["remediation"] == finding["remediation"] + assert accepted.read_bytes() == original + selection = json.loads( + workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + ) + assert selection["resultSha256"] == hashlib.sha256(original).hexdigest() + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "complete" + + +def add_sibling_repairs(scan, count=2): + first = scan.findings[0] + first["title"] = "Shared repair" + first.pop("identity", None) + first.pop("extensions", None) + scan.findings.extend(copy.deepcopy(first) for _ in range(count - 1)) + for index, finding in enumerate(scan.findings, 1): + finding["locations"] = [{"path": "subdir/extract.py", "startLine": index}] + finding["provenance"]["sourceFindingIds"] = [f"review-{index}:candidate-{index}"] + finding["provenance"]["sourceFindings"][0]["id"] = f"review-{index}:candidate-{index}" + finding["codeEvidence"][0].update( + startLine=index, endLine=index, code=f"repair_point_{index}()" + ) + finding["remediation"] = f"Apply distinct repair {index} at this location." + finding["remediationTests"] = [f"Check distinct repair {index}."] + + +@pytest.mark.parametrize( + "case", + [ + "generated", + "candidate", + "authored-first", + "authored-last", + "reserved-instance", + "authored-siblings", + "different-rules", + "different-titles", + ], +) +def test_budget_publication_keeps_generated_sibling_repairs( + workbench_api, workbench_db, publication_scan, case +): + scan = publication_scan() + add_sibling_repairs(scan, 4 if case == "reserved-instance" else 2) + if case == "candidate": + for finding in scan.findings: + finding["extensions"] = {"candidateId": "shared-candidate"} + elif case in {"authored-first", "authored-last"}: + scan.findings[0 if case == "authored-first" else 1]["identity"] = { + "anchor": "shared-repair" + } + elif case == "reserved-instance": + scan.findings[2]["identity"] = {"anchor": "shared-repair"} + scan.findings[3]["identity"] = {"anchor": "shared-repair", "instance": "saved-2"} + elif case == "authored-siblings": + for index, finding in enumerate(scan.findings): + finding["identity"] = {"anchor": "shared-repair", "instance": f"route-{index}"} + elif case == "different-rules": + scan.findings[1]["ruleId"] = "distinct-rule" + elif case == "different-titles": + scan.findings[1]["title"] = "Distinct repair" + originals = copy.deepcopy(scan.findings) + accepted, args = prepare_budget_publication(workbench_db, scan) + original = accepted.read_bytes() + + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + + published = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(published) == len(scan.findings) + assert len({finding["occurrenceId"] for finding in published}) == len(scan.findings) + for expected, actual in zip(scan.findings, published, strict=True): + for key in expected.keys() - {"findingId", "occurrenceId", "fingerprints"}: + assert actual[key] == expected[key] + if "identity" not in expected: + anchor = ( + "shared-candidate" + if case == "candidate" + else expected["title"].lower().replace(" ", "-") + ) + assert actual["identity"]["anchor"] == anchor + assert expected["remediation"] in (scan.scan_dir / "report.md").read_text() + if case in {"different-rules", "different-titles"}: + assert "instance" not in actual["identity"] + selection = workbench_db.execute( + "SELECT finalization_input_json FROM deep_scan_runs" + ).fetchone()[0] + assert json.loads(selection)["resultSha256"] == hashlib.sha256(original).hexdigest() + before = published_bytes(scan) + workbench_api["complete_scan"]( + workbench_db, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + assert published_bytes(scan) == before + assert ( + workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + == selection + ) + assert accepted.read_bytes() == original + assert scan.findings == originals + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "complete" + + +@pytest.mark.parametrize("identity", [{}, {"instance": "first-route"}, None]) +def test_budget_publication_does_not_repair_authored_invalid_identity( + workbench_api, workbench_db, publication_scan, identity +): + scan = publication_scan() + scan.findings[0]["identity"] = identity + accepted, args = prepare_budget_publication(workbench_db, scan) + original = accepted.read_bytes() + with pytest.raises(SystemExit, match="identity"): + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + assert accepted.read_bytes() == original + assert scan.findings[0]["identity"] == identity + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] != "complete" + + +def test_budget_publication_rejects_duplicate_authored_identities( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + add_sibling_repairs(scan) + for finding in scan.findings: + finding["identity"] = {"anchor": "shared-repair", "instance": "authored-route"} + accepted, args = prepare_budget_publication(workbench_db, scan) + original = accepted.read_bytes() + with pytest.raises(SystemExit, match="duplicate occurrence identity"): + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + assert accepted.read_bytes() == original + + +@pytest.mark.parametrize("boundary", ["before-selection-commit", "after-selection-commit"]) +def test_budget_sibling_selection_replays_in_a_fresh_process( + workbench_api, workbench_db, publication_scan, tmp_path, monkeypatch, boundary +): + scan = publication_scan() + add_sibling_repairs(scan) + accepted, args = prepare_budget_publication(workbench_db, scan) + original = accepted.read_bytes() + budget = workbench_api["complete_budget_exhausted_scan"] + drafts = [] + + def interrupt(*args, **kwargs): + drafts.append(json.loads((scan.scan_dir / "findings.json").read_text())["findings"]) + raise RuntimeError("Synthetic publication interruption.") + + state = tmp_path / "state" + state.mkdir(exist_ok=True) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + workbench_db.backup(connection) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + with monkeypatch.context() as patch: + if boundary == "before-selection-commit": + patch.setattr(workbench_api["deep_scan"], "cancel_active_workers", interrupt) + else: + patch.setitem(budget.__globals__, "complete_scan_locked", interrupt) + with pytest.raises(RuntimeError, match="Synthetic publication interruption"): + budget(connection, args) + selection = connection.execute( + "SELECT finalization_input_json FROM deep_scan_runs" + ).fetchone()[0] + if boundary == "before-selection-commit": + assert selection is None + assert not (scan.scan_dir / "findings.json").exists() + else: + assert json.loads(selection)["resultSha256"] == hashlib.sha256(original).hexdigest() + run_workbench( + state, + "complete-budget-exhausted-scan", + "--scan-id", + scan.scan_id, + "--cost-json", + args.cost_json, + ) + published = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert [finding["identity"] for finding in published] == [ + finding["identity"] for finding in drafts[0] + ] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + replayed_selection = connection.execute( + "SELECT finalization_input_json FROM deep_scan_runs" + ).fetchone()[0] + if selection is not None: + assert replayed_selection == selection + assert ( + json.loads(replayed_selection)["resultSha256"] == hashlib.sha256(original).hexdigest() + ) + assert connection.execute("SELECT status FROM scans").fetchone()[0] == "complete" + before = published_bytes(scan) + run_workbench(state, "complete-scan", "--scan-id", scan.scan_id) + assert published_bytes(scan) == before + assert accepted.read_bytes() == original diff --git a/plugins/codex-security/tests/test_budget_report_evidence.py b/plugins/codex-security/tests/test_budget_report_evidence.py new file mode 100644 index 0000000000..ca2803aad2 --- /dev/null +++ b/plugins/codex-security/tests/test_budget_report_evidence.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import json +from argparse import Namespace + +import pytest +from test_budget_selection_publication import accept_unmerged +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_workbench_db import BUDGET_COST + + +@pytest.mark.parametrize("accepted", [False, True], ids=["unfinished-audit", "accepted-unmerged"]) +def test_empty_budget_report_does_not_deny_saved_validation( + workbench_api, workbench_db, publication_scan, accepted +): + scan = publication_scan() + if accepted: + evidence = accept_unmerged(workbench_db, scan) + else: + result = add_worker(workbench_db, scan, status="running") + checkpoints = result.parent / "checkpoints" + checkpoints.mkdir() + evidence = checkpoints / ("a" * 64 + ".json") + evidence.write_text( + json.dumps( + { + "scanId": scan.scan_id, + "complete": False, + "findings": scan.findings, + "coverage": {**scan.coverage, "completeness": "partial"}, + } + ) + ) + (result.parent / "checkpoint-head.json").write_text( + json.dumps({"checkpoint": evidence.name}) + ) + original = evidence.read_bytes() + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'discovery', " + "workflow_version = 'deep-security-scan/v2', manifest_path = NULL, " + "terminal_reason = NULL, completed_at = NULL" + ) + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + + workbench_api["complete_budget_exhausted_scan"]( + workbench_db, + Namespace(scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=None), + ) + assert evidence.read_bytes() == original + assert json.loads(original)["findings"] + assert json.loads((scan.scan_dir / "findings.json").read_text())["findings"] == [] + assert json.loads((scan.scan_dir / "coverage.json").read_text())["completeness"] == "partial" + report = (scan.scan_dir / "report.md").read_text() + assert "No findings were validated" not in report + assert "No findings are included in this partial report." in report + assert "Review the unresolved work in Open Questions And Follow Up." in report diff --git a/plugins/codex-security/tests/test_budget_selection_publication.py b/plugins/codex-security/tests/test_budget_selection_publication.py new file mode 100644 index 0000000000..10150bc284 --- /dev/null +++ b/plugins/codex-security/tests/test_budget_selection_publication.py @@ -0,0 +1,890 @@ +from __future__ import annotations + +import hashlib +import json +import sqlite3 +import subprocess +import sys +from argparse import Namespace +from pathlib import Path + +import pytest +from test_accepted_publication_references import accept_reducer +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_publication_stop_interleavings import published_bytes, saved_selection +from test_workbench_db import BUDGET_COST + + +def accept_unmerged(connection, scan): + result = add_worker(connection, scan) + worker_id = result.parent.name + result.write_text( + json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": scan.findings, + "coverage": { + **scan.coverage, + "completeness": "partial", + "deferred": [ + { + "id": "shared-gap", + "reason": "Independent unresolved review.", + "paths": ["subdir/extract.py"], + } + ], + }, + } + ) + ) + digest = hashlib.sha256(result.read_bytes()).hexdigest() + accepted = result.parent / "accepted" / f"{digest}.json" + accepted.parent.mkdir() + accepted.write_bytes(result.read_bytes()) + with connection: + connection.execute( + "UPDATE deep_scan_workers SET merge_state = 'buffered' WHERE id = ?", (worker_id,) + ) + connection.execute( + "INSERT INTO deep_scan_attempts (scan_id, worker_id, attempt, status, started_at, " + "completed_at, accepted_result_path, accepted_result_sha256) VALUES (?, ?, 1, 'succeeded', ?, ?, ?, ?)", + (scan.scan_id, worker_id, scan.timestamp, scan.timestamp, str(accepted), digest), + ) + return accepted + + +@pytest.mark.parametrize("defect", ["changed-bytes", "incomplete-result"]) +def test_cost_before_selection_does_not_replace_invalid_accepted_evidence( + workbench_api, workbench_db, publication_scan, defect +): + scan = publication_scan() + _, accepted, _ = accept_reducer(workbench_db, scan) + document = json.loads(accepted.read_bytes()) + document["complete"] = False + accepted.write_text(json.dumps(document)) + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'discovery', " + "workflow_version = 'deep-security-scan/v2', finalization_input_json = NULL" + ) + if defect == "incomplete-result": + workbench_db.execute( + "UPDATE deep_scan_attempts SET accepted_result_sha256 = ?", + (hashlib.sha256(accepted.read_bytes()).hexdigest(),), + ) + before = published_bytes(scan) + with pytest.raises(SystemExit): + workbench_api["complete_budget_exhausted_scan"]( + workbench_db, + Namespace(scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=None), + ) + assert published_bytes(scan) == before + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "running" + assert ( + workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + is None + ) + + +@pytest.mark.parametrize("accepted_kind", ["none", "unmerged", "reducer"]) +@pytest.mark.parametrize("cancel_first", [False, True]) +@pytest.mark.parametrize("lower_bound", [False, True]) +def test_cost_before_selection_retains_only_merged_findings( + workbench_api, workbench_db, publication_scan, accepted_kind, cancel_first, lower_bound +): + scan = publication_scan() + accepted = None + if accepted_kind == "reducer": + _, accepted, _ = accept_reducer(workbench_db, scan) + elif accepted_kind == "unmerged": + accepted = accept_unmerged(workbench_db, scan) + active = add_worker(workbench_db, scan, status="running") + active_id = active.parent.name + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET completed_at = NULL WHERE id = ?", (active_id,) + ) + workbench_db.execute( + "INSERT INTO deep_scan_attempts (scan_id, worker_id, attempt, status, started_at) " + "VALUES (?, ?, 1, 'running', ?)", + (scan.scan_id, active_id, scan.timestamp), + ) + counters = tuple( + workbench_db.execute( + "SELECT discovery_runs_dispatched, consecutive_no_new, completion_sequence FROM deep_scan_runs" + ).fetchone() + ) + original = accepted.read_bytes() if accepted is not None else None + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'discovery', " + "workflow_version = 'deep-security-scan/v2', manifest_path = NULL, " + "terminal_reason = NULL, completed_at = NULL, max_discovery_runs = 100" + ) + args = Namespace( + scan_id=scan.scan_id, + cost_json=json.dumps({"lowerBound": BUDGET_COST} if lower_bound else BUDGET_COST), + message="Scan reached its original cost limit.", + ) + if cancel_first: + workbench_api["saved_results"].cancel_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace(scan_id=scan.scan_id, thread_id=None), + ) + before = published_bytes(scan) + with pytest.raises(SystemExit, match="running"): + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + assert published_bytes(scan) == before + else: + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + row = workbench_db.execute("SELECT * FROM scans").fetchone() + run = workbench_db.execute("SELECT * FROM deep_scan_runs").fetchone() + assert row["status"] == "complete" + stored = json.loads(row["cost_json"]) + if lower_bound: + assert "cost" not in stored and "estimatedUsd" not in stored + assert stored["usage"]["coverage"] == "unavailable" + else: + assert stored == BUDGET_COST + assert run["terminal_reason"] == "capped" + assert run["error_message"] == args.message + assert args.message in json.loads(row["completion_warnings_json"]) + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == (1 if accepted_kind == "reducer" else 0) + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert coverage["completeness"] == "partial" + assert any(item["id"] == "scan-cost-limit" for item in coverage["deferred"]) + if accepted_kind == "reducer": + selection = json.loads(run["finalization_input_json"]) + assert selection["resultSha256"] == hashlib.sha256(original).hexdigest() + assert any(item["id"] == "accepted-follow-up" for item in coverage["deferred"]) + assert ( + tuple( + workbench_db.execute( + "SELECT discovery_runs_dispatched, consecutive_no_new, completion_sequence FROM deep_scan_runs" + ).fetchone() + ) + == counters + ) + interrupted = workbench_db.execute( + "SELECT * FROM deep_scan_attempts WHERE worker_id = ?", (active_id,) + ).fetchone() + assert interrupted["status"] == "canceled" + assert interrupted["end_reason"] == "scan_stopped" + if accepted_kind == "unmerged": + assert any( + item.get("provenance", {}).get("sourceId") == "shared-gap" + for item in coverage["deferred"] + ) + before = published_bytes(scan) + workbench_api["complete_scan"]( + workbench_db, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + assert published_bytes(scan) == before + if accepted is not None: + assert accepted.read_bytes() == original + + +@pytest.mark.parametrize( + "bound", [None, {**BUDGET_COST, "estimatedUsd": -1}, {**BUDGET_COST, "estimatedUsd": 0.005}] +) +def test_lower_bound_budget_rejects_invalid_or_unexceeded_cost( + workbench_api, workbench_db, publication_scan, bound +): + scan = publication_scan() + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + before = published_bytes(scan) + with pytest.raises(SystemExit): + workbench_api["complete_budget_exhausted_scan"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, cost_json=json.dumps({"lowerBound": bound}), message=None + ), + ) + assert published_bytes(scan) == before + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "running" + + +def test_lower_bound_is_not_an_ordinary_completion_cost( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + before = published_bytes(scan) + with pytest.raises(SystemExit): + workbench_api["complete_scan"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, + claim_token=None, + cost_json=json.dumps({"lowerBound": BUDGET_COST}), + ), + ) + assert published_bytes(scan) == before + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "running" + + +@pytest.mark.parametrize("reason", ["saturated", "capped"]) +def test_budget_keeps_selection_committed_before_transaction_and_unmerged_obligations( + workbench_api, workbench_db, publication_scan, reason +): + scan = publication_scan() + _, accepted, original_coverage = accept_reducer(workbench_db, scan) + unmerged = [accept_unmerged(workbench_db, scan) for _ in range(2)] + originals = {path: path.read_bytes() for path in [accepted, *unmerged]} + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', workflow_version = 'deep-security-scan/v2'" + ) + # The selected publication may commit before cost recovery reaches its + # transaction. Budget completion must use that exact input and cause. + committed = saved_selection(workbench_db, scan, accepted, reason=reason) + workbench_api["complete_budget_exhausted_scan"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message="Original cost stop." + ), + ) + run = workbench_db.execute("SELECT * FROM deep_scan_runs").fetchone() + recorded = json.loads(run["finalization_input_json"]) + assert {key: recorded[key] for key in committed} == committed + assert len(recorded["publicationSha256"]) == 64 + assert run["terminal_reason"] == reason + assert all(path.read_bytes() == contents for path, contents in originals.items()) + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == 1 + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert original_coverage["deferred"][0] in coverage["deferred"] + gaps = [ + item + for item in coverage["deferred"] + if item.get("provenance", {}).get("sourceId") == "shared-gap" + ] + assert len(gaps) == 2 + assert len({item["id"] for item in gaps}) == 2 + assert {item["provenance"]["workerId"] for item in gaps} == { + path.parent.parent.name for path in unmerged + } + assert all(item["provenance"]["attempt"] == 1 for item in gaps) + + +def test_budget_accepts_complete_reducer_without_optional_complete_flag( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + _, accepted, _ = accept_reducer(workbench_db, scan) + document = json.loads(accepted.read_bytes()) + del document["complete"] + accepted.write_text(json.dumps(document)) + digest = hashlib.sha256(accepted.read_bytes()).hexdigest() + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute("UPDATE deep_scan_attempts SET accepted_result_sha256 = ?", (digest,)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', workflow_version = 'deep-security-scan/v2'" + ) + workbench_api["complete_budget_exhausted_scan"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message="Original cost stop." + ), + ) + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "complete" + assert hashlib.sha256(accepted.read_bytes()).hexdigest() == digest + assert len(json.loads((scan.scan_dir / "findings.json").read_text())["findings"]) == 1 + + +def test_budget_preserves_accepted_scope_and_unmerged_scope_limits( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + _, accepted, _ = accept_reducer(workbench_db, scan) + unmerged = accept_unmerged(workbench_db, scan) + scope = { + "summary": "Synthetic upload boundaries.", + "artifactsReviewed": ["subdir/extract.py"], + "runtimeStatus": "Synthetic runtime unavailable.", + "validationMode": "Static validation of the accepted findings.", + "context": "External deployment behavior remains unresolved.", + "limitations": ["The accepted review did not execute the deployment integration."], + } + unmerged_limit = "A separate unmerged review could not inspect the deployment credentials." + for path, value in [(accepted, scope), (unmerged, {"limitations": [unmerged_limit]})]: + document = json.loads(path.read_bytes()) + document["scope"] = value + path.write_text(json.dumps(document)) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_attempts SET accepted_result_sha256 = ? WHERE accepted_result_path = ?", + (hashlib.sha256(path.read_bytes()).hexdigest(), str(path)), + ) + originals = {p: p.read_bytes() for p in [accepted, unmerged]} + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', workflow_version = 'deep-security-scan/v2'" + ) + warning = "Scan stopped at its original cost limit before further validation." + workbench_api["complete_budget_exhausted_scan"]( + workbench_db, + Namespace(scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=warning), + ) + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text()) + saved_scope = manifest["scan"]["scope"] + for key, value in scope.items(): + if key == "limitations": + assert all(item in saved_scope[key] for item in value) + else: + assert saved_scope[key] == value + assert warning in saved_scope["limitations"] + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert coverage["completeness"] == "partial" + assert any( + item["reason"] == unmerged_limit + and item["provenance"]["workerId"] == unmerged.parent.parent.name + for item in coverage["deferred"] + ) + assert all(p.read_bytes() == contents for p, contents in originals.items()) + assert len(json.loads((scan.scan_dir / "findings.json").read_text())["findings"]) == 1 + + +@pytest.mark.parametrize("has_reducer", [False, True]) +def test_budget_replay_after_draft_commit_keeps_unmerged_coverage_once( + workbench_api, workbench_db, publication_scan, tmp_path, monkeypatch, has_reducer +): + scan = publication_scan() + if has_reducer: + accept_reducer(workbench_db, scan) + unmerged = accept_unmerged(workbench_db, scan) + original = unmerged.read_bytes() + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', workflow_version = 'deep-security-scan/v2'" + ) + budget = workbench_api["complete_budget_exhausted_scan"] + original_complete = budget.__globals__["complete_scan_locked"] + + def fail_before_seal(*args, **kwargs): + raise RuntimeError("Synthetic process loss after budget draft commit.") + + monkeypatch.setitem(budget.__globals__, "complete_scan_locked", fail_before_seal) + args = Namespace( + scan_id=scan.scan_id, + cost_json=json.dumps(BUDGET_COST), + message="Original cost interruption.", + ) + path = tmp_path / "budget-gap.sqlite3" + with sqlite3.connect(path) as connection: + workbench_db.backup(connection) + connection.row_factory = sqlite3.Row + with pytest.raises(RuntimeError, match="after budget draft commit"): + budget(connection, args) + run = connection.execute("SELECT * FROM deep_scan_runs").fetchone() + assert run["status"] == "succeeded" + selection = run["finalization_input_json"] + assert connection.execute("SELECT status FROM scans").fetchone()[0] == "running" + before = json.loads((scan.scan_dir / "coverage.json").read_text()) + monkeypatch.setitem(budget.__globals__, "complete_scan_locked", original_complete) + with sqlite3.connect(path) as connection: + connection.row_factory = sqlite3.Row + budget(connection, args) + assert connection.execute("SELECT status FROM scans").fetchone()[0] == "complete" + assert ( + connection.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + == selection + ) + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + for key in ("surfaces", "explicitExclusions", "deferred", "reviews"): + assert coverage.get(key, []) == before.get(key, []) + assert len({row["id"] for row in coverage["deferred"]}) == len(coverage["deferred"]) + assert unmerged.read_bytes() == original + + +@pytest.mark.parametrize("selected", [False, True], ids=["legacy-v1", "selected-v2"]) +@pytest.mark.parametrize("reason", ["saturated", "capped"]) +@pytest.mark.parametrize("cancel_first", [False, True], ids=["budget-first", "cancel-first"]) +def test_budget_completion_and_cancel_keep_the_committed_outcome( + workbench_api, workbench_db, publication_scan, tmp_path, selected, reason, cancel_first +): + scan = publication_scan() + _, accepted, coverage = accept_reducer(workbench_db, scan) + selection = saved_selection(workbench_db, scan, accepted, reason=reason) if selected else None + scan.coverage = coverage + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute("UPDATE deep_scan_runs SET terminal_reason = ?", (reason,)) + staged = stage_publication( + scan, generation=3, result_path=accepted, title="Selected accepted aggregate" + ) + database_path = tmp_path / "budget-publication.sqlite3" + with sqlite3.connect(database_path) as connection: + workbench_db.backup(connection) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, staged + ) + accepted_bytes = accepted.read_bytes() + warning = "Scan stopped after reaching its configured cost limit." + budget_args = Namespace( + scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=warning + ) + cancel_args = Namespace(scan_id=scan.scan_id, thread_id=None) + if cancel_first: + workbench_api["saved_results"].cancel_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, cancel_args + ) + frozen = published_bytes(scan) + with pytest.raises(SystemExit, match="running"): + workbench_api["complete_budget_exhausted_scan"](connection, budget_args) + else: + workbench_api["complete_budget_exhausted_scan"](connection, budget_args) + frozen = published_bytes(scan) + with pytest.raises(SystemExit, match="running"): + workbench_api["saved_results"].cancel_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, cancel_args + ) + assert published_bytes(scan) == frozen + + with sqlite3.connect(database_path) as connection: + connection.row_factory = sqlite3.Row + row = connection.execute("SELECT * FROM scans").fetchone() + run = connection.execute("SELECT * FROM deep_scan_runs").fetchone() + assert row["status"] == ("failed" if cancel_first else "complete") + assert bool(row["canceled_at"]) == cancel_first + assert run["terminal_reason"] == reason + recorded = json.loads(run["finalization_input_json"] or "null") + if selection is None: + assert recorded is None + else: + assert {key: recorded[key] for key in selection} == selection + assert len(recorded["publicationSha256"]) == 64 + assert accepted.read_bytes() == accepted_bytes + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert coverage["completeness"] == "partial" + assert any(item["id"] == "accepted-follow-up" for item in coverage["deferred"]) + if not cancel_first: + assert any(item["id"] == "scan-cost-limit" for item in coverage["deferred"]) + assert warning in json.loads(row["completion_warnings_json"]) + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == 1 + with pytest.raises(SystemExit, match="stopped"): + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, staged + ) + assert published_bytes(scan) == frozen + + +@pytest.mark.parametrize( + "cut", ["after-findings", "after-coverage", "after-draft-commit", "after-seal"] +) +@pytest.mark.parametrize("has_reducer", [False, True]) +def test_budget_process_death_replays_exact_evidence( + workbench_api, workbench_db, publication_scan, tmp_path, cut, has_reducer +): + scan = publication_scan() + thread_id = "a23e657b-c14c-4da7-bd20-baa9e7579390" + workbench_api["set_scan_thread"]( + workbench_db, Namespace(scan_id=scan.scan_id, thread_id=thread_id) + ) + accepted = None + if has_reducer: + _, accepted, _ = accept_reducer(workbench_db, scan) + unmerged = accept_unmerged(workbench_db, scan) + originals = {str(p): p.read_bytes() for p in [unmerged, *([accepted] if accepted else [])]} + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status='running', workflow_version='deep-security-scan/v2'" + ) + database = tmp_path / "budget-process-cut.sqlite3" + with sqlite3.connect(database) as db: + workbench_db.backup(db) + budget = workbench_api["complete_budget_exhausted_scan"] + args = Namespace( + scan_id=scan.scan_id, + cost_json=json.dumps(BUDGET_COST), + message="Original synthetic cost stop.", + ) + marker = tmp_path / "crash-marker.json" + child_source = """ +from argparse import Namespace +from pathlib import Path +import json +import os +import runpy +import sqlite3 +import sys + +script, database, scan_id, cost_json, warning, cut, marker = sys.argv[1:] +api = runpy.run_path(str(script), run_name='fault_review_budget_child') +budget = api['complete_budget_exhausted_scan'] +deep = budget.__globals__['deep_scan'] +from types import SimpleNamespace +deep.configure(SimpleNamespace(**{**api, "preserve_stopped_results": api["preserve_stopped_results_after_transition"]})) + + +def terminate(stage): + Path(marker).write_text(json.dumps({'stage': stage, 'pid': os.getpid()})) + os._exit(86) + + +if cut == 'after-draft-commit': + budget.__globals__['complete_scan_locked'] = lambda *a, **kw: terminate(cut) +elif cut == 'after-seal': + original = budget.__globals__['_write_prepared_scan_finalization'] + + def seal_then_die(prepared, **kwargs): + original(prepared, **kwargs) + terminate(cut) + + budget.__globals__['_write_prepared_scan_finalization'] = seal_then_die +else: + saved = budget.__globals__['saved_results'] + original = saved.write_scan_local_bytes + + def write_then_die(scan_dir, name, contents): + result = original(scan_dir, name, contents) + if name == ('findings.json' if cut == 'after-findings' else 'coverage.json'): + terminate(cut) + return result + + saved.write_scan_local_bytes = write_then_die + +with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + connection.execute('PRAGMA foreign_keys=ON') + budget(connection, Namespace(scan_id=scan_id, cost_json=cost_json, message=warning)) +os._exit(87) +""" + child = subprocess.run( + [ + sys.executable, + "-c", + child_source, + workbench_api["__file__"], + str(database), + scan.scan_id, + args.cost_json, + args.message, + cut, + str(marker), + ], + capture_output=True, + text=True, + ) + assert child.returncode == 86, (child.returncode, child.stdout, child.stderr) + assert json.loads(marker.read_text())["stage"] == cut + before = { + name: (scan.scan_dir / name).exists() + for name in ("findings.json", "coverage.json", "scan-manifest.json") + } + sealed_bytes = published_bytes(scan) if cut == "after-seal" else None + if sealed_bytes is not None: + assert json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]["sealedAt"] + with sqlite3.connect(database) as db: + db.row_factory = sqlite3.Row + db.execute("PRAGMA foreign_keys=ON") + run = dict(db.execute("SELECT * FROM deep_scan_runs").fetchone()) + snapshot = { + "cut": cut, + "hasReducer": has_reducer, + "database": str(database), + "scanDir": str(scan.scan_dir), + "beforeFiles": before, + "beforeStatus": run["status"], + "beforeSelection": run["finalization_input_json"], + } + saved_scan = workbench_api["require_scan"](db, scan.scan_id) + resumed = workbench_api["scan_history"].cli_scan_resume( + db, + saved_scan, + workbench_api["require_workspace"](db, saved_scan["workspace_id"]), + **{ + name: workbench_api[source] + for name, source in { + "parse_scan_recipe": "parse_scan_recipe", + "scan_contract": "scan_contract", + "require_scan_directory": "require_canonical_scan_directory", + "artifact_path": "artifact_path", + "read_json_object": "read_json_object", + "workbench_completion_binding": "workbench_completion_binding", + }.items() + }, + ) + assert resumed["scanId"] == scan.scan_id + assert resumed["threadId"] == thread_id + try: + budget(db, args) + except BaseException as error: + snapshot["replayError"] = str(error) + (tmp_path / "crash-result.json").write_text(json.dumps(snapshot, indent=2)) + raise + snapshot["afterStatus"] = db.execute("SELECT status FROM scans").fetchone()[0] + snapshot["afterSelection"] = db.execute( + "SELECT finalization_input_json FROM deep_scan_runs" + ).fetchone()[0] + (tmp_path / "crash-result.json").write_text(json.dumps(snapshot, indent=2)) + assert snapshot["afterStatus"] == "complete" + if sealed_bytes is not None: + assert published_bytes(scan) == sealed_bytes + if run["finalization_input_json"] is not None: + assert snapshot["afterSelection"] == run["finalization_input_json"] + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert len(findings) == int(has_reducer) + assert coverage["completeness"] == "partial" + assert len({x["id"] for x in coverage["deferred"]}) == len(coverage["deferred"]) + assert all(Path(path).read_bytes() == data for path, data in originals.items()) + + +@pytest.mark.parametrize("files", ["findings", "coverage", "manifest", "findings-and-coverage"]) +def test_budget_rejects_unrelated_incomplete_drafts( + workbench_api, workbench_db, publication_scan, files +): + scan = publication_scan() + accept_unmerged(workbench_db, scan) + keep = { + "findings": {"findings.json"}, + "coverage": {"coverage.json"}, + "manifest": {"scan-manifest.json"}, + "findings-and-coverage": {"findings.json", "coverage.json"}, + }[files] + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + if name not in keep: + (scan.scan_dir / name).unlink() + before = {name: (scan.scan_dir / name).read_bytes() for name in keep} + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', workflow_version = 'deep-security-scan/v2'" + ) + with pytest.raises(SystemExit, match="incomplete canonical scan draft"): + workbench_api["complete_budget_exhausted_scan"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, + cost_json=json.dumps(BUDGET_COST), + message="Original cost stop.", + ), + ) + assert {name: (scan.scan_dir / name).read_bytes() for name in keep} == before + assert all( + not (scan.scan_dir / name).exists() + for name in ("scan-manifest.json", "findings.json", "coverage.json") + if name not in keep + ) + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "running" + + +@pytest.mark.parametrize( + "state", + ["changed-findings", "wrong-scan", "running-discovery", "canceled", "other-owner", "complete"], +) +def test_budget_sealed_replay_keeps_integrity_and_ownership_guards( + workbench_api, workbench_db, publication_scan, monkeypatch, state +): + scan = publication_scan() + accept_reducer(workbench_db, scan) + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', workflow_version = 'deep-security-scan/v2'" + ) + budget = workbench_api["complete_budget_exhausted_scan"] + complete = budget.__globals__["complete_scan_locked"] + args = Namespace( + scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message="Original cost stop." + ) + with monkeypatch.context() as patch: + patch.setitem( + budget.__globals__, + "complete_scan_locked", + lambda *a: complete(*a, prepare_only=True), + ) + budget(workbench_db, args) + manifest = scan.scan_dir / "scan-manifest.json" + assert json.loads(manifest.read_text())["scan"]["sealedAt"] + if state in {"changed-findings", "wrong-scan"}: + path = scan.scan_dir / ( + "findings.json" if state == "changed-findings" else "scan-manifest.json" + ) + document = json.loads(path.read_text()) + if state == "changed-findings": + document["findings"] = [] + else: + document["scan"]["id"] = "69078890-d24c-4416-a6fa-c286825bef88" + path.write_text(json.dumps(document)) + elif state == "running-discovery": + with workbench_db: + workbench_db.execute("UPDATE deep_scan_runs SET status = 'running'") + elif state == "canceled": + workbench_api["saved_results"].cancel_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace(scan_id=scan.scan_id, thread_id=None), + ) + elif state == "other-owner": + with workbench_db: + workbench_db.execute( + "UPDATE scans SET handoff_claim_token = 'a3292ae4-9b47-430f-8ed6-73ff73db575c'" + ) + else: + complete(workbench_db, scan.scan_id, None, args.cost_json) + before_files = published_bytes(scan) + before_scan = dict(workbench_db.execute("SELECT * FROM scans").fetchone()) + before_run = dict(workbench_db.execute("SELECT * FROM deep_scan_runs").fetchone()) + with pytest.raises(SystemExit): + budget(workbench_db, args) + assert published_bytes(scan) == before_files + assert dict(workbench_db.execute("SELECT * FROM scans").fetchone()) == before_scan + assert dict(workbench_db.execute("SELECT * FROM deep_scan_runs").fetchone()) == before_run + + +@pytest.mark.parametrize( + "state", ["parent-canceled", "stopping", "failed", "canceled", "unselected"] +) +def test_budget_resume_keeps_explicit_stop_and_unselected_guards( + workbench_api, workbench_db, publication_scan, state +): + scan = publication_scan() + workbench_api["set_scan_thread"]( + workbench_db, + Namespace(scan_id=scan.scan_id, thread_id="a23e657b-c14c-4da7-bd20-baa9e7579390"), + ) + with workbench_db: + if state == "parent-canceled": + workbench_db.execute("UPDATE scans SET canceled_at = ?", (scan.timestamp,)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = ?, cancel_requested = 1, finalization_input_json = NULL", + ( + "running" + if state == "stopping" + else state + if state in {"failed", "canceled"} + else "succeeded", + ), + ) + saved_scan = workbench_api["require_scan"](workbench_db, scan.scan_id) + with pytest.raises(SystemExit, match="cannot resume"): + workbench_api["scan_history"].cli_scan_resume( + workbench_db, + saved_scan, + workbench_api["require_workspace"](workbench_db, saved_scan["workspace_id"]), + **{ + name: workbench_api[source] + for name, source in { + "parse_scan_recipe": "parse_scan_recipe", + "scan_contract": "scan_contract", + "require_scan_directory": "require_canonical_scan_directory", + "artifact_path": "artifact_path", + "read_json_object": "read_json_object", + "workbench_completion_binding": "workbench_completion_binding", + }.items() + }, + ) + + +@pytest.mark.parametrize("changed", [False, True], ids=["unchanged", "changed-publication"]) +def test_budget_selection_binds_publication_in_its_commit( + workbench_api, workbench_db, publication_scan, tmp_path, changed +): + scan = publication_scan() + accept_reducer(workbench_db, scan) + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'discovery', " + "workflow_version = 'deep-security-scan/v2', finalization_input_json = NULL" + ) + database = tmp_path / "budget-commit.sqlite3" + with sqlite3.connect(database) as connection: + workbench_db.backup(connection) + child_program = """ +import json, os, runpy, sqlite3, sys +from argparse import Namespace +from types import SimpleNamespace +api = runpy.run_path(sys.argv[1], run_name="budget_publication_commit_loss") +api["deep_scan"].configure(SimpleNamespace(**{**api, "preserve_stopped_results": api["preserve_stopped_results_after_transition"]})) +class CrashConnection(sqlite3.Connection): + def commit(self): + super().commit() + os._exit(73) +connection = sqlite3.connect(sys.argv[2], factory=CrashConnection) +connection.row_factory = sqlite3.Row +connection.execute("PRAGMA foreign_keys = ON") +api["complete_budget_exhausted_scan"](connection, Namespace(scan_id=sys.argv[3], cost_json=sys.argv[4], message=None)) +raise AssertionError("budget selection did not commit") +""" + child = subprocess.run( + [ + sys.executable, + "-c", + child_program, + workbench_api["__file__"], + str(database), + scan.scan_id, + json.dumps(BUDGET_COST), + ], + capture_output=True, + text=True, + ) + assert child.returncode == 73, child.stderr + findings_path = scan.scan_dir / "findings.json" + findings = json.loads(findings_path.read_text()) + assert findings["findings"] + if changed: + findings["findings"][0]["title"] = "Synthetic substituted publication finding" + findings_path.write_text(json.dumps(findings)) + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + args = Namespace(scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=None) + if changed: + with pytest.raises(SystemExit, match="selected Deep Scan publication"): + workbench_api["complete_budget_exhausted_scan"](connection, args) + assert connection.execute("SELECT status FROM scans").fetchone()[0] == "running" + else: + completed = workbench_api["complete_budget_exhausted_scan"](connection, args) + assert completed["scan"]["scanId"] == scan.scan_id + assert connection.execute("SELECT status FROM scans").fetchone()[0] == "complete" diff --git a/plugins/codex-security/tests/test_checkpoint_publication_authority.py b/plugins/codex-security/tests/test_checkpoint_publication_authority.py new file mode 100644 index 0000000000..1d4d17cf51 --- /dev/null +++ b/plugins/codex-security/tests/test_checkpoint_publication_authority.py @@ -0,0 +1,426 @@ +from __future__ import annotations + +import copy +import json +import os +import uuid +from argparse import Namespace + +import pytest +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_workbench_standard_deep_results import deep_scan_fixture, worker_paths +from workbench_test_support import run_workbench, write_checkpoint, write_completed_contract + + +def test_public_stop_retains_accepted_partial_evidence_and_newer_rejection(tmp_path): + state, home, target, scan_dir, scan_id = deep_scan_fixture(tmp_path, workers=2) + environment = {"CODEX_HOME": str(home)} + contract = tmp_path / "contract" + contract.mkdir() + write_completed_contract(contract, scan_id, target, relative_path="app.py") + finding = json.loads((contract / "findings.json").read_text())["findings"][0] + deferred = { + "candidateId": "pending-query", + "reason": "Validation is pending.", + "paths": ["app.py"], + } + coverage = { + "completeness": "partial", + "surfaces": [], + "explicitExclusions": [], + "deferred": [deferred], + } + workers = [] + for accepted in (True, False): + name = "accepted" if accepted else "interrupted" + prompt, output, result = worker_paths(scan_dir, name) + worker_id = str(uuid.uuid4()) + worker_args = ( + "upsert-deep-scan-worker", + "--scan-id", + scan_id, + "--worker-id", + worker_id, + "--kind", + "discovery", + "--prompt-path", + str(prompt), + "--artifact-dir", + str(output), + "--attempt", + "1", + ) + run_workbench(state, *worker_args, "--status", "running", environment=environment) + current = copy.deepcopy(finding) + current["identity"]["anchor"] = name + current["extensions"] = {"candidateId": name} + draft = {"scanId": scan_id, "complete": True, "findings": [current], "coverage": coverage} + result.write_text(json.dumps(draft)) + write_checkpoint(output / "checkpoints", draft) + if accepted: + run_workbench( + state, + *worker_args, + "--status", + "succeeded", + "--result-manifest-path", + str(result), + environment=environment, + ) + else: + rejected = { + **draft, + "complete": False, + "findings": [], + "coverage": { + **coverage, + "surfaces": [ + { + "candidateId": name, + "label": "Reviewed candidate", + "disposition": "rejected", + "receiptRefs": [], + } + ], + }, + } + head = write_checkpoint(output / "checkpoints", rejected) + (output / "checkpoint-head.json").write_text(json.dumps({"checkpoint": head.name})) + workers.append(worker_id) + + stopped = run_workbench( + state, + "fail-deep-scan", + "--scan-id", + scan_id, + "--message", + "Original worker failure.", + "--deep-status", + "interrupted", + environment=environment, + )["deepScan"] + assert stopped["status"] == "interrupted" + scan = run_workbench(state, "get-scan", "--scan-id", scan_id)["scan"] + assert scan["findingCount"] == 1 + findings = json.loads((scan_dir / "findings.json").read_text())["findings"] + assert [item["identity"]["anchor"] for item in findings] == ["accepted"] + retained_coverage = json.loads((scan_dir / "coverage.json").read_text()) + assert retained_coverage["completeness"] == "partial" + assert any(item.get("candidateId") == "pending-query" for item in retained_coverage["deferred"]) + assert any( + item.get("candidateId") == "interrupted" and item["disposition"] == "rejected" + for item in retained_coverage["surfaces"] + ) + assert scan["failureMessage"] == "Original worker failure." + assert {worker["id"]: worker["status"] for worker in stopped["workers"]} == { + workers[0]: "succeeded", + workers[1]: "canceled", + } + + +@pytest.mark.parametrize("archived", [False, True], ids=["current", "archived"]) +@pytest.mark.parametrize("has_head", [True, False], ids=["committed-head", "legacy"]) +@pytest.mark.parametrize("complete", [False, True], ids=["checkpoint", "complete"]) +def test_recovery_honors_rejection_committed_before_result_replacement( + workbench_api, workbench_db, publication_scan, archived, has_head, complete +): + scan = publication_scan() + provisional = copy.deepcopy(scan.findings[0]) + provisional["extensions"] = {"candidateId": "candidate-rejected"} + retained = copy.deepcopy(scan.findings[0]) + retained["identity"]["anchor"] = "independent-finding" + retained["extensions"] = {"candidateId": "candidate-retained"} + retained["locations"][0]["startLine"] = 20 + retained["locations"][0]["endLine"] = 21 + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + result_path = add_worker(workbench_db, scan, status="canceled") + if archived: + result_path = result_path.parent / "attempts" / "attempt-1" / "result.json" + result_path.parent.mkdir(parents=True) + previous = { + "scanId": scan.scan_id, + "complete": True, + "findings": [provisional, retained], + "coverage": scan.coverage, + } + result_path.write_text(json.dumps(previous)) + old_checkpoint = write_checkpoint(result_path.parent / "checkpoints", previous) + rejected = { + **previous, + "complete": complete, + "findings": [retained], + "coverage": { + **scan.coverage, + "surfaces": [ + { + "candidateId": "candidate-rejected", + "label": "Validated candidate disposition", + "disposition": "rejected", + "receiptRefs": [], + } + ], + }, + } + checkpoint = write_checkpoint(result_path.parent / "checkpoints", rejected) + if has_head: + head = result_path.parent / "checkpoint-head.json" + head.write_text(json.dumps({"checkpoint": checkpoint.name})) + observed = result_path.stat().st_mtime_ns + 1_000_000_000 + os.utime(head, ns=(observed, observed)) + assert head.stat().st_mtime_ns > result_path.stat().st_mtime_ns + saved_bytes = {path: path.read_bytes() for path in (result_path, old_checkpoint, checkpoint)} + + stopped = workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."), + )["scan"] + + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert stopped["findingCount"] == len(findings) == (1 if has_head else 2) + assert any(finding["identity"]["anchor"] == "independent-finding" for finding in findings) + assert all(path.read_bytes() == contents for path, contents in saved_bytes.items()) + if has_head: + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert any( + surface.get("candidateId") == "candidate-rejected" + and surface.get("disposition") == "rejected" + for surface in coverage["surfaces"] + ) + + +def save_disposition(scan, directory, disposition): + directory.mkdir(parents=True, exist_ok=True) + finding = copy.deepcopy(scan.findings[0]) + finding["extensions"] = {"candidateId": "candidate-disposition"} + draft = { + "scanId": scan.scan_id, + "complete": True, + "findings": [finding] if disposition == "reported" else [], + "coverage": { + **scan.coverage, + "surfaces": [ + { + "candidateId": "candidate-disposition", + "label": "Validated candidate disposition", + "disposition": disposition, + "receiptRefs": [], + } + ], + }, + } + checkpoint = write_checkpoint(directory / "checkpoints", draft) + (directory / "checkpoint-head.json").write_text(json.dumps({"checkpoint": checkpoint.name})) + return draft + + +@pytest.mark.parametrize("archived", [False, True], ids=["current-head", "newer-archive"]) +@pytest.mark.parametrize("disposition", ["reported", "rejected"]) +def test_newer_checkpoint_disposition_precedes_older_archived_head( + workbench_api, workbench_db, publication_scan, archived, disposition +): + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + result = add_worker(workbench_db, scan, status="canceled") + old = result.parent / "attempts" / "attempt-2" + save_disposition(scan, old, "rejected" if disposition == "reported" else "reported") + current = result.parent / "attempts" / "attempt-10" if archived else result.parent + if not archived: + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET attempt = 3 WHERE scan_id = ?", (scan.scan_id,) + ) + draft = save_disposition(scan, current, disposition) + (current / "result.json").write_text(json.dumps(draft)) + + stopped = workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."), + )["scan"] + + assert stopped["findingCount"] == (1 if disposition == "reported" else 0) + + +@pytest.mark.parametrize("head_change", ["replaced", "removed", "missing-checkpoint"]) +def test_frozen_stopped_replay_ignores_later_worker_head_changes( + workbench_api, workbench_db, publication_scan, monkeypatch, head_change +): + import finalize_scan_contract + + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + result = add_worker(workbench_db, scan, status="canceled") + previous = save_disposition(scan, result.parent, "reported") + result.write_text(json.dumps(previous)) + save_disposition(scan, result.parent, "rejected") + # The rejected head was committed after the result, before its replacement failed. + head = result.parent / "checkpoint-head.json" + observed = result.stat().st_mtime_ns + 1_000_000_000 + os.utime(head, ns=(observed, observed)) + assert head.stat().st_mtime_ns > result.stat().st_mtime_ns + checkpoint_name = json.loads(head.read_text())["checkpoint"] + directory = result.parent.relative_to(scan.scan_dir).as_posix() + expected_heads = {directory: f"{directory}/checkpoints/{checkpoint_name}"} + original_outputs = { + name: (scan.scan_dir / name).read_bytes() + for name in ("findings.json", "coverage.json", "scan-manifest.json") + } + write_bytes = finalize_scan_contract.write_scan_local_bytes + failed_writes = [] + + def fail_coverage_write(directory, relative, payload, **kwargs): + if relative != "coverage.json" or failed_writes: + return write_bytes(directory, relative, payload, **kwargs) + # Exercise the real writer after findings have reached disk. Remove the + # temporary obstruction before the publisher restores its old outputs. + failed_writes.append(json.loads((directory / "findings.json").read_text())) + path = directory / relative + previous_bytes = path.read_bytes() + path.unlink() + path.mkdir() + try: + return write_bytes(directory, relative, payload, **kwargs) + finally: + path.rmdir() + path.write_bytes(previous_bytes) + + with monkeypatch.context() as patch: + patch.setattr(finalize_scan_contract, "write_scan_local_bytes", fail_coverage_write) + workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped." + ), + ) + row = workbench_db.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone() + assert len(failed_writes) == 1 + assert "scanId" in failed_writes[0] + assert row["status"] == "failed" + assert row["failure_message"] == "Audit stopped." + assert row["retained_source_digests_json"] + assert row["seal_manifest_digest"] is None + assert all( + (scan.scan_dir / name).read_bytes() == contents + for name, contents in original_outputs.items() + ) + original_sources = row["retained_source_digests_json"] + original_run = dict( + workbench_db.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,) + ).fetchone() + ) + head = result.parent / "checkpoint-head.json" + if head_change == "replaced": + save_disposition(scan, result.parent, "reported") + elif head_change == "removed": + head.unlink() + else: + head.write_text(json.dumps({"checkpoint": "a" * 64 + ".json"})) + + replayed = workbench_api["saved_results"].preserve_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None + ), + )["scan"] + + assert replayed["findingCount"] == 0 + assert json.loads((scan.scan_dir / "findings.json").read_text())["findings"] == [] + assert json.loads(result.read_text()) == previous + row = workbench_db.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone() + assert row["failure_message"] == "Audit stopped." + assert row["retained_source_digests_json"] == original_sources + assert json.loads(row["retained_checkpoint_heads_json"]) == expected_heads + assert row["seal_manifest_digest"] + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text()) + assert manifest["scan"]["preservedCheckpointHeads"] == expected_heads + assert ( + dict( + workbench_db.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,) + ).fetchone() + ) + == original_run + ) + + +def test_explicit_recovery_observes_head_change_between_existing_checkpoints( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + result = add_worker(workbench_db, scan, status="canceled") + previous = save_disposition(scan, result.parent, "reported") + result.write_text(json.dumps(previous)) + save_disposition(scan, result.parent, "rejected") + head = result.parent / "checkpoint-head.json" + rejected_at = result.stat().st_mtime_ns + 1_000_000_000 + os.utime(head, ns=(rejected_at, rejected_at)) + assert head.stat().st_mtime_ns > result.stat().st_mtime_ns + stopped = workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."), + )["scan"] + assert stopped["findingCount"] == 0 + + save_disposition(scan, result.parent, "reported") + reported_at = rejected_at + 1_000_000_000 + os.utime(head, ns=(reported_at, reported_at)) + assert head.stat().st_mtime_ns > rejected_at + + context = workbench_api["scan_context"](workbench_db, scan.scan_id)["scan"] + assert context["resultsRecoveryNeeded"] is True + recovered = workbench_api["saved_results"].recover_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id) + )["scan"] + assert recovered["findingCount"] == 1 + assert recovered["resultsRecoveryNeeded"] is False + + +def test_legacy_frozen_publication_keeps_result_fallback_without_saved_heads( + workbench_api, workbench_db, publication_scan, monkeypatch +): + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + result = add_worker(workbench_db, scan, status="canceled") + previous = save_disposition(scan, result.parent, "reported") + result.write_text(json.dumps(previous)) + save_disposition(scan, result.parent, "rejected") + (result.parent / "checkpoint-head.json").unlink() + + def fail_before_publication(*args, **kwargs): + raise OSError("Synthetic publication interruption") + + with monkeypatch.context() as patch: + patch.setattr( + workbench_api["saved_results"], + "_write_prepared_scan_finalization", + fail_before_publication, + ) + workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped." + ), + ) + with workbench_db: + workbench_db.execute( + "UPDATE scans SET retained_checkpoint_heads_json = NULL WHERE id = ?", (scan.scan_id,) + ) + save_disposition(scan, result.parent, "rejected") + + replayed = workbench_api["saved_results"].preserve_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None + ), + )["scan"] + + assert replayed["findingCount"] == 1 diff --git a/plugins/codex-security/tests/test_checkpoint_schema.py b/plugins/codex-security/tests/test_checkpoint_schema.py new file mode 100644 index 0000000000..bb471267cb --- /dev/null +++ b/plugins/codex-security/tests/test_checkpoint_schema.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import json +import sqlite3 +from contextlib import closing + +import pytest + + +@pytest.mark.parametrize("upgrade", [False, True], ids=["fresh", "upgrade"]) +def test_frozen_checkpoint_head_migration_preserves_scan_state(workbench_api, upgrade): + migrations = workbench_api["MIGRATIONS"] + timestamp = "2026-07-01T00:00:00Z" + + def migrate(connection, selected): + workbench_api["apply_schema_migrations"]( + connection, selected, lambda: timestamp, workbench_api["backfill_security_targets"] + ) + + with closing(sqlite3.connect(":memory:")) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + migrate( + connection, + tuple(item for item in migrations if item[0] != 47) if upgrade else migrations, + ) + connection.execute( + "INSERT INTO workspaces (id, created_at, updated_at) VALUES (?, ?, ?)", + ("workspace", timestamp, timestamp), + ) + for status in ("running", "failed"): + connection.execute( + "INSERT INTO scans (id, workspace_id, target_path, target_revision, scope, " + "mode, scan_dir, status, phase, started_at, created_at, updated_at, " + "failure_message, retained_source_digests_json) " + "VALUES (?, 'workspace', 'target', 'revision', '.', 'deep', ?, ?, " + "'discovery', ?, ?, ?, ?, ?)", + ( + status, + f"scans/{status}", + status, + timestamp, + timestamp, + timestamp, + "Original stop reason." if status == "failed" else None, + json.dumps({"workers/review/result.json": "b" * 64}) + if status == "failed" + else None, + ), + ) + before = [dict(row) for row in connection.execute("SELECT * FROM scans ORDER BY id")] + migrate(connection, migrations) + after = [dict(row) for row in connection.execute("SELECT * FROM scans ORDER BY id")] + for original, updated in zip(before, after, strict=True): + assert updated.pop("retained_checkpoint_heads_json") is None + original.pop("retained_checkpoint_heads_json", None) + assert updated == original + + heads = json.dumps({"workers/review": "workers/review/checkpoints/" + "a" * 64 + ".json"}) + connection.execute( + "UPDATE scans SET retained_checkpoint_heads_json = ? WHERE id = 'failed'", (heads,) + ) + migrate(connection, migrations) + assert ( + connection.execute( + "SELECT retained_checkpoint_heads_json FROM scans WHERE id = 'failed'" + ).fetchone()[0] + == heads + ) + assert ( + connection.execute( + "SELECT retained_checkpoint_heads_json FROM scans WHERE id = 'running'" + ).fetchone()[0] + is None + ) + assert [ + tuple(row) + for row in connection.execute( + "SELECT version, name FROM schema_migrations WHERE version = 47" + ) + ] == [(47, "freeze stopped scan checkpoint selections")] + assert connection.execute("PRAGMA foreign_key_check").fetchall() == [] diff --git a/plugins/codex-security/tests/test_deep_scan_compatibility.py b/plugins/codex-security/tests/test_deep_scan_compatibility.py new file mode 100644 index 0000000000..57f1a80901 --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_compatibility.py @@ -0,0 +1,385 @@ +"""Compatibility checks use real persisted scans and preserve unsupported state.""" + +from __future__ import annotations + +import json +import os +import sqlite3 +import subprocess +import sys +import uuid +from pathlib import Path + +import pytest +from workbench_test_support import SCRIPT, run_workbench + + +def snapshot(state_dir: Path) -> str: + with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: + return "\n".join(connection.iterdump()) + + +def claim_requiring_original_settings( + state: Path, scan_id: str +) -> subprocess.CompletedProcess[str]: + # Exercise the private MCP requirement against both the parent and fixed + # workbench, without adding a public CLI argument. + return subprocess.run( + [ + sys.executable, + "-c", + "\n".join( + [ + "import runpy, sys", + "script = sys.argv.pop(1)", + "main = runpy.run_path(script)['main']", + "namespace = main.__globals__", + "parse = namespace['parse_args']", + "def parse_with_requirement(*args, **kwargs):", + " result = parse(*args, **kwargs)", + " result.require_execution_settings = True", + " return result", + "namespace['parse_args'] = parse_with_requirement", + "main()", + ] + ), + str(SCRIPT), + "claim-deep-scan-coordinator", + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + ], + env={**os.environ, "CODEX_SECURITY_STATE_DIR": str(state)}, + capture_output=True, + text=True, + timeout=30, + ) + + +@pytest.mark.parametrize( + "workflow,settings_version", + [ + ("deep-security-scan/v1", 99), + ("deep-scan-mcp/v1", 99), + ("deep-security-scan/v2", 99), + ("deep-security-scan/v2", None), + ], +) +def test_missing_or_unsupported_settings_reject_before_takeover( + tmp_path: Path, workflow: str, settings_version: int | None +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + run = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + "--workflow-version", + workflow, + )["deepScan"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute( + "UPDATE deep_scan_runs SET coordinator_generation = 2, " + "phase = 'discovery', updated_at = '2000-01-01T00:00:00Z'" + ) + settings_path = Path(run["scanDir"]) / "artifacts/deep_discovery/execution-settings.json" + saved = None + if settings_version is not None: + settings_path.parent.mkdir(parents=True, exist_ok=True) + saved = json.dumps( + { + "version": settings_version, + "settings": {"codexPath": "/fixture/codex", "codexHome": "/fixture/home"}, + } + ).encode() + settings_path.write_bytes(saved) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute( + "UPDATE deep_scan_runs SET execution_settings_json = ? WHERE scan_id = ?", + (saved.decode(), run["scanId"]), + ) + before = snapshot(state) + result = claim_requiring_original_settings(state, run["scanId"]) + assert snapshot(state) == before, ( + "settings rejection must precede ownership and worker recovery" + ) + assert result.returncode != 0 + assert "execution settings" in result.stderr + assert (settings_path.read_bytes() if settings_path.exists() else None) == saved + + +@pytest.mark.parametrize("workflow", ["deep-security-scan/v1", "deep-scan-mcp/v1"]) +def test_legacy_takeover_does_not_require_or_create_a_new_snapshot( + tmp_path: Path, workflow: str +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + run = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + "--workflow-version", + workflow, + )["deepScan"] + result = claim_requiring_original_settings(state, run["scanId"]) + assert result.returncode == 0, result.stderr + observed = json.loads(result.stdout)["deepScan"] + assert observed["workflowVersion"] == workflow + assert observed["config"] == run["config"] + assert not (Path(run["scanDir"]) / "artifacts/deep_discovery/execution-settings.json").exists() + + +@pytest.mark.parametrize("completion_only", [False, True]) +def test_observation_and_selected_completion_do_not_require_worker_settings( + tmp_path: Path, completion_only: bool +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + run = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + )["deepScan"] + claim = run_workbench( + state, + "claim-deep-scan-coordinator", + "--scan-id", + run["scanId"], + "--thread-id", + "fixture-thread", + ) + if completion_only: + selection = { + "version": 1, + "resultPath": None, + "resultSha256": None, + "terminalReason": "capped", + "omittedWorkerIds": [], + "selectedAt": run["createdAt"], + } + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute( + "UPDATE deep_scan_runs SET finalization_input_json = ?, " + "updated_at = '2000-01-01T00:00:00Z'", + (json.dumps(selection),), + ) + before = snapshot(state) + result = claim_requiring_original_settings(state, run["scanId"]) + assert result.returncode == 0, result.stderr + observed = json.loads(result.stdout) + if completion_only: + assert observed["coordinatorDisposition"] == "adopted" + assert observed["deepScan"]["finalizationInput"] == selection + else: + assert observed["coordinatorDisposition"] == "observing" + assert ( + observed["deepScan"]["coordinatorGeneration"] + == claim["deepScan"]["coordinatorGeneration"] + ) + assert snapshot(state) == before + + +@pytest.mark.parametrize( + "version", ["deep-security-scan/v1", "deep-scan-mcp/v1", "deep-security-scan/v2"] +) +def test_supported_workflows_keep_their_identity(tmp_path: Path, version: str) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + begun = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + "--workflow-version", + version, + )["deepScan"] + claimed = run_workbench( + state, + "claim-deep-scan-coordinator", + "--scan-id", + str(begun["scanId"]), + "--thread-id", + "fixture-thread", + )["deepScan"] + assert claimed["workflowVersion"] == version + assert claimed["schemaVersion"] == 1 + assert claimed["noNewStreak"] == begun["noNewStreak"] + assert claimed["config"] == begun["config"] + + +@pytest.mark.parametrize("field,value", [("workflow_version", "future/v99")]) +@pytest.mark.parametrize("operation", ["begin", "claim", "handoff"]) +def test_unsupported_execution_does_not_mutate( + tmp_path: Path, + field: str, + value: str | int, + operation: str, +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + begun = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + )["deepScan"] + scan_id = str(begun["scanId"]) + if operation == "claim": + artifact_dir = Path(str(begun["scanDir"])) / "artifacts" / "deep_discovery" / "worker" + artifact_dir.mkdir(parents=True) + prompt = artifact_dir / "prompt.md" + prompt.write_text("Original discovery input") + run_workbench( + state, + "upsert-deep-scan-worker", + "--scan-id", + scan_id, + "--worker-id", + str(uuid.uuid4()), + "--kind", + "discovery", + "--status", + "running", + "--prompt-path", + str(prompt), + "--artifact-dir", + str(artifact_dir), + "--attempt", + "1", + ) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute( + f"UPDATE deep_scan_runs SET {field} = ?, updated_at = ?", + (value, "2000-01-01T00:00:00Z"), + ) + if operation == "handoff": + connection.execute( + "UPDATE scans SET deep_scan_owner_thread_id = NULL, recipe_json = '{}'" + ) + connection.execute("UPDATE workspaces SET thread_id = NULL") + before = snapshot(state) + command = "claim-deep-scan-coordinator" if operation == "claim" else "begin-deep-scan" + result = run_workbench( + state, + command, + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + *(["--model", "observer-model"] if operation != "claim" else []), + check=False, + ) + assert result["returncode"] != 0 + assert "unsupported" in str(result["stderr"]).lower() + assert snapshot(state) == before + if operation != "handoff": + observed = run_workbench( + state, "get-deep-scan", "--scan-id", scan_id, "--thread-id", "fixture-thread" + )["deepScan"] + assert ( + observed["workflowVersion" if field == "workflow_version" else "schemaVersion"] == value + ) + assert snapshot(state) == before + + +@pytest.mark.parametrize("original", [None, "Original discovery context"]) +def test_reader_honors_original_context_when_present(tmp_path: Path, original: str | None) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + begun = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + )["deepScan"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + columns = {row[1] for row in connection.execute("PRAGMA table_info(deep_scan_runs)")} + if "discovery_user_context" not in columns: + connection.execute("ALTER TABLE deep_scan_runs ADD COLUMN discovery_user_context TEXT") + connection.execute("UPDATE deep_scan_runs SET discovery_user_context = ?", (original,)) + connection.execute("UPDATE scans SET user_context = 'Later discussion'") + observed = run_workbench( + state, + "get-deep-scan", + "--scan-id", + str(begun["scanId"]), + "--thread-id", + "fixture-thread", + )["deepScan"] + assert observed["userContext"] == original + + +def test_unsupported_new_workflow_does_not_claim_registered_scan(tmp_path: Path) -> None: + state = tmp_path / "state" + target = tmp_path / "target" + target.mkdir() + scan_dir = tmp_path / "scan" + scan_dir.mkdir(mode=0o700) + registered = run_workbench( + state, + "register-cli-scan", + "--scan-dir", + str(scan_dir), + "--repository", + str(target), + "--registration-json-stdin", + input_text=json.dumps( + { + "recipe": { + "config": {}, + "mode": "deep", + "repository": str(target), + "target": {"kind": "repository", "paths": []}, + } + } + ), + ) + before = snapshot(state) + rejected = run_workbench( + state, + "begin-deep-scan", + "--scan-id", + str(registered["scanId"]), + "--thread-id", + "fixture-thread", + "--workflow-version", + "future/v99", + check=False, + ) + assert rejected["returncode"] != 0 + assert "unsupported" in str(rejected["stderr"]).lower() + assert snapshot(state) == before diff --git a/plugins/codex-security/tests/test_deep_scan_finalization_compatibility.py b/plugins/codex-security/tests/test_deep_scan_finalization_compatibility.py new file mode 100644 index 0000000000..6e1cd3a026 --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_finalization_compatibility.py @@ -0,0 +1,123 @@ +"""Selected finalization survives ownership recovery without restarting discovery.""" + +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path + +import pytest +from workbench_test_support import run_workbench + + +@pytest.mark.parametrize("legacy_version", [None, "deep-security-scan/v1", "deep-scan-mcp/v1"]) +def test_new_workflow_default_preserves_existing_run_version( + tmp_path: Path, legacy_version: str | None +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + created = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + *(["--workflow-version", legacy_version] if legacy_version else []), + )["deepScan"] + expected_version = legacy_version or "deep-security-scan/v2" + assert created["workflowVersion"] == expected_version + resumed = run_workbench( + state, + "begin-deep-scan", + "--scan-id", + created["scanId"], + "--thread-id", + "fixture-thread", + )["deepScan"] + assert resumed["workflowVersion"] == expected_version + assert resumed["createdAt"] == created["createdAt"] + + +def selected_scan(tmp_path: Path, version: int) -> tuple[Path, str, dict[str, object]]: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + run = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + )["deepScan"] + selected = { + "version": version, + "resultPath": None, + "resultSha256": None, + "terminalReason": "capped", + "omittedWorkerIds": [], + "selectedAt": "2000-01-01T00:00:00Z", + } + with sqlite3.connect(state / "workbench.sqlite3") as connection: + columns = {row[1] for row in connection.execute("PRAGMA table_info(deep_scan_runs)")} + if "finalization_input_json" not in columns: + connection.execute("ALTER TABLE deep_scan_runs ADD COLUMN finalization_input_json TEXT") + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "finalization_input_json = ?, phase = 'reducing', " + "created_at = '2000-01-01T00:00:00Z', updated_at = '2000-01-01T00:00:00Z'", + (json.dumps(selected),), + ) + return state, str(run["scanId"]), selected + + +def test_claim_preserves_selected_finalization_without_discovery_recovery(tmp_path: Path) -> None: + state, scan_id, selected = selected_scan(tmp_path, 1) + claimed = run_workbench( + state, + "claim-deep-scan-coordinator", + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + )["deepScan"] + assert claimed["finalizationInput"] == selected + assert claimed["phase"] == "reducing" + assert claimed["coordinatorGeneration"] == 2 + assert claimed["dispatchedCount"] == 0 + assert claimed["createdAt"] == "2000-01-01T00:00:00Z" + + +@pytest.mark.parametrize("command", ["begin-deep-scan", "claim-deep-scan-coordinator"]) +def test_unsupported_selection_rejects_without_mutation(tmp_path: Path, command: str) -> None: + state, scan_id, selected = selected_scan(tmp_path, 99) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + before = "\n".join(connection.iterdump()) + rejected = run_workbench( + state, + command, + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + check=False, + ) + assert rejected["returncode"] != 0 + assert "unsupported" in str(rejected["stderr"]).lower() + with sqlite3.connect(state / "workbench.sqlite3") as connection: + assert "\n".join(connection.iterdump()) == before + observed = run_workbench( + state, + "get-deep-scan", + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + )["deepScan"] + assert observed["finalizationInput"] == selected diff --git a/plugins/codex-security/tests/test_deep_scan_persistence.py b/plugins/codex-security/tests/test_deep_scan_persistence.py new file mode 100644 index 0000000000..22f81786d4 --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_persistence.py @@ -0,0 +1,425 @@ +from __future__ import annotations + +import json +import sqlite3 +import subprocess +import sys +import uuid +from pathlib import Path +from types import SimpleNamespace + +import pytest +from test_workbench_deep_scan import ( + begin_target_scan, + commit_reducer, + dispatch_discovery_worker, + upsert_worker, + worker_paths, +) +from workbench_test_support import run_workbench + + +def test_state_snapshot_does_not_mix_concurrent_acceptance( + tmp_path: Path, workbench_api, monkeypatch: pytest.MonkeyPatch +) -> None: + state_dir, codex_home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + initial = begin_target_scan(state_dir, codex_home, target, tmp_path / "scans")["deepScan"] + scan_id = initial["scanId"] + worker_id, _, _, _ = dispatch_discovery_worker( + state_dir, + codex_home, + scan_id=scan_id, + scan_dir=Path(initial["scanDir"]), + name="discovery-1", + succeed=False, + ) + database = state_dir / "workbench.sqlite3" + deep_scan = sys.modules["deep_scan_workbench"] + monkeypatch.setattr( + deep_scan, + "_dependencies", + SimpleNamespace( + **workbench_api, + preserve_stopped_results=workbench_api["preserve_stopped_results_after_transition"], + ), + ) + original = deep_scan.require_deep_scan_run + + def accept_after_read(connection, requested_scan_id): + run = original(connection, requested_scan_id) + with sqlite3.connect(database) as writer: + writer.execute( + "UPDATE deep_scan_runs SET completion_sequence = 1 WHERE scan_id = ?", (scan_id,) + ) + writer.execute( + "UPDATE deep_scan_workers SET status = 'succeeded', completion_sequence = 1 " + "WHERE id = ?", + (worker_id,), + ) + return run + + monkeypatch.setattr(deep_scan, "require_deep_scan_run", accept_after_read) + with sqlite3.connect(database) as reader: + reader.row_factory = sqlite3.Row + snapshot = deep_scan.deep_scan_state(reader, scan_id) + assert snapshot["completionSequence"] == 0 + assert snapshot["workers"][0]["status"] == "running" + assert not reader.in_transaction + + +def test_state_snapshot_preserves_its_callers_transaction( + tmp_path: Path, workbench_api, monkeypatch: pytest.MonkeyPatch +) -> None: + target = tmp_path / "target" + target.mkdir() + initial = begin_target_scan(tmp_path / "state", tmp_path / "codex", target, tmp_path / "scans") + scan_id = initial["deepScan"]["scanId"] + deep_scan = sys.modules["deep_scan_workbench"] + monkeypatch.setattr( + deep_scan, + "_dependencies", + SimpleNamespace( + **workbench_api, + preserve_stopped_results=workbench_api["preserve_stopped_results_after_transition"], + ), + ) + with sqlite3.connect(tmp_path / "state" / "workbench.sqlite3") as connection: + connection.row_factory = sqlite3.Row + connection.execute("BEGIN IMMEDIATE") + connection.execute( + "UPDATE deep_scan_runs SET consecutive_errors = 2 WHERE scan_id = ?", (scan_id,) + ) + snapshot = deep_scan.deep_scan_state(connection, scan_id) + assert snapshot["consecutiveErrors"] == 2 + assert connection.in_transaction + connection.rollback() + assert deep_scan.deep_scan_state(connection, scan_id)["consecutiveErrors"] == 0 + + +def test_replaced_attempts_retain_observed_sessions_and_accepted_result(tmp_path: Path) -> None: + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + run = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + worker_id = str(uuid.uuid4()) + prompt, artifacts, result = worker_paths(Path(run["scanDir"]), "discovery-1") + mutation = dict( + scan_id=run["scanId"], + worker_id=worker_id, + kind="discovery", + prompt_path=prompt, + artifact_dir=artifacts, + ) + upsert_worker(state, home, **mutation, status="running", attempt=1, thread_id="old-session") + upsert_worker( + state, + home, + **mutation, + status="running", + attempt=1, + thread_id="old-session", + error="Artifact validation failed", + ) + upsert_worker(state, home, **mutation, status="running", attempt=2, thread_id="new-session") + result.write_text('{"findings": []}\n') + accepted = upsert_worker( + state, + home, + **mutation, + status="succeeded", + attempt=2, + thread_id="new-session", + result_path=result, + )["deepScan"] + attempts = accepted["attempts"] + assert [(item["attempt"], item["status"]) for item in attempts] == [ + (1, "failed"), + (2, "succeeded"), + ] + assert [item["sdkThreadId"] for item in accepted["attemptSessions"]] == [ + "old-session", + "new-session", + ] + assert attempts[0]["error"] == "Artifact validation failed" + assert attempts[0]["completedAt"] is not None + assert attempts[1]["acceptedResultSha256"] + assert Path(attempts[1]["acceptedResultPath"]).read_text() == result.read_text() + result.unlink() + replayed = upsert_worker( + state, + home, + **mutation, + status="succeeded", + attempt=2, + thread_id="new-session", + result_path=result, + )["deepScan"] + assert replayed == accepted + + +def test_merge_replay_returns_original_operation_after_later_work(tmp_path: Path) -> None: + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + run = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + scan_id, scan_dir = run["scanId"], Path(run["scanDir"]) + inputs = [ + dispatch_discovery_worker( + state, + home, + scan_id=scan_id, + scan_dir=scan_dir, + name=f"discovery-{index}", + )[0] + for index in range(2) + ] + committed = commit_reducer( + state, + home, + scan_id=scan_id, + scan_dir=scan_dir, + name="dedup-1", + input_worker_ids=inputs, + new_findings_count=0, + ) + reducer = next(worker for worker in committed["workers"] if worker["kind"] == "dedup") + frozen = committed["committedMerge"]["resultManifestPath"] + assert Path(frozen).is_file() + Path(reducer["resultManifestPath"]).unlink() + assert [item["discoveryWorkerId"] for item in committed["dedupInputs"]] == inputs + assert all(item["attempt"] == 1 for item in committed["dedupInputs"]) + assert all("/checkpoints/" in item["resultManifestPath"] for item in committed["dedupInputs"]) + later = dispatch_discovery_worker( + state, + home, + scan_id=scan_id, + scan_dir=scan_dir, + name="discovery-2", + )[0] + second_commit = commit_reducer( + state, + home, + scan_id=scan_id, + scan_dir=scan_dir, + name="dedup-2", + input_worker_ids=[later], + new_findings_count=1, + ) + second_claim = second_commit["mergeClaims"][-1] + assert second_claim["previousWorkerId"] == reducer["id"] + assert second_claim["previousResultPath"] == frozen + assert ( + second_claim["previousResultSha256"] == committed["committedMerge"]["resultManifestSha256"] + ) + replay = run_workbench( + state, + "commit-deep-scan-dedup", + "--scan-id", + scan_id, + "--worker-id", + reducer["id"], + "--result-manifest-path", + str(scan_dir / "different.json"), + "--new-findings-count", + "99", + environment={"CODEX_HOME": str(home)}, + )["deepScan"] + assert replay["committedMerge"] == committed["committedMerge"] + assert replay["completionSequence"] == second_commit["completionSequence"] + assert replay["noNewStreak"] == second_commit["noNewStreak"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + receipt = json.loads( + connection.execute( + "SELECT receipt_json FROM deep_scan_merge_claims WHERE worker_id = ?", + (reducer["id"],), + ).fetchone()[0] + ) + assert receipt == committed["committedMerge"] + + +def test_native_usage_keeps_replaced_failed_canceled_attempts_and_descendants( + tmp_path: Path, workbench_api, monkeypatch: pytest.MonkeyPatch +) -> None: + from datetime import datetime, timedelta + + from test_workbench_scan_usage import _counts, _event, _rollout, _state_graph, _token_event + + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + run = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + worker_id = str(uuid.uuid4()) + prompt, artifacts, _ = worker_paths(Path(run["scanDir"]), "discovery-1") + mutation = dict( + scan_id=run["scanId"], + worker_id=worker_id, + kind="discovery", + prompt_path=prompt, + artifact_dir=artifacts, + ) + upsert_worker(state, home, **mutation, status="running", attempt=1, thread_id="old") + upsert_worker(state, home, **mutation, status="running", attempt=2, thread_id="old") + upsert_worker( + state, + home, + **mutation, + status="running", + attempt=2, + thread_id="old", + error="fixture failure", + ) + upsert_worker(state, home, **mutation, status="running", attempt=3, thread_id="replacement") + terminal = upsert_worker( + state, home, **mutation, status="canceled", attempt=3, thread_id="replacement" + )["deepScan"] + assert [item["status"] for item in terminal["attempts"]] == ["replaced", "failed", "canceled"] + environment = { + "CODEX_HOME": str(home), + "CODEX_SQLITE_HOME": str(tmp_path / "native"), + "CODEX_STATE_DB": "", + } + for key, value in environment.items(): + monkeypatch.setenv(key, value) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.row_factory = sqlite3.Row + scan = connection.execute("SELECT * FROM scans WHERE id = ?", (run["scanId"],)).fetchone() + timestamp = datetime.fromisoformat(scan["started_at"]) + timedelta(microseconds=1) + context = _event( + timestamp, "turn_context", {"turn_id": "fixture-turn", "model": "gpt-5.6-sol"} + ) + old = _rollout(tmp_path, "old", [context]) + _state_graph( + environment, + { + "thread-deep-scan": _rollout(tmp_path, "thread-deep-scan", []), + "old": old, + "replacement": _rollout( + tmp_path, "replacement", [context, _token_event(timestamp, 30, 0)] + ), + "child": _rollout( + tmp_path, + "child", + [context, _token_event(timestamp, 5, 0)], + parent_thread_id="old", + ), + "unrelated": _rollout( + tmp_path, + "unrelated", + [context, _token_event(timestamp, 900, 0)], + parent_thread_id="thread-deep-scan", + ), + }, + [("old", "child"), ("thread-deep-scan", "unrelated")], + ) + reader = sys.modules["workbench_scan_usage"] + pending = reader.collect_scan_usage(connection, scan) + assert pending["inputTokens"] == 35 + assert pending["missingThreadCount"] == 2 + old.write_text(old.read_text() + json.dumps(_token_event(timestamp, 20, 0)) + "\n") + measured = reader.collect_scan_usage(connection, scan) + assert measured["inputTokens"] == 55 + assert measured["threadCount"] == 3 + assert measured["missingThreadCount"] == 1 + assert measured["coverage"] == "partial" # Original shared parent turn was unavailable. + assert measured["modelUsage"] == [{"model": "gpt-5.6-sol", **_counts(55, 0, 0)}] + + +def test_claim_replay_preserves_original_inputs_after_concurrent_discovery(tmp_path: Path) -> None: + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + run = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + scan_id, scan_dir = run["scanId"], Path(run["scanDir"]) + inputs = [ + dispatch_discovery_worker( + state, home, scan_id=scan_id, scan_dir=scan_dir, name=f"discovery-{index}" + )[0] + for index in range(2) + ] + prompt, artifacts, _ = worker_paths(scan_dir, "reducer") + args = [ + "claim-deep-scan-dedup", + "--scan-id", + scan_id, + "--worker-id", + str(uuid.uuid4()), + "--prompt-path", + str(prompt), + "--artifact-dir", + str(artifacts), + ] + for worker in inputs: + args.extend(["--input-worker-id", worker]) + claimed = run_workbench(state, *args, environment={"CODEX_HOME": str(home)}) + dispatch_discovery_worker( + state, home, scan_id=scan_id, scan_dir=scan_dir, name="concurrent-discovery" + ) + replayed = run_workbench(state, *args, environment={"CODEX_HOME": str(home)}) + assert replayed["deepScan"]["mergeClaims"] == claimed["deepScan"]["mergeClaims"] + assert replayed["deepScan"]["dedupInputs"] == claimed["deepScan"]["dedupInputs"] + assert ( + replayed["deepScan"]["completionSequence"] == claimed["deepScan"]["completionSequence"] + 1 + ) + + +def test_acceptance_reuses_authoritative_checkpoint_without_rewriting(tmp_path: Path) -> None: + import hashlib + + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + run = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + worker_id = str(uuid.uuid4()) + prompt, artifacts, result = worker_paths(Path(run["scanDir"]), "discovery") + mutation = dict( + scan_id=run["scanId"], + worker_id=worker_id, + kind="discovery", + prompt_path=prompt, + artifact_dir=artifacts, + attempt=1, + ) + upsert_worker(state, home, **mutation, status="running") + draft = {"scanId": run["scanId"], "findings": [], "coverage": {}} + content = json.dumps(draft, indent=2).encode() + b"\n" + checkpoint = artifacts / "checkpoints" / f"{hashlib.sha256(content).hexdigest()}.json" + checkpoint.parent.mkdir() + checkpoint.write_bytes(content) + (artifacts / "checkpoint-head.json").write_text(json.dumps({"checkpoint": checkpoint.name})) + result.write_text(json.dumps({**draft, "handoffClaimToken": "synthetic-claim"})) + accepted = upsert_worker(state, home, **mutation, status="succeeded", result_path=result)[ + "deepScan" + ] + assert accepted["attempts"][0]["acceptedResultPath"] == str(checkpoint) + assert accepted["attempts"][0]["acceptedResultSha256"] == hashlib.sha256(content).hexdigest() + result.unlink() + upsert_worker(state, home, **mutation, status="succeeded", result_path=result) + assert list(checkpoint.parent.iterdir()) == [checkpoint] + assert checkpoint.read_bytes() == content + assert not (artifacts / "accepted").exists() + + +def test_acceptance_rejects_mutable_result_behind_checkpoint_head(tmp_path: Path) -> None: + import hashlib + + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + run = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + worker_id = str(uuid.uuid4()) + prompt, artifacts, result = worker_paths(Path(run["scanDir"]), "discovery") + mutation = dict( + scan_id=run["scanId"], + worker_id=worker_id, + kind="discovery", + prompt_path=prompt, + artifact_dir=artifacts, + attempt=1, + ) + upsert_worker(state, home, **mutation, status="running") + draft = {"scanId": run["scanId"], "findings": [], "coverage": {"deferred": ["unresolved"]}} + content = json.dumps(draft).encode() + checkpoint = artifacts / "checkpoints" / f"{hashlib.sha256(content).hexdigest()}.json" + checkpoint.parent.mkdir() + checkpoint.write_bytes(content) + (artifacts / "checkpoint-head.json").write_text(json.dumps({"checkpoint": checkpoint.name})) + result.write_text(json.dumps({**draft, "coverage": {}})) + with pytest.raises(subprocess.CalledProcessError) as failure: + upsert_worker(state, home, **mutation, status="succeeded", result_path=result) + assert "does not match its current checkpoint head" in failure.value.stderr + assert checkpoint.read_bytes() == content diff --git a/plugins/codex-security/tests/test_deep_scan_publication_authority.py b/plugins/codex-security/tests/test_deep_scan_publication_authority.py new file mode 100644 index 0000000000..04c900bbd2 --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_publication_authority.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import copy +import json +import uuid +from argparse import Namespace + +import pytest +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +def stage_publication(scan, *, generation, result_path, title): + draft_dir = scan.scan_dir / "drafts" + draft_dir.mkdir(exist_ok=True) + draft_path = draft_dir / f"{uuid.uuid4()}.json" + checkpoint_path = draft_dir / f"{uuid.uuid4()}.checkpoint.json" + findings = copy.deepcopy(scan.findings) + findings[0]["title"] = title + draft = { + "manifest": json.loads((scan.scan_dir / "scan-manifest.json").read_text()), + "findings": {"findings": findings}, + "coverage": scan.coverage, + } + if generation is not None: + draft["deepScanPublication"] = { + "coordinatorGeneration": generation, + "resultPath": str(result_path), + } + draft_path.write_text(json.dumps(draft)) + checkpoint_path.write_text( + json.dumps({"scanId": scan.scan_id, "findings": findings, "coverage": scan.coverage}) + ) + return Namespace( + scan_id=scan.scan_id, + claim_token=None, + draft_path=str(draft_path), + checkpoint_path=str(checkpoint_path), + expected_draft_digest=None, + ) + + +@pytest.mark.parametrize("stale", ["generation", "aggregate", "unfenced"]) +def test_stale_coordinator_cannot_replace_newer_canonical_publication( + workbench_api, workbench_db, publication_scan, stale +): + scan = publication_scan() + old_result = add_worker(workbench_db, scan) + new_result = add_worker(workbench_db, scan) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET coordinator_generation = 3 WHERE scan_id = ?", + (scan.scan_id,), + ) + for result, completed_at in ((old_result, "2026-01-01"), (new_result, "2026-01-02")): + workbench_db.execute( + "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none', completed_at = ? " + "WHERE result_manifest_path = ?", + (completed_at, str(result)), + ) + current = stage_publication( + scan, generation=3, result_path=new_result, title="Current accepted aggregate" + ) + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, current + ) + saved = { + path: path.read_bytes() + for path in scan.scan_dir.rglob("*.json") + if "drafts" not in path.parts + } + old = stage_publication( + scan, + generation=None if stale == "unfenced" else 2 if stale == "generation" else 3, + result_path=old_result if stale == "aggregate" else new_result, + title="Superseded aggregate", + ) + + with pytest.raises(SystemExit, match="coordinator|aggregate"): + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, old + ) + + assert { + path: path.read_bytes() + for path in scan.scan_dir.rglob("*.json") + if "drafts" not in path.parts + } == saved diff --git a/plugins/codex-security/tests/test_deep_scan_publication_replay.py b/plugins/codex-security/tests/test_deep_scan_publication_replay.py new file mode 100644 index 0000000000..c232f3e426 --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_publication_replay.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import json +import sqlite3 +import subprocess +import sys +from argparse import Namespace +from pathlib import Path + +import pytest +from test_accepted_publication_references import accept_reducer +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_publication_stop_interleavings import saved_selection + +_CRASH_PUBLICATION = """ +import json, os, runpy, sqlite3, sys +from argparse import Namespace + +api = runpy.run_path(sys.argv[1], run_name="publication_crash_test") +args = Namespace(**json.loads(sys.argv[3])) +boundary = sys.argv[4] + +class CrashConnection(sqlite3.Connection): + def __exit__(self, exc_type, exc_value, traceback): + completing = exc_type is None and self.execute( + "SELECT status FROM scans WHERE id = ?", (args.scan_id,) + ).fetchone()[0] == "complete" + if completing and boundary == "sqlite-before": + os._exit(72) + result = super().__exit__(exc_type, exc_value, traceback) + if completing and boundary == "sqlite-after": + os._exit(73) + return result + +connection = sqlite3.connect(sys.argv[2], factory=CrashConnection) +connection.row_factory = sqlite3.Row +connection.execute("PRAGMA foreign_keys = ON") +if boundary.startswith("sqlite-"): + api["complete_scan"](connection, Namespace( + scan_id=args.scan_id, claim_token=None, cost_json=None + )) +else: + saved = api["saved_results"] + original_write = saved.write_scan_local_bytes + def crash_after_write(root, relative, contents): + original_write(root, relative, contents) + if relative == boundary: + os._exit(71) + saved.write_scan_local_bytes = crash_after_write + api["saved_results"].write_scan_draft(api["_WORKBENCH_DB_CONTEXT"], connection, args) +raise AssertionError("publication never reached the requested crash boundary") +""" + + +@pytest.mark.parametrize( + "boundary", + ["findings.json", "coverage.json", "scan-manifest.json", "sqlite-before", "sqlite-after"], +) +@pytest.mark.parametrize("selection_reason", [None, "saturated", "capped"]) +def test_publication_crash_replays_selected_input_without_stale_overwrite( + workbench_api, workbench_db, publication_scan, tmp_path, boundary, selection_reason +): + scan = publication_scan() + if selection_reason is None: + result_path = add_worker(workbench_db, scan) + else: + _, result_path, _ = accept_reducer(workbench_db, scan) + saved_selection(workbench_db, scan, result_path, reason=selection_reason) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET terminal_reason = ? WHERE scan_id = ?", + (selection_reason, scan.scan_id), + ) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET coordinator_generation = 3 WHERE scan_id = ?", + (scan.scan_id,), + ) + workbench_db.execute( + "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none' WHERE scan_id = ?", + (scan.scan_id,), + ) + current = stage_publication( + scan, generation=3, result_path=result_path, title="Selected aggregate" + ) + stale = stage_publication( + scan, generation=2, result_path=result_path, title="Obsolete coordinator draft" + ) + database_path = tmp_path / "publication.sqlite3" + with sqlite3.connect(database_path) as connection: + workbench_db.backup(connection) + connection.row_factory = sqlite3.Row + if boundary.startswith("sqlite-"): + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, current + ) + + child = subprocess.run( + [ + sys.executable, + "-c", + _CRASH_PUBLICATION, + str(Path(__file__).resolve().parents[1] / "scripts" / "workbench_db.py"), + str(database_path), + json.dumps(vars(current)), + boundary, + ], + capture_output=True, + text=True, + ) + assert child.returncode == {"sqlite-before": 72, "sqlite-after": 73}.get(boundary, 71), ( + child.stdout, + child.stderr, + ) + interrupted = { + path: path.read_bytes() + for path in scan.scan_dir.rglob("*") + if path.is_file() and "drafts" not in path.parts + } + with sqlite3.connect(database_path) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + row = connection.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone() + assert row["status"] == ("complete" if boundary == "sqlite-after" else "running") + assert bool(row["seal_manifest_digest"]) == (boundary == "sqlite-after") + run_before = dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) + workers_before = [ + dict(row) for row in connection.execute("SELECT * FROM deep_scan_workers") + ] + + with pytest.raises(SystemExit, match="coordinator|stopped"): + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, stale + ) + assert all(path.read_bytes() == contents for path, contents in interrupted.items()) + + if not boundary.startswith("sqlite-"): + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, current + ) + completed = workbench_api["complete_scan"]( + connection, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + )["scan"] + assert completed["progress"]["status"] == "complete" + assert completed["findingCount"] == 1 + run_after = dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) + if run_before["finalization_input_json"] is not None: + before_selection = json.loads(run_before["finalization_input_json"]) + after_selection = json.loads(run_after["finalization_input_json"]) + digest = after_selection.pop("publicationSha256") + assert len(digest) == 64 and all(char in "0123456789abcdef" for char in digest) + if "publicationSha256" in before_selection: + assert digest == before_selection.pop("publicationSha256") + assert after_selection == before_selection + run_after["finalization_input_json"] = run_before["finalization_input_json"] + assert run_after == run_before + assert [ + dict(row) for row in connection.execute("SELECT * FROM deep_scan_workers") + ] == workers_before + assert connection.execute("SELECT COUNT(*) FROM finding_occurrences").fetchone()[0] == 1 + published = { + path: path.read_bytes() + for path in scan.scan_dir.rglob("*") + if path.is_file() and "drafts" not in path.parts + } + workbench_api["complete_scan"]( + connection, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + assert all(path.read_bytes() == contents for path, contents in published.items()) + if boundary.startswith("sqlite-"): + assert published == interrupted + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert findings[0]["title"] == "Selected aggregate" diff --git a/plugins/codex-security/tests/test_deep_scan_recovery_settings.py b/plugins/codex-security/tests/test_deep_scan_recovery_settings.py new file mode 100644 index 0000000000..37976c3e13 --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_recovery_settings.py @@ -0,0 +1,131 @@ +"""Original discovery input and observation remain stable across reconstruction.""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest +from workbench_test_support import run_workbench + + +@pytest.mark.parametrize("original_context", [None, "Audit the original parser"]) +def test_reconstruction_preserves_discovery_input_settings_and_deadline( + tmp_path: Path, + original_context: str | None, +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + codex_home = tmp_path / "home" + config = codex_home / "codex-security" / "config.toml" + config.parent.mkdir(parents=True) + config.write_text("[deep_scan]\nworkers = 2\nmax_time_hours = 2.5\n") + environment = {"CODEX_HOME": str(codex_home)} + begun = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + "--model", + "original-model", + "--reasoning-effort", + "high", + *(["--user-context-stdin"] if original_context is not None else []), + input_text=original_context, + environment=environment, + )["deepScan"] + scan_id = str(begun["scanId"]) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute("UPDATE scans SET user_context = 'Later discussion'") + connection.execute("UPDATE deep_scan_runs SET updated_at = '2000-01-01T00:00:00Z'") + before = "\n".join(connection.iterdump()) + config.write_text("[deep_scan]\nworkers = 8\nmax_time_hours = 12\n") + joined = run_workbench( + state, + "begin-deep-scan", + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + "--model", + "observer-model", + "--reasoning-effort", + "low", + environment=environment, + )["deepScan"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + assert "\n".join(connection.iterdump()) == before + assert connection.execute("SELECT model, reasoning_effort FROM scans").fetchone() == ( + "original-model", + "high", + ) + recovered = run_workbench( + state, + "claim-deep-scan-coordinator", + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + environment=environment, + )["deepScan"] + for run in (joined, recovered): + assert run["model"] == "original-model" + assert run["reasoningEffort"] == "high" + assert run["userContext"] == original_context + assert run["createdAt"] == begun["createdAt"] + assert run["config"] == begun["config"] + assert run["workflowVersion"] == begun["workflowVersion"] + + +@pytest.mark.parametrize("workflow_version", ["deep-security-scan/v1", "deep-scan-mcp/v1"]) +def test_supported_old_run_snapshots_context_on_upgrade( + tmp_path: Path, workflow_version: str +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + begun = run_workbench( + state, + "begin-deep-scan", + "--workflow-version", + workflow_version, + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + "--user-context", + "Legacy context", + )["deepScan"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute("ALTER TABLE deep_scan_runs DROP COLUMN discovery_user_context") + connection.execute("DELETE FROM schema_migrations WHERE version = 44") + upgraded = run_workbench( + state, + "get-deep-scan", + "--scan-id", + str(begun["scanId"]), + "--thread-id", + "fixture-thread", + )["deepScan"] + assert upgraded["workflowVersion"] == workflow_version + assert upgraded["userContext"] == "Legacy context" + assert upgraded["config"] == begun["config"] + assert upgraded["createdAt"] == begun["createdAt"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute("UPDATE scans SET user_context = 'Later discussion'") + observed = run_workbench( + state, + "get-deep-scan", + "--scan-id", + str(begun["scanId"]), + "--thread-id", + "fixture-thread", + )["deepScan"] + assert observed["userContext"] == "Legacy context" diff --git a/plugins/codex-security/tests/test_deep_scan_settings_creation.py b/plugins/codex-security/tests/test_deep_scan_settings_creation.py new file mode 100644 index 0000000000..f23212ee7a --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_settings_creation.py @@ -0,0 +1,375 @@ +"""Original settings survive real creation/claim process death and competing starts.""" + +from __future__ import annotations + +import json +import os +import sqlite3 +import subprocess +import sys +from pathlib import Path + +import pytest +from workbench_test_support import ( + SCRIPT, + create_saved_workspace, + run_workbench, + start_delivered_scan, +) + +# Run the real workbench, pausing only at a requested transaction boundary. +# The parse hook also runs against the parent source, where the settings input +# was ignored, so its regression fails on missing durable bytes. +BOUNDARY_PROCESS = """ +import json, runpy, sys +from pathlib import Path +boundary = sys.argv.pop(1) +script = sys.argv.pop(1) +main = runpy.run_path(script)['main'] +namespace = main.__globals__ +deep = namespace['deep_scan'] +parse = namespace['parse_args'] +def parse_with_settings(*args, **kwargs): + result = parse(*args, **kwargs) + payload = json.loads(sys.stdin.readline()) + result.execution_settings = payload['executionSettings'] + result.user_context = payload.get('userContext') + result.user_context_stdin = False + return result +namespace['parse_args'] = parse_with_settings +def pause(scan): + print(json.dumps({'scanId': scan['id'], 'scanDir': scan['scan_dir']}), flush=True) + sys.stdin.readline() +write = deep.write_scan_local_bytes +def write_settings(scan_dir, relative_path, payload, **kwargs): + if boundary == 'before-write' and relative_path.endswith('/execution-settings.json'): + pause({'id': None, 'scan_dir': str(scan_dir)}) + return write(scan_dir, relative_path, payload, **kwargs) +deep.write_scan_local_bytes = write_settings +ensure = deep.ensure_deep_scan_run +def ensure_run(connection, scan, *args): + result = ensure(connection, scan, *args) + if boundary == 'before-commit': + pause(scan) + return result +deep.ensure_deep_scan_run = ensure_run +begin = deep.begin_deep_scan +def begin_run(connection, args): + result = begin(connection, args) + if boundary == 'after-commit': + pause({'id': result['deepScan']['scanId'], 'scan_dir': result['deepScan']['scanDir']}) + return result +deep.begin_deep_scan = begin_run +claim = deep.claim_deep_scan_coordinator +def claim_run(connection, args): + result = claim(connection, args) + if boundary == 'after-claim': + pause({'id': result['deepScan']['scanId'], 'scan_dir': result['deepScan']['scanDir']}) + return result +deep.claim_deep_scan_coordinator = claim_run +main() +""" + + +def settings(root: Path, name: str) -> dict[str, object]: + return { + "codexPath": str(root / name / "codex"), + "codexHome": str(root / name / "home"), + "model": f"{name}-model", + "reasoningEffort": "high", + "reasoningSummary": "concise", + "parentSandbox": {"filesystemDenies": [str(root / name / "denied")]}, + } + + +def start_process( + root: Path, + selected: dict[str, object] | None, + *args: str, + boundary: str = "none", +) -> subprocess.Popen[str]: + process = subprocess.Popen( + [sys.executable, "-c", BOUNDARY_PROCESS, boundary, str(SCRIPT), *args], + env={ + **os.environ, + "CODEX_SECURITY_STATE_DIR": str(root / "state"), + "CODEX_HOME": str(root / "current-home"), + }, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert process.stdin is not None + process.stdin.write(json.dumps({"executionSettings": selected}) + "\n") + process.stdin.flush() + return process + + +def finish(process: subprocess.Popen[str]) -> dict[str, object]: + stdout, stderr = process.communicate(timeout=30) + assert process.returncode == 0, stderr + return json.loads(stdout) + + +def begin_args(root: Path, *, name: str = "target") -> list[str]: + (root / name).mkdir(exist_ok=True) + return [ + "begin-deep-scan", + "--thread-id", + "original-thread", + "--target-path", + str(root / name), + "--scan-root", + str(root / "scans"), + ] + + +def snapshot(scan_dir: str) -> Path: + return Path(scan_dir) / "artifacts" / "deep_discovery" / "execution-settings.json" + + +@pytest.mark.parametrize( + "boundary", ["before-write", "before-commit", "after-commit", "after-claim"] +) +def test_process_death_keeps_original_settings_or_no_recoverable_run( + tmp_path: Path, boundary: str +) -> None: + original = settings(tmp_path, "original") + args = begin_args(tmp_path) + if boundary == "after-claim": + begun = finish(start_process(tmp_path, original, *args))["deepScan"] + args = [ + "claim-deep-scan-coordinator", + "--scan-id", + begun["scanId"], + "--thread-id", + "original-thread", + ] + process = start_process(tmp_path, original, *args, boundary=boundary) + try: + assert process.stdout is not None + paused = json.loads(process.stdout.readline()) + path = snapshot(paused["scanDir"]) + with sqlite3.connect(tmp_path / "state" / "workbench.sqlite3") as connection: + rows = connection.execute("SELECT scan_id FROM deep_scan_runs").fetchall() + if boundary in {"before-write", "before-commit"}: + assert rows == [], "uncommitted settings must not leave a recoverable run" + else: + assert rows == [(paused["scanId"],)] + if boundary != "before-write": + assert json.loads(path.read_bytes())["settings"] == original + finally: + process.kill() + process.communicate(timeout=30) + with sqlite3.connect(tmp_path / "state" / "workbench.sqlite3") as connection: + remaining = connection.execute("SELECT scan_id FROM deep_scan_runs").fetchall() + if boundary in {"before-write", "before-commit"}: + assert remaining == [] + return + before = path.read_bytes() + with sqlite3.connect(tmp_path / "state" / "workbench.sqlite3") as connection: + connection.execute("UPDATE deep_scan_runs SET updated_at = '2000-01-01T00:00:00Z'") + run_workbench( + tmp_path / "state", + "claim-deep-scan-coordinator", + "--scan-id", + paused["scanId"], + "--thread-id", + "original-thread", + ) + joined = finish(start_process(tmp_path, settings(tmp_path, "later"), *begin_args(tmp_path))) + assert joined["startDisposition"] == "joined" + assert path.read_bytes() == before + + +def test_competing_creation_and_observer_cannot_replace_original_settings(tmp_path: Path) -> None: + first = start_process( + tmp_path, settings(tmp_path, "original"), *begin_args(tmp_path), boundary="before-commit" + ) + second = None + try: + assert first.stdout is not None + paused = json.loads(first.stdout.readline()) + path = snapshot(paused["scanDir"]) + before = path.read_bytes() + second = start_process(tmp_path, settings(tmp_path, "later"), *begin_args(tmp_path)) + # Release the creation lock with the original process still alive. The + # contender must observe the committed run, not rewrite the settings. + created = finish(first) + joined = finish(second) + assert created["startDisposition"] == "created" + assert joined["startDisposition"] == "joined" + assert joined["deepScan"]["scanId"] == paused["scanId"] + assert path.read_bytes() == before + observer = finish(start_process(tmp_path, None, *begin_args(tmp_path))) + assert observer["startDisposition"] == "joined" + assert path.read_bytes() == before + other = finish( + start_process( + tmp_path, settings(tmp_path, "other"), *begin_args(tmp_path, name="other-target") + ) + ) + assert json.loads(snapshot(other["deepScan"]["scanDir"]).read_bytes())[ + "settings" + ] == settings(tmp_path, "other") + finally: + for process in (first, second): + if process is not None and process.poll() is None: + process.kill() + process.communicate(timeout=30) + + +def test_internal_settings_input_preserves_public_user_context_and_shape(tmp_path: Path) -> None: + context = "Review the parser.\nKeep this second line." + process = subprocess.run( + [ + sys.executable, + "-c", + ( + "import runpy, sys; script = sys.argv.pop(1); " + "runpy.run_path(script)['main'](with_execution_settings=True)" + ), + str(SCRIPT), + *begin_args(tmp_path), + ], + env={**os.environ, "CODEX_SECURITY_STATE_DIR": str(tmp_path / "state")}, + input=json.dumps( + {"executionSettings": settings(tmp_path, "original"), "userContext": context} + ), + capture_output=True, + text=True, + timeout=30, + ) + assert process.returncode == 0, process.stderr + result = json.loads(process.stdout) + assert result["deepScan"]["userContext"] == context + assert result["deepScan"]["executionSettings"] == { + "version": 1, + "settings": settings(tmp_path, "original"), + } + public = run_workbench( + tmp_path / "state", + "get-deep-scan", + "--scan-id", + result["deepScan"]["scanId"], + "--thread-id", + "original-thread", + ) + assert "executionSettings" not in public["deepScan"] + assert json.loads(snapshot(result["deepScan"]["scanDir"]).read_bytes())["settings"] == settings( + tmp_path, "original" + ) + + +def test_managed_creation_retry_preserves_saved_settings_before_commit(tmp_path: Path) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + workspace = create_saved_workspace(state, target, thread_id="original-thread", mode="deep") + started = start_delivered_scan( + state, + "--workspace-id", + workspace["id"], + "--scan-root", + str(tmp_path / "scans"), + "--model", + "original-model", + "--reasoning-effort", + "high", + ) + scan = started["results"] + args = ["begin-deep-scan", "--scan-id", scan["scanId"], "--thread-id", "original-thread"] + original = settings(tmp_path, "original") + process = start_process(tmp_path, original, *args, boundary="before-commit") + try: + assert process.stdout is not None + paused = json.loads(process.stdout.readline()) + path = snapshot(paused["scanDir"]) + before = path.read_bytes() + finally: + process.kill() + process.communicate(timeout=30) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + assert connection.execute("SELECT COUNT(*) FROM deep_scan_runs").fetchone() == (0,) + recovered = finish( + start_process( + tmp_path, + original, + *args, + ) + ) + assert recovered["startDisposition"] == "created" + assert path.read_bytes() == before + assert json.loads(before)["settings"] == original + assert recovered["deepScan"]["model"] == "original-model" + assert recovered["deepScan"]["reasoningEffort"] == "high" + with sqlite3.connect(state / "workbench.sqlite3") as connection: + bound = json.loads( + connection.execute( + "SELECT execution_settings_json FROM deep_scan_runs WHERE scan_id = ?", + (scan["scanId"],), + ).fetchone()[0] + ) + assert bound == {"version": 1, "settings": original} + + +def test_preexisting_artifact_does_not_choose_original_execution_settings(tmp_path: Path) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + workspace = create_saved_workspace(state, target, thread_id="original-thread", mode="deep") + scan = start_delivered_scan( + state, "--workspace-id", workspace["id"], "--scan-root", str(tmp_path / "scans") + )["results"] + path = snapshot(scan["scanDir"]) + path.parent.mkdir(parents=True) + path.write_text(json.dumps({"version": 1, "settings": settings(tmp_path, "substituted")})) + original = settings(tmp_path, "original") + begun = finish( + start_process( + tmp_path, + original, + "begin-deep-scan", + "--scan-id", + scan["scanId"], + "--thread-id", + "original-thread", + ) + ) + assert begun["startDisposition"] == "created" + assert json.loads(path.read_text()) == {"version": 1, "settings": original} + with sqlite3.connect(state / "workbench.sqlite3") as connection: + bound = json.loads( + connection.execute( + "SELECT execution_settings_json FROM deep_scan_runs WHERE scan_id = ?", + (scan["scanId"],), + ).fetchone()[0] + ) + assert bound == {"version": 1, "settings": original} + path.write_text(json.dumps({"version": 1, "settings": settings(tmp_path, "substituted")})) + joined = finish( + start_process( + tmp_path, + settings(tmp_path, "observer"), + "begin-deep-scan", + "--scan-id", + scan["scanId"], + "--thread-id", + "original-thread", + ) + ) + assert joined["startDisposition"] == "joined" + with sqlite3.connect(state / "workbench.sqlite3") as connection: + repeated = json.loads( + connection.execute( + "SELECT execution_settings_json FROM deep_scan_runs WHERE scan_id = ?", + (scan["scanId"],), + ).fetchone()[0] + ) + assert repeated == bound + public = run_workbench(state, "get-scan", "--scan-id", scan["scanId"]) + assert public["scan"]["executionAttribution"]["workerCodexHome"] == original["codexHome"] + path.unlink() + public = run_workbench(state, "get-scan", "--scan-id", scan["scanId"]) + assert public["scan"]["executionAttribution"]["workerCodexHome"] == original["codexHome"] diff --git a/plugins/codex-security/tests/test_deep_scan_successful_publication.py b/plugins/codex-security/tests/test_deep_scan_successful_publication.py index ef5eb6b44b..c40131dff4 100644 --- a/plugins/codex-security/tests/test_deep_scan_successful_publication.py +++ b/plugins/codex-security/tests/test_deep_scan_successful_publication.py @@ -66,7 +66,7 @@ def create(*, mode="deep", scope="."): "INSERT INTO deep_scan_runs (scan_id, schema_version, workflow_version, " "status, phase, workers, subagents, stop_after_no_new, max_discovery_runs, " "manifest_path, terminal_reason, created_at, updated_at, completed_at) " - "VALUES (?, 1, 'publication-test', 'succeeded', 'terminal', 1, 0, 1, 1, " + "VALUES (?, 1, 'deep-security-scan/v1', 'succeeded', 'terminal', 1, 0, 1, 1, " "?, 'saturated', ?, ?, ?)", ( scan_id, @@ -168,6 +168,38 @@ def assert_published_aggregate(scan): assert (scan.scan_dir / "report.md").is_file() +def test_deep_publication_renders_each_source_remediation( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + finding = scan.findings[0] + first = copy.deepcopy(finding) + first.pop("provenance") + first["remediation"] = "Check the destination before writing the archive entry." + first["remediationTests"] = ["Reject an archive entry outside the destination."] + second = copy.deepcopy(first) + second["remediation"] = "Reject symbolic links before opening the destination." + second["remediationTests"] = ["Reject a symbolic link inside the destination."] + second["preventiveControls"] = ["Use a directory-relative file handle."] + finding["remediation"] = first["remediation"] + finding["remediationTests"] = first["remediationTests"] + finding["provenance"]["sourceFindings"] = [ + {"id": "review-1:0", "finding": first}, + {"id": "review-2:0", "finding": second}, + ] + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": scan.findings})) + + complete(workbench_api, workbench_db, scan) + + assert_published_aggregate(scan) + report = (scan.scan_dir / "report.md").read_text() + for source in (first, second): + assert report.count(source["remediation"]) == 1 + for test in source["remediationTests"]: + assert report.count(test) == 1 + assert "Use a directory-relative file handle." in report + + @pytest.mark.parametrize("scope", [".", "subdir"], ids=["repository", "scoped"]) def test_deep_publication_keeps_configured_scope_without_worker_observations( workbench_api, workbench_db, publication_scan, scope @@ -349,7 +381,8 @@ def test_stopped_deep_scan_still_salvages_saved_findings( result.write_text("{interrupted worker output") result_bytes = result.read_bytes() - stopped = workbench_api["fail_scan"]( + stopped = workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace( scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Scan interrupted." @@ -369,8 +402,8 @@ def test_stopped_deep_scan_still_salvages_saved_findings( artifact_names = ("scan-manifest.json", "findings.json", "coverage.json") published = {name: (scan.scan_dir / name).read_bytes() for name in artifact_names} - recovered = workbench_api["recover_scan_results"]( - workbench_db, Namespace(scan_id=scan.scan_id) + recovered = workbench_api["saved_results"].recover_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id) )["scan"] assert recovered["findingCount"] == len(expected_summaries) assert {name: (scan.scan_dir / name).read_bytes() for name in artifact_names} == published @@ -412,7 +445,8 @@ def test_stopped_deep_scan_ignores_non_reducer_sources_without_coverage( source_bytes = source_path.read_bytes() source_relative = source_path.relative_to(scan.scan_dir).as_posix() - stopped = workbench_api["fail_scan"]( + stopped = workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace( scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Scan interrupted." @@ -434,8 +468,8 @@ def test_stopped_deep_scan_ignores_non_reducer_sources_without_coverage( artifact_names = ("scan-manifest.json", "findings.json", "coverage.json") published = {name: (scan.scan_dir / name).read_bytes() for name in artifact_names} - recovered = workbench_api["recover_scan_results"]( - workbench_db, Namespace(scan_id=scan.scan_id) + recovered = workbench_api["saved_results"].recover_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id) )["scan"] assert recovered["findingCount"] == 1 @@ -450,8 +484,38 @@ def test_standard_publication_preserves_deliberately_partial_coverage( scan = publication_scan(mode="standard") scan.coverage["completeness"] = "partial" scan.coverage["deferred"] = [{"id": "remaining-review", "reason": "Another surface remains."}] + scan.coverage["reviews"] = [] (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage)) complete(workbench_api, workbench_db, scan) assert_published_aggregate(scan) + + +def test_deep_publication_write_failure_keeps_original_terminal_cause( + workbench_api, workbench_db, publication_scan, monkeypatch +): + scan = publication_scan() + finalizer_globals = workbench_api["_write_prepared_scan_finalization"].__globals__ + write_bytes = finalizer_globals["write_scan_local_bytes"] + + def fail_report(scan_dir, relative_path, payload, **kwargs): + if relative_path == "report.md": + raise finalizer_globals["ContractError"]("Synthetic report write interruption") + return write_bytes(scan_dir, relative_path, payload, **kwargs) + + with monkeypatch.context() as patch: + patch.setitem(finalizer_globals, "write_scan_local_bytes", fail_report) + with pytest.raises(SystemExit, match="Synthetic report write interruption"): + complete(workbench_api, workbench_db, scan) + + assert ( + workbench_db.execute("SELECT status FROM scans WHERE id = ?", (scan.scan_id,)).fetchone()[0] + == "running" + ) + run = workbench_db.execute( + "SELECT status, terminal_reason FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,) + ).fetchone() + assert tuple(run) == ("succeeded", "saturated") + assert complete(workbench_api, workbench_db, scan)["progress"]["status"] == "complete" + assert_published_aggregate(scan) diff --git a/plugins/codex-security/tests/test_deep_scan_usage_owner.py b/plugins/codex-security/tests/test_deep_scan_usage_owner.py new file mode 100644 index 0000000000..9b5cb237cd --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_usage_owner.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import json +import sqlite3 +from datetime import datetime, timezone +from pathlib import Path + +from test_workbench_scan_usage import _event, _rollout, _state_graph +from workbench_test_support import run_workbench + + +def test_original_usage_turn_survives_join_and_coordinator_recovery(tmp_path: Path) -> None: + state = tmp_path / "state" + environment = { + "CODEX_HOME": str(tmp_path / "codex"), + "CODEX_SQLITE_HOME": str(tmp_path / "native"), + "CODEX_STATE_DB": "", + } + timestamp = datetime.now(timezone.utc) + rollout = _rollout( + tmp_path, + "shared-parent", + [ + _event(timestamp, "turn_context", {"turn_id": "original-turn", "model": "gpt-5.6-sol"}), + ], + ) + _state_graph(environment, {"shared-parent": rollout}, []) + target = tmp_path / "target" + target.mkdir() + begun = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "shared-parent", + "--target-path", + str(target), + "--scope", + ".", + "--scan-root", + str(tmp_path / "scans"), + environment=environment, + )["deepScan"] + owner = begun["usageOwner"] + assert owner["threadId"] == "shared-parent" + assert owner["turnId"] == "original-turn" + assert owner["dedicated"] is False + rollout.write_text( + rollout.read_text() + + json.dumps( + _event(timestamp, "turn_context", {"turn_id": "later-turn", "model": "gpt-6-astra"}) + ) + + "\n" + ) + joined = run_workbench( + state, + "begin-deep-scan", + "--scan-id", + begun["scanId"], + "--thread-id", + "shared-parent", + environment=environment, + )["deepScan"] + assert joined["usageOwner"] == owner + claim_args = [ + "claim-deep-scan-coordinator", + "--scan-id", + begun["scanId"], + "--thread-id", + "shared-parent", + ] + claimed = run_workbench(state, *claim_args, environment=environment)["deepScan"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute( + "UPDATE deep_scan_runs SET updated_at = '2000-01-01T00:00:00+00:00' WHERE scan_id = ?", + (begun["scanId"],), + ) + recovered = run_workbench(state, *claim_args, environment=environment)["deepScan"] + assert recovered["coordinatorGeneration"] == claimed["coordinatorGeneration"] + 1 + assert recovered["usageOwner"] == owner + other_target = tmp_path / "other-target" + other_target.mkdir() + other = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "shared-parent", + "--target-path", + str(other_target), + "--scope", + ".", + "--scan-root", + str(tmp_path / "scans"), + environment=environment, + )["deepScan"] + assert other["usageOwner"]["turnId"] == "later-turn" + original = run_workbench( + state, "get-scan", "--scan-id", begun["scanId"], environment=environment + )["scan"] + assert original["executionAttribution"]["owner"] == owner diff --git a/plugins/codex-security/tests/test_finalization_selection_process_loss.py b/plugins/codex-security/tests/test_finalization_selection_process_loss.py new file mode 100644 index 0000000000..a599e73ff7 --- /dev/null +++ b/plugins/codex-security/tests/test_finalization_selection_process_loss.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import io +import json +import sqlite3 +import subprocess +import sys +from argparse import Namespace +from pathlib import Path + +import pytest +from test_accepted_publication_references import accept_reducer +from test_checkpoint_publication_authority import save_disposition +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_publication_stop_interleavings import published_bytes, stop_scan + +_CRASH_SELECTION = """ +import io, json, os, runpy, sqlite3, sys +from argparse import Namespace + +api = runpy.run_path(sys.argv[1], run_name="selection_commit_crash_test") +deep = api["deep_scan"] +from types import SimpleNamespace +deep.configure(SimpleNamespace(**{**api, "preserve_stopped_results": api["preserve_stopped_results_after_transition"]})) +class CrashConnection(sqlite3.Connection): + def commit(self): + if sys.argv[4] == "before": + os._exit(72) + super().commit() + os._exit(73) +connection = sqlite3.connect(sys.argv[2], factory=CrashConnection) +connection.row_factory = sqlite3.Row +connection.execute("PRAGMA foreign_keys = ON") +sys.stdin = io.StringIO(json.dumps({"resultPath": sys.argv[5]})) +deep.finish_deep_scan(connection, Namespace(**json.loads(sys.argv[3])), select_finalization=True) +raise AssertionError("selection never reached its commit boundary") +""" + + +@pytest.mark.parametrize("reason", ["saturated", "capped"]) +@pytest.mark.parametrize("cut", ["before", "after"]) +@pytest.mark.parametrize("cause", ["cancel", "cost"]) +@pytest.mark.parametrize("stop_before_replay", [False, True]) +def test_selection_commit_loss_replays_accepted_identity_before_stopping( + workbench_api, + workbench_db, + publication_scan, + tmp_path, + monkeypatch, + reason, + cut, + cause, + stop_before_replay, +): + scan = publication_scan() + result, accepted, _ = accept_reducer(workbench_db, scan) + omissions = [] + if reason == "saturated": + omitted = add_worker(workbench_db, scan) + omitted.write_text(json.dumps(save_disposition(scan, omitted.parent, "reported"))) + save_disposition(scan, omitted.parent, "rejected") + omissions.append(omitted.parent.name) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET merge_state = 'buffered' WHERE id = ?", + (omitted.parent.name,), + ) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "status = 'running', phase = 'reducing', terminal_reason = NULL, completed_at = NULL, " + "consecutive_no_new = stop_after_no_new, discovery_runs_dispatched = max_discovery_runs " + "WHERE scan_id = ?", + (scan.scan_id,), + ) + args = Namespace( + scan_id=scan.scan_id, + coordinator_generation=3, + terminal_reason=reason, + manifest_path=str(scan.scan_dir / "scan-manifest.json"), + staged_manifest_path=None, + omitted_worker_id=omissions, + ) + database = tmp_path / "selection.sqlite3" + with sqlite3.connect(database) as connection: + workbench_db.backup(connection) + before = published_bytes(scan) + child = subprocess.run( + [ + sys.executable, + "-c", + _CRASH_SELECTION, + str(Path(__file__).resolve().parents[1] / "scripts" / "workbench_db.py"), + str(database), + json.dumps(vars(args)), + cut, + str(result), + ], + capture_output=True, + text=True, + ) + assert child.returncode == (72 if cut == "before" else 73), (child.stdout, child.stderr) + assert published_bytes(scan) == before + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + run = connection.execute("SELECT * FROM deep_scan_runs").fetchone() + selected = json.loads(run["finalization_input_json"]) if cut == "after" else None + if cut == "before": + assert run["finalization_input_json"] is None + assert run["terminal_reason"] is None + # The request names the deleted output. Selection resolves its committed + # accepted attempt rather than reading that replaceable file again. + monkeypatch.setattr(sys, "stdin", io.StringIO(json.dumps({"resultPath": str(result)}))) + if stop_before_replay: + stop_scan(workbench_api, connection, scan, cause) + stopped = published_bytes(scan) + stopped_database = "\n".join(connection.iterdump()) + with pytest.raises(SystemExit, match="running|stopped|failed|canceled"): + workbench_api["deep_scan"].finish_deep_scan( + connection, args, select_finalization=True + ) + assert "\n".join(connection.iterdump()) == stopped_database + assert published_bytes(scan) == stopped + run = connection.execute("SELECT * FROM deep_scan_runs").fetchone() + assert run["status"] == ("canceled" if cause == "cancel" else "failed") + assert run["terminal_reason"] == (reason if cut == "after" else None) + assert ( + json.loads(run["finalization_input_json"]) if cut == "after" else None + ) == selected + assert accepted.read_bytes() == before[accepted.relative_to(scan.scan_dir).as_posix()] + return + replayed = workbench_api["deep_scan"].finish_deep_scan( + connection, args, select_finalization=True + )["deepScan"] + selection = replayed["finalizationInput"] + if selected is not None: + assert selection == selected + assert selection["resultPath"] == accepted.relative_to(scan.scan_dir).as_posix() + assert selection["resultSha256"] == accepted.stem + assert selection["terminalReason"] == reason + assert selection["omittedWorkerIds"] == omissions + assert replayed["terminalReason"] == reason + assert replayed["status"] == "running" + assert published_bytes(scan) == before + stale = stage_publication(scan, generation=2, result_path=accepted, title="Stale aggregate") + with pytest.raises(SystemExit, match="generation"): + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, stale + ) + assert published_bytes(scan) == before + stop_scan(workbench_api, connection, scan, cause) + stopped = published_bytes(scan) + late = stage_publication(scan, generation=3, result_path=accepted, title="Late aggregate") + with pytest.raises(SystemExit, match="stopped"): + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, late + ) + assert published_bytes(scan) == stopped + run = connection.execute("SELECT * FROM deep_scan_runs").fetchone() + assert run["status"] == ("canceled" if cause == "cancel" else "failed") + assert run["terminal_reason"] == reason + assert json.loads(run["finalization_input_json"]) == selection + assert accepted.read_bytes() == before[selection["resultPath"]] + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == 1 + assert findings[0].get("extensions", {}).get("candidateId") != "candidate-disposition" diff --git a/plugins/codex-security/tests/test_publication_stop_interleavings.py b/plugins/codex-security/tests/test_publication_stop_interleavings.py new file mode 100644 index 0000000000..ea796a6d88 --- /dev/null +++ b/plugins/codex-security/tests/test_publication_stop_interleavings.py @@ -0,0 +1,469 @@ +from __future__ import annotations + +import json +import sqlite3 +import subprocess +import sys +from argparse import Namespace +from pathlib import Path + +import pytest +from test_accepted_publication_references import accept_reducer +from test_checkpoint_publication_authority import save_disposition +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +def saved_selection(connection, scan, accepted, omitted=None, *, reason="saturated"): + selection = { + "version": 1, + "resultPath": accepted.relative_to(scan.scan_dir).as_posix(), + "resultSha256": accepted.stem, + "terminalReason": reason, + "omittedWorkerIds": [omitted.parent.name] if omitted is not None else [], + "selectedAt": scan.timestamp, + } + with connection: + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "finalization_input_json = ?, terminal_reason = ?, phase = 'terminal' " + "WHERE scan_id = ?", + (json.dumps(selection), reason, scan.scan_id), + ) + return selection + + +def stop_scan(api, connection, scan, cause): + if cause == "cancel": + return api["saved_results"].cancel_scan( + api["_WORKBENCH_DB_CONTEXT"], + connection, + Namespace(scan_id=scan.scan_id, thread_id=None), + ) + return api["saved_results"].fail_scan( + api["_WORKBENCH_DB_CONTEXT"], + connection, + Namespace( + scan_id=scan.scan_id, + claim_token=None, + cost_json=None, + message="Scan stopped after reaching the configured cost limit.", + ), + ) + + +def published_bytes(scan): + return { + path.relative_to(scan.scan_dir).as_posix(): path.read_bytes() + for path in scan.scan_dir.rglob("*") + if path.is_file() and "drafts" not in path.relative_to(scan.scan_dir).parts + } + + +_CRASH_STOPPED_PUBLICATION = """ +import os, runpy, sqlite3, sys +from argparse import Namespace + +api = runpy.run_path(sys.argv[1], run_name="stopped_publication_crash_test") +scan_id, cause, boundary = sys.argv[3:] + +class CrashConnection(sqlite3.Connection): + def __exit__(self, *args): + sealing = self.execute( + "SELECT seal_manifest_digest FROM scans WHERE id = ?", (scan_id,) + ).fetchone()[0] is not None + if sealing and boundary == "sqlite-before": + os._exit(72) + result = super().__exit__(*args) + if sealing and boundary == "sqlite-after": + os._exit(73) + return result + +connection = sqlite3.connect(sys.argv[2], factory=CrashConnection) +connection.row_factory = sqlite3.Row +connection.execute("PRAGMA foreign_keys = ON") +import finalize_scan_contract as contract +original_write = contract.write_scan_local_bytes +def crash_after_write(root, relative, contents, **kwargs): + original_write(root, relative, contents, **kwargs) + if relative == boundary: + os._exit(71) +contract.write_scan_local_bytes = crash_after_write +if cause == "cancel": + api["saved_results"].cancel_scan(api["_WORKBENCH_DB_CONTEXT"], connection, Namespace(scan_id=scan_id, thread_id=None)) +else: + api["saved_results"].fail_scan(api["_WORKBENCH_DB_CONTEXT"], connection, Namespace( + scan_id=scan_id, claim_token=None, cost_json=None, + message="Scan stopped after reaching the configured cost limit." + )) +raise AssertionError("stop never reached the requested publication boundary") +""" + +_CRASH_SELECTION_RECOVERY = """ +import os, runpy, sqlite3, sys +from argparse import Namespace + +api = runpy.run_path(sys.argv[1], run_name="selection_recovery_crash_test") +deep = api["deep_scan"] +from types import SimpleNamespace +deep.configure(SimpleNamespace(**{**api, "preserve_stopped_results": api["preserve_stopped_results_after_transition"]})) + +class CrashConnection(sqlite3.Connection): + def __exit__(self, kind, error, traceback): + if kind is None: + if sys.argv[4] == "before": + os._exit(72) + super().__exit__(kind, error, traceback) + os._exit(73) + return super().__exit__(kind, error, traceback) + + def commit(self): + if sys.argv[4] == "before": + os._exit(72) + super().commit() + os._exit(73) + +connection = sqlite3.connect(sys.argv[2], factory=CrashConnection) +connection.row_factory = sqlite3.Row +connection.execute("PRAGMA foreign_keys = ON") +deep.claim_deep_scan_coordinator(connection, Namespace( + scan_id=sys.argv[3], thread_id="fixture-owner", + claim_token=None, coordinator_generation=None, +)) +raise AssertionError("recovery never reached the requested commit boundary") +""" + + +@pytest.mark.parametrize("cause", ["cancel", "cost"]) +@pytest.mark.parametrize("cut", ["before-selection", "selected", "published", "sealed"]) +def test_stop_and_publication_keep_the_winning_terminal_outcome( + workbench_api, workbench_db, publication_scan, tmp_path, cause, cut +): + scan = publication_scan() + _, accepted, coverage = accept_reducer(workbench_db, scan) + omitted = add_worker(workbench_db, scan) + rejected = save_disposition(scan, omitted.parent, "reported") + omitted.write_text(json.dumps(rejected)) + save_disposition(scan, omitted.parent, "rejected") + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET merge_state = 'buffered' WHERE id = ?", + (omitted.parent.name,), + ) + if cut in {"before-selection", "selected"}: + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'reducing', " + "terminal_reason = NULL, completed_at = NULL WHERE scan_id = ?", + (scan.scan_id,), + ) + selection = ( + None + if cut == "before-selection" + else saved_selection(workbench_db, scan, accepted, omitted) + ) + staged = stage_publication( + scan, generation=3, result_path=accepted, title="Selected accepted aggregate" + ) + # The stop path must recover the accepted bytes and the rejection disposition. + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + evidence = {accepted: accepted.read_bytes(), omitted: omitted.read_bytes()} + database_path = tmp_path / "stop-publication.sqlite3" + with sqlite3.connect(database_path) as connection: + workbench_db.backup(connection) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + if cut in {"published", "sealed"}: + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, staged + ) + complete_args = Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + if cut == "sealed": + workbench_api["complete_scan"](connection, complete_args) + before_stop = published_bytes(scan) + if cut == "sealed": + with pytest.raises(SystemExit, match="running|completed"): + stop_scan(workbench_api, connection, scan, cause) + assert published_bytes(scan) == before_stop + else: + stop_scan(workbench_api, connection, scan, cause) + + # Reconnect after the winning commit, then deliver the old publisher response. + with sqlite3.connect(database_path) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + row = dict(connection.execute("SELECT * FROM scans").fetchone()) + run = dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) + assert row["status"] == ("complete" if cut == "sealed" else "failed") + assert bool(row["canceled_at"]) == (cause == "cancel" and cut != "sealed") + if cause == "cost" and cut != "sealed": + assert row["failure_message"] == ( + "Scan stopped after reaching the configured cost limit." + ) + recorded_selection = json.loads(run["finalization_input_json"] or "null") + if selection is None: + assert recorded_selection is None + else: + digest = recorded_selection.pop("publicationSha256", None) + assert recorded_selection == selection + if cut in {"published", "sealed"}: + assert isinstance(digest, str) and len(digest) == 64 + else: + assert digest is None + if selection is not None: + assert run["terminal_reason"] == selection["terminalReason"] + if cut != "sealed" and cause == "cancel": + assert run["status"] == "canceled" + if cut in {"published", "sealed"}: + assert run["terminal_reason"] == "saturated" + frozen = published_bytes(scan) + with pytest.raises(SystemExit, match="stopped"): + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, staged + ) + if cut == "sealed": + workbench_api["complete_scan"](connection, complete_args) + else: + with pytest.raises(SystemExit): + workbench_api["complete_scan"](connection, complete_args) + workbench_api["saved_results"].preserve_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], + connection, + Namespace( + scan_id=scan.scan_id, + claim_token=None, + thread_id=None, + coordinator_generation=None, + ), + ) + assert published_bytes(scan) == frozen + assert dict(connection.execute("SELECT * FROM scans").fetchone()) == row + assert dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) == run + assert all(path.read_bytes() == contents for path, contents in evidence.items()) + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == 1 + assert all( + finding.get("extensions", {}).get("candidateId") != "candidate-disposition" + for finding in findings + ) + if cut != "sealed": + published_coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert coverage["deferred"][0] in published_coverage["deferred"] + assert published_coverage["completeness"] == "partial" + + +@pytest.mark.parametrize("cause", ["cancel", "cost"]) +@pytest.mark.parametrize( + "boundary", + ["findings.json", "coverage.json", "scan-manifest.json", "sqlite-before", "sqlite-after"], +) +def test_stopped_publication_process_loss_keeps_frozen_rejection_and_original_selection( + workbench_api, workbench_db, publication_scan, tmp_path, cause, boundary +): + scan = publication_scan() + _, accepted, _ = accept_reducer(workbench_db, scan) + omitted = add_worker(workbench_db, scan) + reported = save_disposition(scan, omitted.parent, "reported") + omitted.write_text(json.dumps(reported)) + save_disposition(scan, omitted.parent, "rejected") + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET merge_state = 'buffered' WHERE id = ?", + (omitted.parent.name,), + ) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'reducing', " + "terminal_reason = NULL, completed_at = NULL WHERE scan_id = ?", + (scan.scan_id,), + ) + selection = saved_selection(workbench_db, scan, accepted, omitted) + staged = stage_publication( + scan, generation=3, result_path=accepted, title="Obsolete selected publication" + ) + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + evidence = {accepted: accepted.read_bytes(), omitted: omitted.read_bytes()} + database_path = tmp_path / "stopped-crash.sqlite3" + with sqlite3.connect(database_path) as connection: + workbench_db.backup(connection) + child = subprocess.run( + [ + sys.executable, + "-c", + _CRASH_STOPPED_PUBLICATION, + str(Path(__file__).resolve().parents[1] / "scripts" / "workbench_db.py"), + str(database_path), + scan.scan_id, + cause, + boundary, + ], + capture_output=True, + text=True, + ) + assert child.returncode == {"sqlite-before": 72, "sqlite-after": 73}.get(boundary, 71), ( + child.stdout, + child.stderr, + ) + with sqlite3.connect(database_path) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + row = connection.execute("SELECT * FROM scans").fetchone() + assert row["status"] == "failed" + assert bool(row["canceled_at"]) == (cause == "cancel") + assert bool(row["seal_manifest_digest"]) == (boundary == "sqlite-after") + frozen_sources = row["retained_source_digests_json"] + frozen_heads = row["retained_checkpoint_heads_json"] + assert frozen_sources and frozen_heads + run = dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) + workers = [dict(row) for row in connection.execute("SELECT * FROM deep_scan_workers")] + assert json.loads(run["finalization_input_json"]) == selection + assert run["terminal_reason"] == selection["terminalReason"] + assert run["status"] == ("canceled" if cause == "cancel" else "failed") + # The replacement process sees a different live head, but replays the + # already committed stopped selection instead of restoring the candidate. + save_disposition(scan, omitted.parent, "reported") + interrupted = published_bytes(scan) + with pytest.raises(SystemExit, match="stopped"): + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, staged + ) + assert published_bytes(scan) == interrupted + args = Namespace( + scan_id=scan.scan_id, + claim_token=None, + thread_id=None, + coordinator_generation=None, + ) + workbench_api["saved_results"].preserve_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, args + ) + row = connection.execute("SELECT * FROM scans").fetchone() + assert row["status"] == "failed" + assert bool(row["canceled_at"]) == (cause == "cancel") + assert row["retained_source_digests_json"] == frozen_sources + assert row["retained_checkpoint_heads_json"] == frozen_heads + assert row["seal_manifest_digest"] + assert dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) == run + assert [ + dict(row) for row in connection.execute("SELECT * FROM deep_scan_workers") + ] == workers + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == 1 + assert findings[0].get("extensions", {}).get("candidateId") != "candidate-disposition" + assert all(path.read_bytes() == contents for path, contents in evidence.items()) + sealed = published_bytes(scan) + workbench_api["saved_results"].preserve_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, args + ) + assert published_bytes(scan) == sealed + assert connection.execute("SELECT COUNT(*) FROM finding_occurrences").fetchone()[0] == 1 + + +@pytest.mark.parametrize("cut", ["before", "after"]) +def test_interrupted_selection_recovery_fences_observers_and_keeps_original_deadline( + workbench_api, workbench_db, publication_scan, tmp_path, cut +): + scan = publication_scan() + _, accepted, _ = accept_reducer(workbench_db, scan) + omitted = add_worker(workbench_db, scan) + selection = saved_selection(workbench_db, scan, accepted, omitted) + with workbench_db: + workbench_db.execute( + "UPDATE scans SET deep_scan_owner_thread_id = 'fixture-owner' WHERE id = ?", + (scan.scan_id,), + ) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', " + "completed_at = NULL, max_time_hours = 1, " + "created_at = '2000-01-01T00:00:00Z', updated_at = '2000-01-01T00:00:00Z' " + "WHERE scan_id = ?", + (scan.scan_id,), + ) + workbench_db.execute( + "UPDATE deep_scan_workers SET merge_state = 'buffered' WHERE id = ?", + (omitted.parent.name,), + ) + database_path = tmp_path / "recovery.sqlite3" + with sqlite3.connect(database_path) as connection: + workbench_db.backup(connection) + original = "\n".join(connection.iterdump()) + before = published_bytes(scan) + child = subprocess.run( + [ + sys.executable, + "-c", + _CRASH_SELECTION_RECOVERY, + str(Path(__file__).resolve().parents[1] / "scripts" / "workbench_db.py"), + str(database_path), + scan.scan_id, + cut, + ], + capture_output=True, + text=True, + ) + assert child.returncode == (72 if cut == "before" else 73), (child.stdout, child.stderr) + deep = workbench_api["deep_scan"] + args = Namespace( + scan_id=scan.scan_id, + thread_id="fixture-owner", + claim_token=None, + coordinator_generation=None, + ) + with sqlite3.connect(database_path) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + if cut == "before": + assert "\n".join(connection.iterdump()) == original + run = connection.execute("SELECT * FROM deep_scan_runs").fetchone() + assert run["coordinator_generation"] == (3 if cut == "before" else 4) + workers = [dict(row) for row in connection.execute("SELECT * FROM deep_scan_workers")] + attempts = [dict(row) for row in connection.execute("SELECT * FROM deep_scan_attempts")] + replayed = deep.claim_deep_scan_coordinator(connection, args) + assert replayed["coordinatorDisposition"] == ("adopted" if cut == "before" else "observing") + assert replayed["deepScan"]["coordinatorGeneration"] == 4 + assert replayed["deepScan"]["finalizationInput"] == selection + assert replayed["deepScan"]["createdAt"] == "2000-01-01T00:00:00Z" + assert replayed["deepScan"]["config"]["maxTimeHours"] == 1 + assert replayed["deepScan"]["phase"] == "terminal" + assert replayed["deepScan"]["terminalReason"] == selection["terminalReason"] + assert deep.deep_scan_deadline_reached( + connection.execute("SELECT * FROM deep_scan_runs").fetchone() + ) + stable = "\n".join(connection.iterdump()) + stale_claim = Namespace(**{**vars(args), "coordinator_generation": 3}) + with pytest.raises(SystemExit, match="generation"): + deep.claim_deep_scan_coordinator(connection, stale_claim) + assert "\n".join(connection.iterdump()) == stable + stale = stage_publication(scan, generation=3, result_path=accepted, title="Old coordinator") + with pytest.raises(SystemExit, match="generation"): + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, stale + ) + assert published_bytes(scan) == before + assert [ + dict(row) for row in connection.execute("SELECT * FROM deep_scan_workers") + ] == workers + assert [ + dict(row) for row in connection.execute("SELECT * FROM deep_scan_attempts") + ] == attempts + current = stage_publication( + scan, generation=4, result_path=accepted, title="Recovered selected aggregate" + ) + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], connection, current + ) + assert accepted.read_bytes() == before[accepted.relative_to(scan.scan_dir).as_posix()] + recorded_selection = json.loads( + connection.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + ) + digest = recorded_selection.pop("publicationSha256") + assert isinstance(digest, str) and len(digest) == 64 + assert recorded_selection == selection + # Successful host publication adds only its binding; every other database byte is stable. + with sqlite3.connect(":memory:") as comparison: + connection.backup(comparison) + comparison.execute( + "UPDATE deep_scan_runs SET finalization_input_json = ?", + (json.dumps(selection),), + ) + comparison.commit() + assert "\n".join(comparison.iterdump()) == stable diff --git a/plugins/codex-security/tests/test_report_projection.py b/plugins/codex-security/tests/test_report_projection.py index d797ce3268..69d54d22d6 100644 --- a/plugins/codex-security/tests/test_report_projection.py +++ b/plugins/codex-security/tests/test_report_projection.py @@ -61,6 +61,26 @@ def test_projection_normalizes_structured_fields() -> None: assert "Text: ## Injected remediation - unsafe instruction" in markdown +def test_linked_writeup_retains_distinct_source_fixes() -> None: + manifest, findings, coverage = canonical_documents() + finding = findings["findings"][0] + finding["writeup"] = {"reportPath": "findings/parser/parser.md"} + finding["remediation"] = "Validate the record length." + finding["provenance"] = { + "sourceFindings": [ + {"id": "review-1:0", "finding": {"remediation": "Validate the record length."}}, + {"id": "review-2:0", "finding": {"remediation": "Reject duplicate record keys."}}, + {"id": "review-3:0", "finding": {"remediation": "Reject duplicate record keys."}}, + ] + } + + markdown = PROJECTION.build_report_markdown(manifest, findings, coverage) + + assert "findings/parser/parser.md" in markdown + assert markdown.count("Validate the record length.") == 1 + assert markdown.count("Reject duplicate record keys.") == 1 + + def test_projection_renders_inline_code_and_section_code_evidence() -> None: manifest, findings, coverage = canonical_documents() finding = findings["findings"][0] @@ -730,8 +750,8 @@ def test_projection_explains_unvalidated_findings_after_cost_limit(reason: str) assert "| Reportable findings | 0 |" in markdown assert "| Coverage | partial |" in markdown assert ( - "No findings were validated before the scan reached its cost limit. " - "Review the deferred candidates in Open Questions And Follow Up." + "No findings are included in this partial report. " + "Review the unresolved work in Open Questions And Follow Up." ) in markdown assert "No reportable findings survived" not in markdown assert "## Open Questions And Follow Up" in markdown @@ -856,7 +876,22 @@ def test_projection_keeps_deferred_follow_up_with_open_questions() -> None: assert "Surfaces: parser-surface." in markdown -def test_projection_includes_surface_evidence_receipts() -> None: +@pytest.mark.parametrize( + ("details", "expected"), + [ + ({"notes": "Reviewed parser entrypoints."}, "Reviewed parser entrypoints."), + ({"reason": "Caller policy is unknown."}, "Caller policy is unknown."), + ( + {"notes": "Reviewed parser entrypoints.", "reason": "Caller policy is unknown."}, + "Reviewed parser entrypoints. Caller policy is unknown.", + ), + ({"notes": "Policy checked.", "reason": "Policy checked."}, "Policy checked."), + ({"notes": ""}, ""), + ({"notes": " \t "}, ""), + ({}, "No additional canonical notes were recorded."), + ], +) +def test_projection_includes_surface_evidence_receipts(details: dict, expected: str) -> None: manifest, findings, coverage = canonical_documents() coverage["surfaces"] = [ { @@ -864,10 +899,13 @@ def test_projection_includes_surface_evidence_receipts() -> None: "label": "Parser", "disposition": "no_issue_found", "receiptRefs": ["artifacts/receipts/parser.jsonl"], - "notes": "Reviewed parser entrypoints.", + **details, } ] markdown = PROJECTION.build_report_markdown(manifest, findings, coverage) - assert "Reviewed parser entrypoints. Evidence: artifacts/receipts/parser.jsonl" in markdown + row = next(line for line in markdown.splitlines() if "artifacts/receipts/parser.jsonl" in line) + assert row.endswith( + f"| {expected + ' ' if expected else ''}Evidence: artifacts/receipts/parser.jsonl |" + ) diff --git a/plugins/codex-security/tests/test_selected_publication_authority.py b/plugins/codex-security/tests/test_selected_publication_authority.py new file mode 100644 index 0000000000..c87d7304ac --- /dev/null +++ b/plugins/codex-security/tests/test_selected_publication_authority.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import hashlib +import json + +import pytest +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +@pytest.mark.parametrize("publication", ["selected", "mutable", "stale-generation", "unfenced"]) +def test_publication_uses_committed_finalization_selection( + workbench_api, workbench_db, publication_scan, publication +): + scan = publication_scan() + result = add_worker(workbench_db, scan) + contents = json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": scan.findings, + "coverage": scan.coverage, + } + ).encode() + digest = hashlib.sha256(contents).hexdigest() + accepted = result.parent / "accepted" / f"{digest}.json" + accepted.parent.mkdir() + accepted.write_bytes(contents) + selection = { + "version": 1, + "resultPath": accepted.relative_to(scan.scan_dir).as_posix(), + "resultSha256": digest, + "terminalReason": "saturated", + "omittedWorkerIds": [], + "selectedAt": scan.timestamp, + } + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none' " + "WHERE result_manifest_path = ?", + (str(result),), + ) + workbench_db.execute( + "UPDATE deep_scan_runs SET coordinator_generation = ?, finalization_input_json = ?, " + "workflow_version = 'deep-security-scan/v2' " + "WHERE scan_id = ?", + (1 if publication == "unfenced" else 3, json.dumps(selection), scan.scan_id), + ) + # The accepted bytes survive replacement or deletion of the worker's output. + result.unlink(missing_ok=True) + staged = stage_publication( + scan, + generation=None + if publication == "unfenced" + else 2 + if publication == "stale-generation" + else 3, + result_path=result if publication == "mutable" else accepted, + title="Selected accepted aggregate", + ) + before = {path: path.read_bytes() for path in scan.scan_dir.rglob("*.json")} + + if publication == "selected": + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, staged + ) + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert findings[0]["title"] == "Selected accepted aggregate" + else: + with pytest.raises(SystemExit, match="coordinator|publication|aggregate"): + workbench_api["saved_results"].write_scan_draft( + workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, staged + ) + assert {path: path.read_bytes() for path in scan.scan_dir.rglob("*.json")} == before + assert accepted.read_bytes() == contents + current = json.loads( + workbench_db.execute( + "SELECT finalization_input_json FROM deep_scan_runs WHERE scan_id = ?", + (scan.scan_id,), + ).fetchone()[0] + ) + if publication == "selected": + digest = current.pop("publicationSha256") + assert len(digest) == 64 and all(char in "0123456789abcdef" for char in digest) + current_scan = workbench_api["require_scan"](workbench_db, scan.scan_id) + db = workbench_api["_WORKBENCH_DB_CONTEXT"] + saved = workbench_api["saved_results"] + prepared = saved._prepare_scan_finalization( + scan.scan_dir, + expected_coverage_mode=db.expected_coverage_mode(current_scan), + completion_binding=db.workbench_completion_binding(current_scan, db.now()), + ) + saved.require_selected_publication(db, workbench_db, current_scan, prepared) + assert current == selection diff --git a/plugins/codex-security/tests/test_selected_publication_integrity.py b/plugins/codex-security/tests/test_selected_publication_integrity.py new file mode 100644 index 0000000000..42f1659ba7 --- /dev/null +++ b/plugins/codex-security/tests/test_selected_publication_integrity.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +import json +import os +import sqlite3 +from argparse import Namespace + +import pytest +from test_accepted_publication_references import accept_reducer +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_publication_stop_interleavings import published_bytes, saved_selection +from test_workbench_db import BUDGET_COST + + +def publish_selected(api, connection, scan): + _, accepted, coverage = accept_reducer(connection, scan) + scan.coverage = coverage + selection = saved_selection(connection, scan, accepted) + staged = stage_publication( + scan, generation=3, result_path=accepted, title=scan.findings[0]["title"] + ) + api["saved_results"].write_scan_draft(api["_WORKBENCH_DB_CONTEXT"], connection, staged) + recorded = json.loads( + connection.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + ) + assert {key: recorded[key] for key in selection} == selection + assert len(recorded["publicationSha256"]) == 64 + return accepted, recorded + + +def substitute(scan, accepted, defect): + if defect == "accepted-result": + path = accepted + document = json.loads(path.read_bytes()) + document["findings"][0]["title"] = "Substituted accepted finding" + else: + path = scan.scan_dir / ("findings.json" if defect == "findings" else "coverage.json") + document = json.loads(path.read_bytes()) + if defect == "findings": + document["findings"][0]["title"] = "Substituted valid canonical finding" + else: + document["deferred"][0]["reason"] = "Substituted valid unresolved review" + path.write_text(json.dumps(document)) + + +@pytest.mark.parametrize("defect", [None, "findings", "coverage", "accepted-result"]) +@pytest.mark.parametrize("persisted", [False, True], ids=["writer", "persisted-reader"]) +def test_sdk_completion_keeps_selected_publication_binding( + workbench_api, workbench_db, publication_scan, defect, persisted, request +): + scan = publication_scan() + accepted, recorded = publish_selected(workbench_api, workbench_db, scan) + if persisted: + # A resumed reader opens another connection to the persisted selected binding. + recovered = sqlite3.connect(":memory:") + recovered.row_factory = sqlite3.Row + recovered.execute("PRAGMA foreign_keys = ON") + workbench_db.backup(recovered) + request.addfinalizer(recovered.close) + workbench_db = recovered + if defect is not None: + substitute(scan, accepted, defect) + before = published_bytes(scan) + args = Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + + def complete(): + workbench_api["complete_scan"](workbench_db, args, prepare_only=True) + return workbench_api["complete_scan"](workbench_db, args) + + if defect is not None: + with pytest.raises(SystemExit, match="selected.*publication|changed|accepted"): + complete() + assert published_bytes(scan) == before + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "running" + else: + complete() + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "complete" + assert ( + json.loads((scan.scan_dir / "findings.json").read_text())["findings"][0]["title"] + == scan.findings[0]["title"] + ) + # Completion time and seal fields do not invalidate the selected projection. + frozen = published_bytes(scan) + complete() + assert published_bytes(scan) == frozen + assert ( + json.loads( + workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + ) + == recorded + ) + + +@pytest.mark.parametrize("defect", [None, "findings", "accepted-result"]) +@pytest.mark.parametrize("lower_bound", [False, True]) +def test_budget_completion_validates_then_binds_intentional_projection( + workbench_api, workbench_db, publication_scan, defect, lower_bound +): + scan = publication_scan() + accepted, recorded = publish_selected(workbench_api, workbench_db, scan) + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + if defect is not None: + substitute(scan, accepted, defect) + before = published_bytes(scan) + args = Namespace( + scan_id=scan.scan_id, + cost_json=json.dumps({"lowerBound": BUDGET_COST} if lower_bound else BUDGET_COST), + message="Synthetic scan reached its configured cost limit.", + ) + if defect is not None: + with pytest.raises(SystemExit, match="selected.*publication|changed|accepted"): + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + assert published_bytes(scan) == before + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "running" + assert ( + json.loads( + workbench_db.execute( + "SELECT finalization_input_json FROM deep_scan_runs" + ).fetchone()[0] + ) + == recorded + ) + else: + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + rebound = json.loads( + workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + ) + assert {key: rebound[key] for key in recorded if key != "publicationSha256"} == { + key: value for key, value in recorded.items() if key != "publicationSha256" + } + assert rebound["publicationSha256"] != recorded["publicationSha256"] + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "complete" + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert coverage["completeness"] == "partial" + assert any(item["id"] == "scan-cost-limit" for item in coverage["deferred"]) + + +@pytest.mark.parametrize("parent_newer_by_ns", [0, 1_000_000_000], ids=["tied", "newer"]) +def test_stopped_recovery_keeps_accepted_reviews_with_later_parent_coverage( + workbench_api, workbench_db, publication_scan, parent_newer_by_ns +): + scan = publication_scan() + _, accepted, coverage = accept_reducer(workbench_db, scan) + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + parent_coverage = scan.scan_dir / "coverage.json" + timestamp = accepted.stat().st_mtime_ns + parent_newer_by_ns + os.utime(parent_coverage, ns=(timestamp, timestamp)) + assert parent_coverage.stat().st_mtime_ns >= accepted.stat().st_mtime_ns + contents = accepted.read_bytes() + + stopped = workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace( + scan_id=scan.scan_id, + claim_token=None, + cost_json=None, + message="Synthetic stopped review.", + ), + )["scan"] + + assert stopped["findingCount"] == 1 + published = json.loads(parent_coverage.read_text()) + assert published["reviews"] == coverage["reviews"] + assert coverage["deferred"][0] in published["deferred"] + manifest = (scan.scan_dir / "scan-manifest.json").read_bytes() + workbench_api["saved_results"].preserve_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None + ), + ) + assert (scan.scan_dir / "scan-manifest.json").read_bytes() == manifest + assert accepted.read_bytes() == contents diff --git a/plugins/codex-security/tests/test_stopped_accepted_digests.py b/plugins/codex-security/tests/test_stopped_accepted_digests.py new file mode 100644 index 0000000000..b2d427377e --- /dev/null +++ b/plugins/codex-security/tests/test_stopped_accepted_digests.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import sqlite3 +from argparse import Namespace + +import pytest +from test_accepted_publication_references import accept_reducer +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +@pytest.mark.parametrize("recorded", [True, False], ids=["accepted-receipt", "legacy"]) +@pytest.mark.parametrize("changed", [False, True], ids=["original", "changed"]) +@pytest.mark.parametrize("historical", [False, True], ids=["current-attempt", "prior-attempt"]) +def test_stopped_recovery_checks_recorded_accepted_bytes( + workbench_api, workbench_db, publication_scan, tmp_path, recorded, changed, historical +): + scan = publication_scan() + result, accepted, _ = accept_reducer(workbench_db, scan) + checkpoint = result.parent / "checkpoints" / accepted.name + checkpoint.parent.mkdir() + accepted.rename(checkpoint) + original = checkpoint.read_bytes() + digest = hashlib.sha256(original).hexdigest() + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_attempts SET accepted_result_path = ? WHERE worker_id = ?", + (str(checkpoint), result.parent.name), + ) + if not recorded: + workbench_db.execute( + "DELETE FROM deep_scan_attempts WHERE worker_id = ?", (result.parent.name,) + ) + if historical: + workbench_db.execute( + "UPDATE deep_scan_workers SET attempt = 2, status = 'running' WHERE id = ?", + (result.parent.name,), + ) + if changed: + damaged = json.loads(original) + damaged["findings"][0]["summary"] = "Unaccepted changed evidence." + checkpoint.write_text(json.dumps(damaged)) + accepted_bytes = checkpoint.read_bytes() + # An unrelated valid source must still survive stopped partial preservation. + healthy = add_worker(workbench_db, scan) + finding = copy.deepcopy(scan.findings[0]) + finding["summary"] = "Independent preserved evidence." + finding["identity"]["anchor"] += ".independent" + finding["locations"][0]["startLine"] = 2 + finding["locations"][0]["endLine"] = 2 + healthy.write_text( + json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": [finding], + "coverage": scan.coverage, + } + ) + ) + healthy_bytes = healthy.read_bytes() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + database = tmp_path / "accepted-digests.sqlite3" + with sqlite3.connect(database) as connection: + workbench_db.backup(connection) + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + stopped = workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + connection, + Namespace( + scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped." + ), + )["scan"] + summaries = {finding["summary"] for finding in stopped["findings"]} + assert "Independent preserved evidence." in summaries + assert ("Unaccepted changed evidence." in summaries) == (changed and not recorded) + if not changed: + assert scan.findings[0]["summary"] in summaries + if recorded: + assert ( + connection.execute( + "SELECT accepted_result_sha256 FROM deep_scan_attempts WHERE worker_id = ?", + (result.parent.name,), + ).fetchone()[0] + == digest + ) + if recorded and changed: + warnings = json.loads( + connection.execute("SELECT completion_warnings_json FROM scans").fetchone()[0] + ) + assert any("changed after acceptance" in warning for warning in warnings) + assert checkpoint.read_bytes() == accepted_bytes + assert healthy.read_bytes() == healthy_bytes diff --git a/plugins/codex-security/tests/test_stopped_result_version_boundary.py b/plugins/codex-security/tests/test_stopped_result_version_boundary.py new file mode 100644 index 0000000000..334f44ba0d --- /dev/null +++ b/plugins/codex-security/tests/test_stopped_result_version_boundary.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import hashlib +import json +from argparse import Namespace + +import pytest +from test_checkpoint_publication_authority import save_disposition +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +def snapshot(connection, scan_dir): + return { + "database": "\n".join(connection.iterdump()), + "files": { + path.relative_to(scan_dir).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() + for path in scan_dir.rglob("*") + if path.is_file() + }, + } + + +@pytest.mark.parametrize("operation", ["preserve", "recover"]) +@pytest.mark.parametrize( + "protocol", + ["supported", "supported-v2", "supported-mcp-v1", "future-workflow", "future-selection"], +) +def test_stopped_result_publication_requires_supported_protocol( + workbench_api, workbench_db, publication_scan, monkeypatch, operation, protocol +): + scan = publication_scan() + supported_workflows = { + "supported": "deep-security-scan/v1", + "supported-v2": "deep-security-scan/v2", + "supported-mcp-v1": "deep-scan-mcp/v1", + } + if protocol in supported_workflows: + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET workflow_version = ? WHERE scan_id = ?", + (supported_workflows[protocol], scan.scan_id), + ) + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + result = add_worker(workbench_db, scan, status="canceled") + draft = save_disposition(scan, result.parent, "reported") + result.write_text(json.dumps(draft)) + + def interrupt_publication(*args, **kwargs): + raise OSError("Synthetic publication interruption") + + with monkeypatch.context() as patch: + patch.setattr( + workbench_api["saved_results"], + "_write_prepared_scan_finalization", + interrupt_publication, + ) + workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace( + scan_id=scan.scan_id, + claim_token=None, + cost_json=None, + message="Original worker stop.", + ), + ) + row = workbench_db.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone() + assert row["status"] == "failed" + assert row["retained_source_digests_json"] + assert row["seal_manifest_digest"] is None + + with workbench_db: + if protocol == "future-workflow": + workbench_db.execute( + "UPDATE deep_scan_runs SET workflow_version = 'future/v99' WHERE scan_id = ?", + (scan.scan_id,), + ) + elif protocol == "future-selection": + workbench_db.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "finalization_input_json = ? WHERE scan_id = ?", + (json.dumps({"version": 99}), scan.scan_id), + ) + before = snapshot(workbench_db, scan.scan_dir) + error = None + try: + if operation == "preserve": + workbench_api["saved_results"].preserve_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace( + scan_id=scan.scan_id, + claim_token=None, + thread_id=None, + coordinator_generation=None, + ), + ) + else: + workbench_api["saved_results"].recover_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace(scan_id=scan.scan_id), + ) + except SystemExit as failure: + error = str(failure) + after = snapshot(workbench_db, scan.scan_dir) + changed_files = sorted( + path + for path in before["files"].keys() | after["files"].keys() + if before["files"].get(path) != after["files"].get(path) + ) + print( + json.dumps( + { + "operation": operation, + "protocol": protocol, + "error": error, + "database_changed": before["database"] != after["database"], + "changed_files": changed_files, + } + ) + ) + if protocol in supported_workflows: + assert error is None + assert before != after + row = workbench_db.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone() + assert row["seal_manifest_digest"] + assert row["failure_message"] == "Original worker stop." + run = workbench_db.execute( + "SELECT workflow_version FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,) + ).fetchone() + assert run["workflow_version"] == supported_workflows[protocol] + else: + assert error is not None and "unsupported" in error.lower() + assert after == before diff --git a/plugins/codex-security/tests/test_stopped_source_coverage.py b/plugins/codex-security/tests/test_stopped_source_coverage.py new file mode 100644 index 0000000000..3f5ddee647 --- /dev/null +++ b/plugins/codex-security/tests/test_stopped_source_coverage.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import copy +import json +from argparse import Namespace + +import pytest +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +@pytest.mark.parametrize("host_coverage", [True, False], ids=["accepted-projection", "legacy"]) +@pytest.mark.parametrize("parent_draft", [True, False], ids=["parent-draft", "no-parent"]) +def test_stopped_recovery_preserves_accepted_coverage_without_worker_id_collisions( + workbench_api, workbench_db, publication_scan, host_coverage, parent_draft +): + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + source_coverage = { + "completeness": "partial", + "surfaces": [], + "explicitExclusions": [], + "deferred": [], + "reviews": [], + } + source_files = [] + for disposition in ("needs_follow_up", "rejected"): + result = add_worker(workbench_db, scan) + worker_id = result.parent.name + surface = { + "id": "surface-1", + "candidateId": "candidate-1", + "label": "Independent review", + "disposition": disposition, + "receiptRefs": [], + } + deferred = {"candidateId": "candidate-1", "reason": "Validation remains unresolved."} + coverage = { + "completeness": "partial" if disposition == "needs_follow_up" else "complete", + "surfaces": [surface], + "explicitExclusions": [], + "deferred": [deferred] if disposition == "needs_follow_up" else [], + } + result.write_text( + json.dumps( + {"scanId": scan.scan_id, "complete": True, "findings": [], "coverage": coverage} + ) + ) + source_files.append(result) + prefix = f"{worker_id}-attempt-1" + provenance = {"workerId": worker_id, "attempt": 1, "candidateId": "candidate-1"} + source_coverage["reviews"].append( + {"workerId": worker_id, "attempt": 1, "completeness": coverage["completeness"]} + ) + source_coverage["surfaces"].append( + { + **surface, + "id": f"{prefix}-surface-1", + "provenance": {**provenance, "sourceId": "surface-1"}, + } + ) + if coverage["deferred"]: + source_coverage["deferred"].append( + { + **deferred, + "id": f"{prefix}-deferred-1", + "candidateId": f"{prefix}-candidate-1", + "provenance": provenance, + } + ) + reducer = add_worker(workbench_db, scan) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none' " + "WHERE result_manifest_path = ?", + (str(reducer),), + ) + aggregate = {"scanId": scan.scan_id, "complete": True, "findings": []} + if host_coverage: + aggregate["sourceCoverage"] = copy.deepcopy(source_coverage) + reducer.write_text(json.dumps(aggregate)) + source_files.append(reducer) + saved_bytes = {path: path.read_bytes() for path in source_files} + if not parent_draft: + for filename in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / filename).unlink() + + stopped = workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."), + )["scan"] + + assert stopped["progress"]["status"] == "failed" + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert coverage["completeness"] == "partial" + assert len(coverage["deferred"]) == 2 + assert coverage["deferred"][-1]["id"] == "scan-stopped" + assert len(coverage["surfaces"]) == 2 + if host_coverage: + for field in ("reviews", "surfaces", "deferred"): + assert ( + coverage[field][:-1] if field == "deferred" else coverage[field] + ) == source_coverage[field] + else: + assert coverage["deferred"][0]["candidateId"] == "candidate-1" + manifest = (scan.scan_dir / "scan-manifest.json").read_bytes() + workbench_api["saved_results"].preserve_scan_results( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None + ), + ) + assert (scan.scan_dir / "scan-manifest.json").read_bytes() == manifest + assert all(path.read_bytes() == contents for path, contents in saved_bytes.items()) + + +def test_stopped_recovery_keeps_unmerged_coverage_after_accepted_review( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + accepted = add_worker(workbench_db, scan) + accepted.write_text( + json.dumps( + {"scanId": scan.scan_id, "complete": True, "findings": [], "coverage": scan.coverage} + ) + ) + reducer = add_worker(workbench_db, scan) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none' " + "WHERE result_manifest_path = ?", + (str(reducer),), + ) + reviews = [{"workerId": accepted.parent.name, "attempt": 1, "completeness": "complete"}] + reducer.write_text( + json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": [], + "sourceCoverage": {**scan.coverage, "reviews": reviews}, + } + ) + ) + pending = add_worker(workbench_db, scan, status="canceled") + deferred = {"id": "pending-review", "reason": "The independent review remains unresolved."} + pending.write_text( + json.dumps( + { + "scanId": scan.scan_id, + "complete": False, + "findings": [], + "coverage": {**scan.coverage, "completeness": "partial", "deferred": [deferred]}, + } + ) + ) + + workbench_api["saved_results"].fail_scan( + workbench_api["_WORKBENCH_DB_CONTEXT"], + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."), + ) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert coverage["completeness"] == "partial" + assert coverage["reviews"] == reviews + assert deferred in coverage["deferred"] diff --git a/plugins/codex-security/tests/test_workbench_completion_binding.py b/plugins/codex-security/tests/test_workbench_completion_binding.py index ff27188ad7..e08f1ee416 100644 --- a/plugins/codex-security/tests/test_workbench_completion_binding.py +++ b/plugins/codex-security/tests/test_workbench_completion_binding.py @@ -55,6 +55,40 @@ def _start_deep_scan_with_draft_findings(tmp_path: Path) -> tuple[Path, str, Pat return state_dir, scan_id, scan_dir +@pytest.mark.parametrize("completeness", ["partial", "complete", "unknown"]) +def test_deep_completion_returns_sealed_coverage_on_replay( + tmp_path: Path, completeness: str +) -> None: + state_dir, scan_id, scan_dir = _start_deep_scan_with_draft_findings(tmp_path) + coverage_path = scan_dir / "coverage.json" + coverage = json.loads(coverage_path.read_text()) + coverage["completeness"] = completeness + coverage["deferred"] = ( + [{"id": "external-control", "reason": "Deployment configuration is unavailable."}] + if completeness == "partial" + else [] + ) + coverage_path.write_text(json.dumps(coverage)) + completed = run_workbench(state_dir, "complete-scan", "--scan-id", scan_id) + canonical = json.loads(coverage_path.read_text()) + assert canonical["completeness"] == completeness + assert len(canonical["deferred"]) == (1 if completeness == "partial" else 0) + expected = { + "completeness": canonical["completeness"], + "surfaceCount": len(canonical["surfaces"]), + "deferredCount": len(canonical["deferred"]), + "explicitExclusionCount": len(canonical["explicitExclusions"]), + } + artifacts = ["scan-manifest.json", "findings.json", "coverage.json", "report.md"] + sealed = [(scan_dir / name).read_bytes() for name in artifacts] + replayed = run_workbench(state_dir, "complete-scan", "--scan-id", scan_id) + for result in (completed, replayed): + assert result["scan"]["progress"]["status"] == "complete" + assert result["coverageSummary"] == expected + assert result["scan"]["usage"] == completed["scan"]["usage"] + assert [(scan_dir / name).read_bytes() for name in artifacts] == sealed + + def register_cli_scan( state_dir: Path, target: Path, diff --git a/plugins/codex-security/tests/test_workbench_db.py b/plugins/codex-security/tests/test_workbench_db.py index 5b679d25c9..af37f19e08 100644 --- a/plugins/codex-security/tests/test_workbench_db.py +++ b/plugins/codex-security/tests/test_workbench_db.py @@ -14,6 +14,7 @@ from typing import Any import pytest +import workbench_test_support from workbench_test_support import ( SCRIPT, create_saved_git_workspace, @@ -59,6 +60,9 @@ BUDGET_WARNING = "Scan stopped: estimated cost $0.00625 exceeded the $0.005 cost limit." EXPECTED_TABLES = { + "deep_scan_attempts", + "deep_scan_attempt_sessions", + "deep_scan_merge_claims", "deep_scan_dedup_inputs", "deep_scan_runs", "deep_scan_workers", @@ -173,6 +177,198 @@ def budget_scan_fixture( return state_dir, target, scan_dir, scan_id, ledger +@pytest.mark.parametrize("operation", ["complete-scan", "complete-budget-exhausted-scan"]) +@pytest.mark.parametrize("protocol", ["supported", "future-workflow", "future-selection"]) +def test_completion_rejects_unknown_protocol_before_mutation( + workbench_api, monkeypatch, tmp_path, operation, protocol +): + script = str(workbench_api["__file__"]) + monkeypatch.setattr(workbench_test_support, "SCRIPT", script) + monkeypatch.setenv("CODEX_HOME", str(tmp_path / "home")) + state_dir, _, scan_dir, scan_id, _ = budget_scan_fixture(tmp_path) + environment = {**os.environ, "CODEX_SECURITY_STATE_DIR": str(state_dir)} + cost_args = ["--scan-id", scan_id, "--cost-json", json.dumps(BUDGET_COST)] + if operation == "complete-scan": + cut_program = """ +import os, runpy, sys +script, *args = sys.argv[1:] +api = runpy.run_path(script, run_name="completion_version_test") +namespace = api["main"].__globals__ +original = namespace["_write_prepared_scan_finalization"] +def after_seal(*args, **kwargs): + original(*args, **kwargs) + os._exit(86) +namespace["_write_prepared_scan_finalization"] = after_seal +sys.argv = [script, *args] +api["main"]() +""" + cut = subprocess.run( + [ + sys.executable, + "-I", + "-B", + "-c", + cut_program, + script, + "complete-budget-exhausted-scan", + *cost_args, + "--message", + BUDGET_WARNING, + ], + env=environment, + capture_output=True, + text=True, + ) + assert cut.returncode == 86, cut.stderr + assert json.loads((scan_dir / "scan-manifest.json").read_text())["scan"]["sealedAt"] + database = state_dir / "workbench.sqlite3" + with sqlite3.connect(database) as connection: + assert connection.execute("SELECT status FROM scans").fetchone() == ("running",) + if protocol == "future-workflow": + connection.execute("UPDATE deep_scan_runs SET workflow_version = 'future/v99'") + elif protocol == "future-selection": + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "finalization_input_json = ?", + (json.dumps({"version": 99}),), + ) + + def snapshot(): + with sqlite3.connect(database) as connection: + return list(connection.iterdump()), { + str(path.relative_to(scan_dir)): path.read_bytes() + for path in scan_dir.rglob("*") + if path.is_file() + } + + before = snapshot() + result = subprocess.run( + [ + sys.executable, + "-I", + "-B", + script, + operation, + *cost_args, + *([] if operation == "complete-scan" else ["--message", BUDGET_WARNING]), + ], + env=environment, + capture_output=True, + text=True, + ) + after = snapshot() + if protocol == "supported": + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout)["scan"]["progress"]["status"] == "complete" + if operation == "complete-scan": + assert after[1] == before[1] + else: + assert result.returncode != 0 + assert "unsupported" in result.stderr.lower() + assert after == before + + +@pytest.mark.parametrize("legacy_digest", [False, True]) +@pytest.mark.parametrize( + "protocol", ["supported", "legacy-no-run", "future-workflow", "future-selection"] +) +def test_completed_replay_checks_protocol_before_cost_or_digest_writes( + workbench_api, monkeypatch, tmp_path, protocol, legacy_digest +): + monkeypatch.setattr(workbench_test_support, "SCRIPT", workbench_api["__file__"]) + monkeypatch.setenv("CODEX_HOME", str(tmp_path / "home")) + state_dir, _, scan_dir, scan_id, _ = budget_scan_fixture(tmp_path) + complete_budget_scan(state_dir, scan_id) + database = state_dir / "workbench.sqlite3" + with sqlite3.connect(database) as connection: + assert connection.execute("SELECT status FROM scans").fetchone() == ("complete",) + if legacy_digest: + connection.execute("UPDATE scans SET seal_manifest_digest = NULL") + if protocol == "future-workflow": + connection.execute("UPDATE deep_scan_runs SET workflow_version = 'future/v99'") + elif protocol == "future-selection": + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "finalization_input_json = ?", + (json.dumps({"version": 99}),), + ) + elif protocol == "legacy-no-run": + connection.execute("DELETE FROM deep_scan_runs") + + def snapshot(): + with sqlite3.connect(database) as connection: + return list(connection.iterdump()), { + str(path.relative_to(scan_dir)): path.read_bytes() + for path in scan_dir.rglob("*") + if path.is_file() + } + + before = snapshot() + cost = {**BUDGET_COST, "inputTokens": 1500, "estimatedUsd": 0.0075} + result = run_workbench( + state_dir, + "complete-scan", + "--scan-id", + scan_id, + "--cost-json", + json.dumps(cost), + check=not protocol.startswith("future-"), + ) + after = snapshot() + if protocol.startswith("future-"): + assert after == before + assert result["returncode"] != 0 + assert "unsupported" in result["stderr"].lower() + else: + assert result["scan"]["progress"]["status"] == "complete" + assert result["scan"]["cost"] == cost + assert after[1] == before[1] + with sqlite3.connect(database) as connection: + assert connection.execute("SELECT seal_manifest_digest FROM scans").fetchone()[0] + + +@pytest.mark.parametrize("terminal", [False, True]) +@pytest.mark.parametrize("continuation", ["current", "pending", "claimed"]) +def test_budget_completion_checks_continuation_before_draft_writes( + workbench_api, monkeypatch, tmp_path, terminal, continuation +): + monkeypatch.setattr(workbench_test_support, "SCRIPT", workbench_api["__file__"]) + monkeypatch.setenv("CODEX_HOME", str(tmp_path / "home")) + state_dir, _, scan_dir, scan_id, _ = budget_scan_fixture(tmp_path, terminal=terminal) + database = state_dir / "workbench.sqlite3" + with sqlite3.connect(database) as connection: + connection.execute("UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2'") + if continuation != "current": + connection.execute("UPDATE scans SET handoff_status = 'pending'") + if continuation == "claimed": + run_workbench( + state_dir, + "claim-handoff-delivery", + "--scan-id", + scan_id, + "--claim-token", + str(uuid.uuid4()), + ) + + def snapshot(): + with sqlite3.connect(database) as connection: + return list(connection.iterdump()), { + str(path.relative_to(scan_dir)): path.read_bytes() + for path in scan_dir.rglob("*") + if path.is_file() + } + + before = snapshot() + result = complete_budget_scan(state_dir, scan_id, check=continuation == "current") + after = snapshot() + if continuation == "current": + assert result["scan"]["progress"]["status"] == "complete" + else: + assert after == before + assert result["returncode"] != 0 + assert "owned by another continuation" in result["stderr"] + + def complete_budget_scan(state_dir: Path, scan_id: str, *, check: bool = True) -> dict[str, object]: return run_workbench( state_dir, @@ -261,7 +457,7 @@ def test_budget_exhaustion_preserves_unvalidated_discovery_as_deferred_work( assert "User input reaches a SQL statement" in coverage["deferred"][0]["reason"] assert coverage["surfaces"][0]["disposition"] == "needs_follow_up" report = (scan_dir / "report.md").read_text() - assert "No findings were validated before the scan reached its cost limit" in report + assert "No findings are included in this partial report" in report assert "User input reaches a SQL statement" in report @@ -401,10 +597,7 @@ def test_budget_exhaustion_preserves_existing_deferred_work_with_one_trusted_mar "id": "scan-cost-limit", "reason": "Validation was deferred because the scan reached its cost limit.", } - assert ( - "No findings were validated before the scan reached its cost limit" - in (scan_dir / "report.md").read_text() - ) + assert "No findings are included in this partial report" in (scan_dir / "report.md").read_text() def test_budget_exhaustion_preserves_capped_discovery(tmp_path: Path) -> None: @@ -465,17 +658,33 @@ def test_budget_exhaustion_rejects_scan_below_configured_limit(tmp_path: Path) - assert "has not exceeded its configured cost limit" in str(rejected["stderr"]) -def test_budget_exhaustion_rejects_incomplete_discovery(tmp_path: Path) -> None: - state_dir, _, _, scan_id, _ = budget_scan_fixture(tmp_path, terminal=False) - - rejected = complete_budget_scan(state_dir, scan_id, check=False) - - assert rejected["returncode"] != 0 - assert "requires successfully completed Deep Scan discovery" in str(rejected["stderr"]) - assert ( - run_workbench(state_dir, "get-scan", "--scan-id", scan_id)["scan"]["progress"]["status"] - == "running" - ) +@pytest.mark.parametrize("workflow", ["deep-security-scan/v1", "deep-security-scan/v2"]) +def test_budget_exhaustion_during_discovery_preserves_workflow_contract( + tmp_path: Path, workflow: str +) -> None: + state_dir, _, scan_dir, scan_id, ledger = budget_scan_fixture(tmp_path, terminal=False) + original = ledger.read_bytes() + with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = ? WHERE scan_id = ?", (workflow, scan_id) + ) + result = complete_budget_scan(state_dir, scan_id, check=False) + if workflow == "deep-security-scan/v1": + assert result["returncode"] != 0 + assert "requires successfully completed Deep Scan discovery" in str(result["stderr"]) + assert ( + run_workbench(state_dir, "get-scan", "--scan-id", scan_id)["scan"]["progress"]["status"] + == "running" + ) + else: + assert result["returncode"] == 0 + saved = run_workbench(state_dir, "get-scan", "--scan-id", scan_id)["scan"] + assert saved["progress"]["status"] == "complete" + assert saved["findings"] == [] + assert saved["cost"] == BUDGET_COST + assert saved["warnings"] == [BUDGET_WARNING] + assert json.loads((scan_dir / "coverage.json").read_text())["completeness"] == "partial" + assert ledger.read_bytes() == original def test_budget_exhaustion_rejects_standard_scan(tmp_path: Path) -> None: @@ -1017,7 +1226,7 @@ def test_workbench_persists_progress_and_indexes_completed_findings(tmp_path: Pa ) } assert tables == EXPECTED_TABLES - assert connection.execute("SELECT COUNT(*) FROM schema_migrations").fetchone() == (41,) + assert connection.execute("SELECT COUNT(*) FROM schema_migrations").fetchone() == (47,) assert connection.execute("SELECT COUNT(*) FROM findings").fetchone() == (1,) assert connection.execute("SELECT COUNT(*) FROM finding_locations").fetchone() == (1,) diff --git a/plugins/codex-security/tests/test_workbench_deep_scan.py b/plugins/codex-security/tests/test_workbench_deep_scan.py index f32e9cdd9d..faeed8405d 100644 --- a/plugins/codex-security/tests/test_workbench_deep_scan.py +++ b/plugins/codex-security/tests/test_workbench_deep_scan.py @@ -278,7 +278,9 @@ def claim() -> dict[str, object]: return claim_deep_scan_coordinator(state_dir, codex_home, scan_id) with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: - assert connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone() == (41,) + migrations = connection.execute( + "SELECT * FROM schema_migrations ORDER BY version" + ).fetchall() assert claim()["deepScan"]["coordinatorGeneration"] == 2 assert claim()["coordinatorDisposition"] == "observing" expire_deep_scan_coordinator(state_dir, scan_id) @@ -310,6 +312,11 @@ def claim() -> dict[str, object]: assert sum(result["coordinatorDisposition"] == "adopted" for result in results) == 1 assert sum(result["coordinatorDisposition"] == "observing" for result in results) == 3 assert {result["deepScan"]["coordinatorGeneration"] for result in results} == {3} + with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: + assert ( + connection.execute("SELECT * FROM schema_migrations ORDER BY version").fetchall() + == migrations + ) def test_legacy_generation_with_active_worker_observes_grace_then_adopts(tmp_path: Path) -> None: @@ -1867,10 +1874,21 @@ def test_failed_reducer_rebuffers_claimed_inputs_for_same_generation_replacement error="fixture reducer exhausted its attempts", coordinator_generation=2, )["deepScan"] - replayed_workers = {worker["id"]: worker for worker in replayed["workers"]} + assert replayed["workerReceipt"] == failed["workerReceipt"] assert replayed["phase"] == "reducing" - assert replayed["consecutiveErrors"] == counter_before_failure - assert all(replayed_workers[worker]["mergeState"] == "merging" for worker in replacement_inputs) + current = run_workbench( + state_dir, + "get-deep-scan", + "--scan-id", + scan_id, + "--thread-id", + "thread-deep-scan", + environment=deep_environment(codex_home), + )["deepScan"] + current_workers = {worker["id"]: worker for worker in current["workers"]} + assert current["phase"] == "reducing" + assert current["consecutiveErrors"] == counter_before_failure + assert all(current_workers[worker]["mergeState"] == "merging" for worker in replacement_inputs) upsert_worker( state_dir, diff --git a/plugins/codex-security/tests/test_workbench_scan_usage.py b/plugins/codex-security/tests/test_workbench_scan_usage.py index 7d76b3cada..5cfc36356f 100644 --- a/plugins/codex-security/tests/test_workbench_scan_usage.py +++ b/plugins/codex-security/tests/test_workbench_scan_usage.py @@ -1,11 +1,14 @@ from __future__ import annotations import json +import os import runpy import sqlite3 +import subprocess import sys import tempfile import uuid +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from datetime import datetime, timedelta from pathlib import Path @@ -13,6 +16,7 @@ import pytest from workbench_test_support import ( + SCRIPT, create_saved_workspace, initialize_git_repository, mark_deep_coordinator_succeeded, @@ -492,6 +496,32 @@ def test_completion_reports_unavailable_without_fabricating_zero(tmp_path: Path) assert "totalTokens" not in usage +@pytest.mark.parametrize("reported", ["missing", "null-counter", "explicit-zero"]) +def test_completion_distinguishes_missing_token_records_from_zero( + tmp_path: Path, reported: str +) -> None: + fixture = _start_scan(tmp_path) + counted = fixture.started_at + timedelta(microseconds=1) + events = [] + if reported == "explicit-zero": + events.append(_token_event(counted, 0, 0)) + elif reported == "null-counter": + events.append( + _event(counted, "event_msg", {"type": "token_count", "info": None, "rate_limits": None}) + ) + parent = _rollout(tmp_path, "scan-parent", events) + _state_graph(fixture.environment, {"scan-parent": parent}, []) + usage = _complete_scan(fixture)["scan"]["usage"] + if reported == "explicit-zero": + assert usage["coverage"] == "complete" + assert usage["totalTokens"] == 0 + else: + assert usage["coverage"] == "unavailable" + assert "token_usage_unavailable" in usage["warnings"] + assert "token_record_invalid" not in usage["warnings"] + assert "totalTokens" not in usage + + @pytest.mark.skipif(sys.platform != "darwin", reason="macOS system path aliases") @pytest.mark.parametrize("temporary_root", [tempfile.gettempdir(), "/tmp"], ids=["var", "tmp"]) def test_completion_accepts_macos_system_rollout_alias( @@ -605,11 +635,354 @@ def test_completion_counts_deep_sdk_workers_and_descendants(tmp_path: Path) -> N ) usage = _complete_scan(fixture)["scan"]["usage"] assert usage == { - "coverage": "complete", + "coverage": "partial", "source": "codex_rollout", - **_counts(37, 0, 10), - "threadCount": 3, + **_counts(27, 0, 7), + "threadCount": 2, + "missingThreadCount": 1, + "warnings": ["scan_owner_turn_unavailable"], + "modelUsage": [{"model": None, **_counts(27, 0, 7)}], + } + + +@pytest.mark.parametrize( + "worker_home", + [ + "recorded", + "current", + "inherited-sqlite", + "current-prefix", + "recorded-prefix", + "current-equal-incomplete", + "recorded-equal-incomplete", + "current-unreadable", + "current-mismatched", + "external-sqlite", + "external-shared-home", + "external-missing-copy", + "external-missing-child", + "external-current-middle-missing", + "external-current-middle-complete", + "unavailable", + ], +) +def test_completion_keeps_owner_and_workers_in_their_recorded_homes( + tmp_path: Path, worker_home: str +) -> None: + current_home = tmp_path / "current-home" + copied_rollout = worker_home in { + "current-prefix", + "recorded-prefix", + "current-equal-incomplete", + "recorded-equal-incomplete", + "current-unreadable", + "current-mismatched", } + environment = { + "CODEX_HOME": str(current_home), + "CODEX_SQLITE_HOME": str(current_home / "sqlite"), + "CODEX_STATE_DB": str(current_home / "sqlite" / "state_5.sqlite"), + } + owners = { + f"owner-{index}": _rollout( + tmp_path, + f"owner-{index}", + [_event(datetime.now().astimezone(), "turn_context", {"turn_id": "original"})], + ) + for index in (1, 2) + } + _state_graph(environment, owners, []) + + def complete(index: int) -> dict[str, Any]: + root = tmp_path / f"scan-{index}" + target = root / "target" + target.mkdir(parents=True) + selected_home = current_home if worker_home == "current" else root / "original-home" + if worker_home == "external-shared-home": + selected_home = tmp_path / "shared-original-home" + process = subprocess.run( + [ + sys.executable, + "-c", + ( + "import runpy, sys; script = sys.argv.pop(1); " + "runpy.run_path(script)['main'](with_execution_settings=True)" + ), + str(SCRIPT), + "begin-deep-scan", + "--thread-id", + f"owner-{index}", + "--target-path", + str(target), + "--scan-root", + str(root / "scans"), + ], + env={**os.environ, **environment, "CODEX_SECURITY_STATE_DIR": str(root / "state")}, + input=json.dumps( + { + "executionSettings": { + "codexHome": str(selected_home), + "codexPath": sys.executable, + } + } + ), + capture_output=True, + text=True, + timeout=30, + ) + assert process.returncode == 0, process.stderr + deep = json.loads(process.stdout)["deepScan"] + scan_id, scan_dir = deep["scanId"], Path(deep["scanDir"]) + snapshot = scan_dir / "artifacts/deep_discovery/execution-settings.json" + original_bytes = snapshot.read_bytes() + fixture = ScanFixture( + root / "state", + target, + scan_id, + scan_dir, + datetime.fromisoformat(deep["createdAt"]), + environment, + "deep", + ) + counted = fixture.started_at + timedelta(microseconds=1) + with owners[f"owner-{index}"].open("a") as stream: + stream.write(json.dumps(_token_event(counted, index * 10, 2)) + "\n") + stream.write(json.dumps(_event(counted, "turn_context", {"turn_id": "later"})) + "\n") + stream.write(json.dumps(_token_event(counted, 9000, 900)) + "\n") + worker_threads = {} + for kind in ("discovery", "second-discovery"): + thread_id = f"{kind}-{index}" + artifact = scan_dir / "artifacts" / thread_id + artifact.mkdir() + prompt = artifact / "prompt.md" + prompt.write_text("Review the synthetic target.\n") + run_workbench( + fixture.state_dir, + "upsert-deep-scan-worker", + "--scan-id", + scan_id, + "--worker-id", + str(uuid.uuid4()), + "--kind", + "discovery", + "--status", + "running", + "--prompt-path", + str(prompt), + "--artifact-dir", + str(artifact), + "--sdk-thread-id", + thread_id, + environment=environment, + ) + worker_threads[thread_id] = _rollout( + root, + thread_id, + [ + *( + [_event(counted, "turn_context", {"model": "model-alpha"})] + if copied_rollout and kind == "discovery" + else [] + ), + _token_event(counted, index * 20, 3), + _event( + counted, + "turn_context", + { + "turn_id": "resumed", + **( + {"model": "model-beta"} + if copied_rollout and kind == "discovery" + else {} + ), + }, + ), + _token_event(counted, index * 30, 5), + ], + ) + child_id = f"child-{index}" + worker_threads[child_id] = _rollout( + root, + child_id, + [_token_event(counted, index * 7, 1)], + parent_thread_id=f"discovery-{index}", + ) + if worker_home in {"current", "inherited-sqlite"}: + with sqlite3.connect(environment["CODEX_STATE_DB"]) as connection: + connection.executemany( + "INSERT INTO threads VALUES (?, ?)", + [(key, str(path)) for key, path in worker_threads.items()], + ) + connection.execute( + "INSERT INTO thread_spawn_edges VALUES (?, ?)", (f"discovery-{index}", child_id) + ) + if worker_home == "inherited-sqlite": + # Earlier launches used A; the resumed process forwards its + # explicit SQLite home C even while workers keep Codex home A. + _state_graph( + {"CODEX_SQLITE_HOME": str(selected_home)}, + {f"discovery-{index}": worker_threads[f"discovery-{index}"]}, + [], + ) + elif worker_home in { + "recorded", + "current-prefix", + "recorded-prefix", + "current-equal-incomplete", + "recorded-equal-incomplete", + "current-unreadable", + "current-mismatched", + }: + recorded_threads = dict(worker_threads) + if worker_home != "recorded": + thread_id = f"discovery-{index}" + full = worker_threads[thread_id] + copied = full.with_name(f"copied-{thread_id}.jsonl") + copied.write_bytes(b"\n".join(full.read_bytes().splitlines()[:3]) + b"\n") + if worker_home in {"current-equal-incomplete", "recorded-equal-incomplete"}: + copied.write_bytes(full.read_bytes() + b'{"type":"event_msg"') + if worker_home == "current-unreadable": + copied.write_text("invalid session metadata\n") + elif worker_home == "current-mismatched": + copied.write_text(full.read_text().replace(thread_id, "unrelated-thread")) + current_copy = copied + if worker_home in {"recorded-prefix", "recorded-equal-incomplete"}: + recorded_threads[thread_id] = copied + current_copy = full + with sqlite3.connect(environment["CODEX_STATE_DB"]) as connection: + connection.execute( + "INSERT INTO threads VALUES (?, ?)", (thread_id, str(current_copy)) + ) + _state_graph( + {"CODEX_SQLITE_HOME": str(selected_home)}, + recorded_threads, + [(f"discovery-{index}", child_id)], + ) + elif worker_home in { + "external-sqlite", + "external-shared-home", + "external-missing-copy", + "external-missing-child", + "external-current-middle-missing", + "external-current-middle-complete", + }: + if worker_home.startswith("external-current-middle"): + grandchild_id = f"grandchild-{index}" + worker_threads[grandchild_id] = _rollout( + root, + grandchild_id, + [_token_event(counted, index * 11, 2)], + parent_thread_id=child_id, + ) + # Native keeps rollouts in its Codex home even when its SQLite + # index lives elsewhere and recovery chooses a different index. + sessions = selected_home / "sessions" / "2026" / "01" / "01" + sessions.mkdir(parents=True, exist_ok=True) + for thread_id, path in worker_threads.items(): + recorded = sessions / f"rollout-{thread_id}.jsonl" + path.rename(recorded) + worker_threads[thread_id] = recorded + _state_graph( + {"CODEX_SQLITE_HOME": str(root / "original-external-sqlite")}, + worker_threads, + [(f"discovery-{index}", child_id)], + ) + if worker_home in {"external-missing-copy", "external-missing-child"}: + first_id = f"discovery-{index}" + with sqlite3.connect(environment["CODEX_STATE_DB"]) as connection: + connection.execute( + "INSERT INTO threads VALUES (?, ?)", + ( + first_id, + str(root / "missing-copy.jsonl") + if worker_home == "external-missing-copy" + else str(worker_threads[first_id]), + ), + ) + if worker_home == "external-missing-child": + connection.execute( + "INSERT INTO thread_spawn_edges VALUES (?, ?)", (first_id, child_id) + ) + worker_threads[child_id].unlink() + if worker_home.startswith("external-current-middle"): + # Recovery's current index sees the grandchild while its parent + # rollout is restored later from the worker's original home. + with sqlite3.connect(environment["CODEX_STATE_DB"]) as connection: + connection.executemany( + "INSERT INTO threads VALUES (?, ?)", + [ + (f"discovery-{index}", str(worker_threads[f"discovery-{index}"])), + ( + child_id, + str(root / "missing-middle.jsonl") + if worker_home.endswith("missing") + else str(worker_threads[child_id]), + ), + (grandchild_id, str(worker_threads[grandchild_id])), + ], + ) + connection.executemany( + "INSERT INTO thread_spawn_edges VALUES (?, ?)", + [(f"discovery-{index}", child_id), (child_id, grandchild_id)], + ) + result = _complete_scan(fixture)["scan"]["usage"] + assert snapshot.read_bytes() == original_bytes + return result + + # Both completions share current home B; each scan retains its own worker home A. + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(complete, (1, 2))) + for index, usage in enumerate(results, 1): + if worker_home == "unavailable": + assert usage["coverage"] == "partial" + assert usage["inputTokens"] == index * 10 + assert usage["outputTokens"] == 2 + assert usage["threadCount"] == 1 + assert usage["missingThreadCount"] == 2 + elif worker_home == "external-missing-child": + assert usage == { + "coverage": "partial", + "source": "codex_rollout", + **_counts(index * 70, 0, 12), + "threadCount": 3, + "missingThreadCount": 1, + "warnings": [ + "codex_state_unavailable", + "rollout_unavailable", + "scan_root_unavailable", + ], + "modelUsage": [{"model": None, **_counts(index * 70, 0, 12)}], + } + else: + assert usage == { + "coverage": "complete", + "source": "codex_rollout", + **_counts( + index * (88 if worker_home.startswith("external-current-middle") else 77), + 0, + 15 if worker_home.startswith("external-current-middle") else 13, + ), + "threadCount": 5 if worker_home.startswith("external-current-middle") else 4, + "modelUsage": ( + [ + {"model": None, **_counts(index * 47, 0, 8)}, + {"model": "model-alpha", **_counts(index * 20, 0, 3)}, + {"model": "model-beta", **_counts(index * 10, 0, 2)}, + ] + if copied_rollout + else [ + { + "model": None, + **_counts( + index + * (88 if worker_home.startswith("external-current-middle") else 77), + 0, + 15 if worker_home.startswith("external-current-middle") else 13, + ), + } + ] + ), + } def test_completion_preserves_explicit_legacy_cost(tmp_path: Path) -> None: @@ -684,3 +1057,405 @@ def test_failed_scan_preserves_legacy_failure_behavior(tmp_path: Path) -> None: )["scan"] assert failed["progress"]["status"] == "failed" assert "usage" not in failed + + +def test_rollout_usage_reconciles_stale_cumulative_events_and_models( + tmp_path: Path, workbench_api +) -> None: + usage_reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + events = [ + _event(start, "turn_context", {"turn_id": "own-turn", "model": "gpt-5.6-sol"}), + _token_event(start, 100, 10), + _token_event(start, 60, 6), + _event(start, "turn_context", {"turn_id": "own-turn", "model": "gpt-6-astra"}), + _token_event(start, 200, 20), + ] + rollout = _rollout(tmp_path, "worker", events) + counts, warnings = usage_reader._read_rollout_usage( + usage_reader.RolloutSession("worker", None, rollout), + started_at=start, + completed_at=None, + ) + assert counts == _counts(200, 0, 20) + assert warnings == {"token_counter_regressed"} + models = {} + counts, warnings = usage_reader._read_rollout_usage( + usage_reader.RolloutSession("worker", None, rollout), + started_at=start, + completed_at=None, + model_usage=models, + ) + assert counts == _counts(200, 0, 20) + assert warnings == {"token_counter_regressed"} + assert models == {"gpt-5.6-sol": _counts(100, 0, 10), "gpt-6-astra": _counts(100, 0, 10)} + + +def test_shared_parent_usage_requires_original_turn_and_scan_interval( + tmp_path: Path, workbench_api +) -> None: + usage_reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + end = start + timedelta(seconds=5) + events = [ + _event( + start - timedelta(seconds=1), + "turn_context", + {"turn_id": "prior", "model": "gpt-5.6-sol"}, + ), + _token_event(start - timedelta(seconds=1), 100, 10), + _event(start, "turn_context", {"turn_id": "scan-turn", "model": "gpt-6-astra"}), + _token_event(start, 110, 12), + _event(start, "turn_context", {"turn_id": "unrelated", "model": "gpt-5.6-sol"}), + _token_event(start, 910, 92), + _event( + end + timedelta(seconds=1), + "turn_context", + {"turn_id": "scan-turn", "model": "gpt-6-astra"}, + ), + _token_event(end + timedelta(seconds=1), 1000, 100), + ] + models = {} + counts, warnings = usage_reader._read_rollout_usage( + usage_reader.RolloutSession("parent", None, _rollout(tmp_path, "parent", events)), + started_at=start, + completed_at=end, + owner_turn_id="scan-turn", + model_usage=models, + ) + assert counts == _counts(10, 0, 2) + assert warnings == set() + assert models == {"gpt-6-astra": _counts(10, 0, 2)} + + +@pytest.mark.parametrize( + "counter_info,receipt_state,expected_warning", + [ + (None, "complete", None), + ({}, "complete", "token_record_invalid"), + ({"total_token_usage": {"input_tokens": -1}}, "complete", "token_record_invalid"), + (None, "missing-response", "token_receipts_incomplete"), + (None, "incomplete-line", "rollout_record_incomplete"), + (None, "invalid-timestamp", "token_record_invalid"), + ], + ids=[ + "null-counter", + "empty-info", + "malformed-usage", + "missing-response", + "incomplete-line", + "invalid-timestamp", + ], +) +def test_completion_handles_no_usage_counter_without_hiding_incomplete_receipts( + tmp_path: Path, counter_info: Any, receipt_state: str, expected_warning: str | None +) -> None: + fixture = _start_scan(tmp_path) + counted = fixture.started_at + timedelta(microseconds=1) + tokens = dict(input_tokens=100, cached_input_tokens=20, output_tokens=10, total_tokens=110) + response = _event( + counted, + "token_usage_record", + dict( + response_id="response-one", + thread_id="scan-parent", + model="gpt-5.6-sol", + usage=tokens, + thread_token_usage=( + {**tokens, "input_tokens": 150, "total_tokens": 160} + if receipt_state == "missing-response" + else tokens + ), + ), + ) + counter = _event( + counted, + "event_msg", + {"type": "token_count", "info": counter_info, "rate_limits": None}, + ) + events = [counter, response, counter] + if receipt_state == "invalid-timestamp": + events.append( + { + **response, + "timestamp": None, + "payload": {**response["payload"], "response_id": "response-two"}, + } + ) + parent = _rollout(tmp_path, "scan-parent", events) + if receipt_state == "incomplete-line": + with parent.open("a") as stream: + stream.write('{"type":"token_usage_record"') + _state_graph(fixture.environment, {"scan-parent": parent}, []) + usage = _complete_scan(fixture)["scan"]["usage"] + assert usage["totalTokens"] == 110 + assert usage["modelUsage"] == [{"model": "gpt-5.6-sol", **_counts(100, 20, 10)}] + assert usage["coverage"] == ("partial" if expected_warning else "complete") + assert usage.get("warnings", []) == ([expected_warning] if expected_warning else []) + + +@pytest.mark.parametrize("counters", [False, True]) +def test_response_receipts_count_compaction_once_across_resets( + tmp_path: Path, workbench_api, counters: bool +) -> None: + reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + + def usage(input_tokens, cached, output): + return dict( + input_tokens=input_tokens, + cached_input_tokens=cached, + cache_write_input_tokens=0, + output_tokens=output, + reasoning_output_tokens=0, + total_tokens=input_tokens + output, + ) + + def receipt(response, count, cumulative, model="gpt-5.6-sol", turn="scan-turn", second=1): + return _event( + start + timedelta(seconds=second), + "token_usage_record", + dict( + response_id=response, + thread_id="parent", + turn_id=turn, + model=model, + usage=count, + thread_token_usage=cumulative, + ), + ) + + first = receipt("first", usage(100, 80, 10), usage(100, 80, 10)) + compact = receipt("compaction", usage(50, 40, 5), usage(150, 120, 15), model="gpt-6-astra") + second = receipt("second", usage(120, 90, 12), usage(120, 90, 12)) + events = [ + first, + *([_token_event(start + timedelta(seconds=1), 100, 10)] if counters else []), + compact, + _event(start + timedelta(seconds=1), "compacted", {"message": "Synthetic summary"}), + compact, + second, + *([_token_event(start + timedelta(seconds=1), 220, 22)] if counters else []), + first, + receipt("other", usage(900, 0, 0), usage(900, 0, 0), turn="other-turn"), + receipt("post", usage(800, 0, 0), usage(1700, 0, 0), second=11), + ] + models = {} + total, warnings = reader._read_rollout_usage( + reader.RolloutSession("parent", None, _rollout(tmp_path, "parent", events)), + started_at=start, + completed_at=start + timedelta(seconds=10), + owner_turn_id="scan-turn", + model_usage=models, + ) + assert total == _counts(270, 210, 27) + assert warnings == set() + assert models == {"gpt-5.6-sol": _counts(220, 170, 22), "gpt-6-astra": _counts(50, 40, 5)} + + +def test_delayed_response_receipt_resolves_missing_cumulative_usage( + tmp_path: Path, workbench_api +) -> None: + reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + + def receipt(response, tokens, cumulative): + def usage(value): + return dict(input_tokens=value, output_tokens=0, total_tokens=value) + + return _event( + start, + "token_usage_record", + dict( + response_id=response, + thread_id="parent", + model="gpt-5.6-sol", + usage=usage(tokens), + thread_token_usage=usage(cumulative), + ), + ) + + rollout = _rollout(tmp_path, "parent", [receipt("first", 100, 100), receipt("third", 50, 180)]) + session = reader.RolloutSession("parent", None, rollout) + total, warnings = reader._read_rollout_usage(session, started_at=start, completed_at=None) + assert total == _counts(150, 0, 0) + assert warnings == {"token_receipts_incomplete"} + with rollout.open("a") as source: + source.write(json.dumps(receipt("second", 30, 130)) + "\n") + total, warnings = reader._read_rollout_usage(session, started_at=start, completed_at=None) + assert total == _counts(180, 0, 0) + assert warnings == set() + + +def test_exact_receipts_replace_overlapping_legacy_counter(tmp_path: Path, workbench_api) -> None: + reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + + def receipt(response, tokens, cumulative): + def usage(value): + return dict(input_tokens=value, output_tokens=0, total_tokens=value) + + return _event( + start, + "token_usage_record", + dict( + response_id=response, + thread_id="parent", + model="gpt-5.6-sol", + usage=usage(tokens), + thread_token_usage=usage(cumulative), + ), + ) + + rollout = _rollout( + tmp_path, + "parent", + [ + _token_event(start, 100, 0), + receipt("new", 10, 110), + receipt("old", 100, 100), + _token_event(start, 10, 0), + ], + ) + models = {} + total, warnings = reader._read_rollout_usage( + reader.RolloutSession("parent", None, rollout), + started_at=start, + completed_at=None, + model_usage=models, + ) + assert total == _counts(110, 0, 0) + assert warnings == set() + assert models == {"gpt-5.6-sol": _counts(110, 0, 0)} + + +@pytest.mark.parametrize( + "receipt,counter,expected", + [ + ((150, 15), (100, 200), (150, 200)), + ((150, 15), (100, 20), (150, 20)), + ((100, 50), (110, 10), (110, 50)), + ((100, 50), (200, 10), (200, 50)), + ((100, 10), (100, 10), (100, 10)), + ], + ids=[ + "counter-output", + "smaller-counter-output", + "smaller-counter-input", + "counter-input", + "equal-control", + ], +) +def test_completion_retains_receipt_and_counter_categories(tmp_path, receipt, counter, expected): + fixture = _start_scan(tmp_path) + counted = fixture.started_at + timedelta(microseconds=1) + response = _event( + counted, + "token_usage_record", + { + "response_id": "category-response", + "thread_id": "scan-parent", + "model": "gpt-5.6-sol", + "usage": { + "input_tokens": receipt[0], + "output_tokens": receipt[1], + "total_tokens": sum(receipt), + }, + "thread_token_usage": { + "input_tokens": counter[0], + "output_tokens": counter[1], + "total_tokens": sum(counter), + }, + }, + ) + parent = _rollout(tmp_path, "scan-parent", [response, _token_event(counted, *counter)]) + _state_graph(fixture.environment, {"scan-parent": parent}, []) + usage = _complete_scan(fixture)["scan"]["usage"] + assert (usage["inputTokens"], usage["outputTokens"], usage["totalTokens"]) == ( + *expected, + sum(expected), + ) + for category in ("inputTokens", "outputTokens", "totalTokens"): + assert sum(row[category] for row in usage["modelUsage"]) == usage[category] + assert usage["coverage"] == ("complete" if receipt == counter else "partial") + + +@pytest.mark.parametrize("counter_input,expected_cached", [(100, 20), (250, 90)]) +def test_completion_keeps_cached_subsets_valid_when_categories_diverge( + tmp_path, counter_input, expected_cached +): + fixture = _start_scan(tmp_path) + counted = fixture.started_at + timedelta(microseconds=1) + response = _event( + counted, + "token_usage_record", + { + "response_id": "cache-response", + "thread_id": "scan-parent", + "model": "gpt-5.6-sol", + "usage": { + "input_tokens": 150, + "cached_input_tokens": 20, + "output_tokens": 15, + "total_tokens": 165, + }, + }, + ) + parent = _rollout( + tmp_path, + "scan-parent", + [response, _token_event(counted, counter_input, 200, cached_input_tokens=90)], + ) + _state_graph(fixture.environment, {"scan-parent": parent}, []) + usage = _complete_scan(fixture)["scan"]["usage"] + assert (usage["inputTokens"], usage["cachedInputTokens"], usage["outputTokens"]) == ( + max(150, counter_input), + expected_cached, + 200, + ) + for row in usage["modelUsage"]: + assert row["cachedInputTokens"] + row["cacheWriteInputTokens"] <= row["inputTokens"] + for key in ("inputTokens", "cachedInputTokens", "outputTokens", "totalTokens"): + assert sum(row[key] for row in usage["modelUsage"]) == usage[key] + + +@pytest.mark.parametrize("receipt_owner", ["other-turn", "outside-window"]) +def test_owned_legacy_model_survives_unrelated_first_receipt( + tmp_path: Path, workbench_api, receipt_owner: str +) -> None: + reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + end = start + timedelta(seconds=5) + events = [ + _event(start, "turn_context", {"turn_id": "scan-turn", "model": "gpt-5.6-sol"}), + _token_event(start, 1000, 10), + _event( + end + timedelta(seconds=1) if receipt_owner == "outside-window" else start, + "token_usage_record", + { + "thread_id": "parent", + "turn_id": "other-turn" if receipt_owner == "other-turn" else "scan-turn", + "response_id": "unrelated-response", + "model": "gpt-6-astra", + "usage": { + "input_tokens": 50, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "output_tokens": 0, + "reasoning_output_tokens": 0, + "total_tokens": 50, + }, + }, + ), + ] + models = {} + counts, warnings = reader._read_rollout_usage( + reader.RolloutSession("parent", None, _rollout(tmp_path, "parent", events)), + started_at=start, + completed_at=end, + owner_turn_id="scan-turn", + model_usage=models, + ) + assert counts == _counts(1000, 0, 10) + assert warnings == set() + assert models == {"gpt-5.6-sol": _counts(1000, 0, 10)} diff --git a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py index 1adafa9839..b056b29129 100644 --- a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py +++ b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py @@ -70,6 +70,12 @@ (39, "store dedupe checkpoint bindings in columns"), (40, "index finding identity and comparison history"), (41, "checkpoint finding severity assessments"), + (44, "preserve original deep scan discovery context"), + (45, "retain deep scan attempts and exact merge inputs"), + (46, "persist selected deep scan finalization input"), + (47, "freeze stopped scan checkpoint selections"), + (48, "bind original deep scan parent usage turn"), + (51, "bind original deep scan execution settings"), ] @@ -488,7 +494,7 @@ def test_workbench_serializes_concurrent_migrations(tmp_path: Path, upgrade: boo {"databasePath": str(state_dir / "workbench.sqlite3")}, ] with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: - assert connection.execute("SELECT COUNT(*) FROM schema_migrations").fetchone() == (41,) + assert connection.execute("SELECT COUNT(*) FROM schema_migrations").fetchone() == (47,) def test_workbench_retries_writer_admission_and_legacy_backfill(tmp_path: Path) -> None: @@ -1079,7 +1085,7 @@ def test_workbench_upgrades_preexisting_database(tmp_path: Path) -> None: connection.execute("ALTER TABLE scans DROP COLUMN handoff_claim_token") run_workbench(state_dir, "database-info") with sqlite3.connect(database) as connection: - assert connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone() == (41,) + assert connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone() == (51,) assert {row[1] for row in connection.execute("PRAGMA table_info(scans)")} >= { "handoff_claimed_at", "handoff_claim_token", diff --git a/plugins/codex-security/tests/test_workbench_standard_deep_results.py b/plugins/codex-security/tests/test_workbench_standard_deep_results.py index 5de07a5b1f..ff6b0db5f5 100644 --- a/plugins/codex-security/tests/test_workbench_standard_deep_results.py +++ b/plugins/codex-security/tests/test_workbench_standard_deep_results.py @@ -74,8 +74,10 @@ def test_stopped_deep_scan_ignores_late_worker_checkpoints_without_reducer( # The latest incomplete attempt need not be parseable for a saved checkpoint to survive. result_path.write_text("{incomplete") with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: + # This is new incomplete work, not a rewrite of the accepted attempt. connection.execute( - "UPDATE deep_scan_workers SET status = 'running' WHERE id = ?", (worker_id,) + "UPDATE deep_scan_workers SET status = 'running', attempt = 2 WHERE id = ?", + (worker_id,), ) environment = {"CODEX_HOME": str(codex_home)} if termination == "canceled": @@ -493,6 +495,9 @@ def test_explicit_recovery_preserves_sealed_parent_with_empty_source_map( state_dir, codex_home, target, scan_dir, scan_id = deep_scan_fixture(tmp_path) _, result_path = accepted_standard_worker(state_dir, codex_home, scan_dir, scan_id) result_path.unlink() + # Remove the immutable accepted copy too, leaving no recoverable worker source. + for checkpoint in (result_path.parent / "checkpoints").glob("*.json"): + checkpoint.unlink() contract_dir = tmp_path / "contract" contract_dir.mkdir() scripts_dir = Path(__file__).resolve().parents[1] / "scripts" @@ -1020,6 +1025,9 @@ def test_canceled_scan_reports_noop_coordinator_publication(tmp_path: Path) -> N state_dir, codex_home, _, scan_dir, scan_id = deep_scan_fixture(tmp_path) _, result_path = accepted_standard_worker(state_dir, codex_home, scan_dir, scan_id) result_path.write_text("{incomplete") + # A valid immutable copy would let publication recover despite this corruption. + for checkpoint in (result_path.parent / "checkpoints").glob("*.json"): + checkpoint.unlink() wrapper = tmp_path / "fail_before_canceled_sources_are_frozen.py" canceled = run_workbench_with_fault( @@ -1210,6 +1218,8 @@ def deep_scan_fixture( run_workbench( state_dir, "begin-deep-scan", + "--workflow-version", + "deep-security-scan/v1", "--thread-id", "standard-worker-thread", "--scan-id", @@ -1220,6 +1230,8 @@ def deep_scan_fixture( begun = run_workbench( state_dir, "begin-deep-scan", + "--workflow-version", + "deep-security-scan/v1", "--thread-id", "standard-worker-thread", "--target-path", @@ -1289,6 +1301,19 @@ def accepted_standard_worker( str(result_path), environment=environment, ) + # These fixtures represent workers saved before immutable attempt acceptance. + with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: + accepted = connection.execute( + "SELECT accepted_result_path FROM deep_scan_attempts " + "WHERE worker_id = ? AND attempt = 1", + (worker_id,), + ).fetchone() + if accepted is not None and accepted[0] is not None: + Path(accepted[0]).unlink() + connection.execute( + "DELETE FROM deep_scan_attempts WHERE worker_id = ? AND attempt = 1", + (worker_id,), + ) return worker_id, result_path @@ -1396,6 +1421,7 @@ def test_failure_preserves_last_committed_reducer_without_parent_draft(tmp_path: state_dir, codex_home, scan_dir, scan_id, worker_id, result_path ) reduced = json.loads(reducer_path.read_text()) + accepted_summary = reduced["findings"][0]["summary"] reduced["findings"][0]["summary"] = ( "The reducer retained additional independently reviewed evidence." ) @@ -1404,7 +1430,8 @@ def test_failure_preserves_last_committed_reducer_without_parent_draft(tmp_path: failed = run_workbench(state_dir, "get-scan", "--scan-id", scan_id)["scan"] assert failed["progress"]["status"] == "failed" assert failed["findingCount"] == 1 - assert failed["findings"][0]["summary"] == reduced["findings"][0]["summary"] + assert failed["findings"][0]["summary"] == accepted_summary + assert json.loads(reducer_path.read_text()) == reduced @pytest.mark.parametrize("tied_head", [False, True]) @@ -1859,7 +1886,7 @@ def test_recovery_selects_strongest_same_finding_checkpoint(tmp_path: Path) -> N strong["confidence"]["level"] = "high" strong["summary"] = "Later strong checkpoint evidence." checkpoint_dir = result_path.parent / "checkpoints" - checkpoint_dir.mkdir() + checkpoint_dir.mkdir(exist_ok=True) for name, finding in (("0" * 64, weak), ("f" * 64, strong)): (checkpoint_dir / f"{name}.json").write_text( json.dumps(saved_draft(scan_id, findings=[finding], completeness="partial")) diff --git a/plugins/codex-security/tests/workbench_test_support.py b/plugins/codex-security/tests/workbench_test_support.py index 8bfb6cf56b..cef3c17d0d 100644 --- a/plugins/codex-security/tests/workbench_test_support.py +++ b/plugins/codex-security/tests/workbench_test_support.py @@ -336,7 +336,7 @@ def worker_paths(scan_dir: Path, name: str) -> tuple[Path, Path, Path]: def mark_deep_coordinator_succeeded(state_dir: Path, scan_id: str, scan_dir: Path) -> Path: manifest = scan_dir / "artifacts" / "deep_discovery" / "coordinator-manifest.json" - manifest.parent.mkdir(parents=True) + manifest.parent.mkdir(parents=True, exist_ok=True) manifest.write_text('{"status":"succeeded"}\n') with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: connection.execute( diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index 7d3b8a31b6..e4f8149d53 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -257,6 +257,17 @@ within each worker. Time and run limits apply to discovery. After discovery stops, the scan combines and returns completed findings. See [deep scan settings](docs/cli.md#configure-deep-scans) for defaults and saved settings. +New Deep scans preserve the selected aggregate and original terminal reason through +publication and completion retries. Resuming a selected result reuses accepted +findings instead of starting another discovery or reducer run; older scans keep +their recorded workflow. Remaining workers use their recorded execution settings, +while authentication continues to use the original configured account. + +The SDK's `onDeepProgress` callback reports completed, active and maximum independent +reviews. Saved execution attribution and recorded worker homes let resumed usage and +session logs retain their original scan membership. A known priced lower bound can +still stop an explicit budget when another model's price is unavailable. + ### Progress and cost Use `onProgress` for scan progress, `onWorkerStatus` for individual workers, and diff --git a/sdk/typescript/scripts/check-package.mjs b/sdk/typescript/scripts/check-package.mjs index 665308b933..064207f57b 100644 --- a/sdk/typescript/scripts/check-package.mjs +++ b/sdk/typescript/scripts/check-package.mjs @@ -145,6 +145,11 @@ const allowedFiles = new Set([ "package/dist/server/dashboard/THIRD_PARTY_NOTICES.txt", ...[ "api", + "reasoning-summary", + "preflight-config", + "deep-scan-finalization", + "codex-session", + "accepted-audit", "artifact-export", "auth", "bulk-scan-discovery", diff --git a/sdk/typescript/scripts/ci-test-durations.json b/sdk/typescript/scripts/ci-test-durations.json index e41fd18718..128d60583c 100644 --- a/sdk/typescript/scripts/ci-test-durations.json +++ b/sdk/typescript/scripts/ci-test-durations.json @@ -3,6 +3,8 @@ "https://github.com/openai/codex-security/actions/runs/33215429477", "https://github.com/openai/codex-security/actions/runs/33227512538", "https://github.com/openai/codex-security/actions/runs/33233690024", + "https://github.com/openai/codex-security/actions/runs/34813432568", + "https://github.com/openai/codex-security/actions/runs/34822813350", "https://github.com/openai/codex-security/actions/runs/37420072376", "https://github.com/openai/codex-security/actions/runs/37472306800" ], @@ -247,6 +249,7 @@ "custom-validation.test.ts": 23.262, "dedupe-records.test.ts": 0.861, "deep-config.test.ts": 0.492, + "deep-finalization.test.ts": 212.524, "deep-progress.test.ts": 0.005, "deep-review-input.test.ts": 15.492, "deep-scan-parent-denials.test.ts": 0.034, diff --git a/sdk/typescript/scripts/fixtures/package-deep-codex.mjs b/sdk/typescript/scripts/fixtures/package-deep-codex.mjs new file mode 100644 index 0000000000..fc59a91e50 --- /dev/null +++ b/sdk/typescript/scripts/fixtures/package-deep-codex.mjs @@ -0,0 +1,182 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; +import { appendFile, writeFile } from "node:fs/promises"; +import { basename, join } from "node:path"; +import { createInterface } from "node:readline"; +import { startRpc } from "./package-rpc.mjs"; + +try { + await run(); + process.exit(0); +} catch (error) { + await trace({ phase: "fixture-error", error: error.stack }); + console.error(error); + process.exit(1); +} + +async function trace(event) { + await appendFile( + process.env.PACKAGE_DEEP_TRACE, + `${JSON.stringify(event)}\n`, + ); +} + +async function run() { + const args = process.argv.slice(2); + if (args.includes("app-server")) { + await trace({ phase: "preflight", args }); + for await (const line of createInterface({ input: process.stdin })) { + const message = JSON.parse(line); + if (message.id === undefined) continue; + let result; + switch (message.method) { + case "initialize": + result = { userAgent: "package-fixture" }; + break; + case "config/read": + result = { + config: { + default_permissions: "codex_security_deep_scan_worker", + permissions: { + codex_security_deep_scan_worker: { + extends: ":read-only", + filesystem: { ":root": "read" }, + network: { enabled: false }, + }, + }, + }, + origins: {}, + layers: null, + }; + break; + case "permissionProfile/list": + result = { + data: [ + { + id: "codex_security_deep_scan_worker", + description: null, + allowed: true, + }, + ], + nextCursor: null, + }; + break; + case "account/read": + result = { account: null, requiresOpenaiAuth: true }; + break; + default: + throw new Error(`Unexpected preflight method: ${message.method}`); + } + console.log(JSON.stringify({ jsonrpc: "2.0", id: message.id, result })); + } + return; + } + let prompt = ""; + for await (const chunk of process.stdin) prompt += chunk; + const config = {}; + for (let index = 0; index < args.length; index++) { + if (args[index] !== "-c" && args[index] !== "--config") continue; + const setting = args[++index]; + const equals = setting.indexOf("="); + config[setting.slice(0, equals)] = setting.slice(equals + 1); + } + const prefix = "mcp_servers.cs_artifacts."; + const env = Object.fromEntries( + Object.entries(config) + .filter(([name]) => name.startsWith(`${prefix}env.`)) + .map(([name, value]) => [ + name.slice(`${prefix}env.`.length), + JSON.parse(value), + ]), + ); + const root = env.CODEX_SECURITY_ARTIFACT_ROOT; + assert.ok(root, "The real worker must supply its bound artifact root."); + assert.equal(config["mcp_servers.codex-security.enabled"], "false"); + const layout = env.CODEX_SECURITY_ARTIFACT_LAYOUT; + const threadId = `package-${layout}-${basename(root)}-${basename(join(root, ".."))}`; + console.log(JSON.stringify({ type: "thread.started", thread_id: threadId })); + if ( + layout === "worker" && + basename(join(root, "..")) === "discovery-0002" && + process.env.PACKAGE_DEEP_HOLD && + existsSync(process.env.PACKAGE_DEEP_HOLD) + ) { + await trace({ phase: "held", scanId: env.CODEX_SECURITY_SCAN_ID }); + await new Promise(() => setInterval(() => {}, 1_000)); + } + const server = await startRpc( + JSON.parse(config[`${prefix}command`]), + JSON.parse(config[`${prefix}args`]), + { cwd: root, env: { ...process.env, ...env } }, + ); + let complete = true; + try { + if (layout === "worker") { + const draft = { + scanId: env.CODEX_SECURITY_SCAN_ID, + findings: [], + coverage: { + completeness: "complete", + surfaces: [], + explicitExclusions: [], + deferred: [], + }, + }; + const marker = process.env.PACKAGE_DEEP_EMPTY_ONCE; + if (marker && !existsSync(marker)) { + await writeFile(marker, "process completed without a final artifact"); + complete = false; + } else { + await server.call("record_codex_security_scan_draft", { + ...draft, + complete: false, + }); + await server.call("record_codex_security_scan_draft", { + ...draft, + complete: true, + }); + } + } else { + assert.equal(layout, "reducer"); + let document = ""; + let cursor; + do { + const page = await server.call( + "get_codex_security_deep_reducer_inputs", + { maxBytes: 4096, ...(cursor === undefined ? {} : { cursor }) }, + ); + document += page.json; + cursor = page.nextCursor; + } while (cursor !== undefined); + const inputs = JSON.parse(document); + assert.ok(inputs.discoveries.length > 0); + await server.call("record_codex_security_deep_reduction", { + scanId: env.CODEX_SECURITY_SCAN_ID, + findings: [], + }); + } + await trace({ + phase: layout, + complete, + resumed: args.includes("resume"), + scanId: env.CODEX_SECURITY_SCAN_ID, + home: process.env.CODEX_HOME, + hasApiKey: process.env.CODEX_API_KEY === "synthetic-package-deep-key", + root, + args, + }); + console.log( + JSON.stringify({ + type: "turn.completed", + usage: { + input_tokens: 1, + cached_input_tokens: 0, + output_tokens: 1, + }, + }), + ); + } finally { + await server.close(); + } +} diff --git a/sdk/typescript/scripts/fixtures/package-deep-scan.mjs b/sdk/typescript/scripts/fixtures/package-deep-scan.mjs new file mode 100644 index 0000000000..7027049034 --- /dev/null +++ b/sdk/typescript/scripts/fixtures/package-deep-scan.mjs @@ -0,0 +1,595 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; +import { + chmod, + copyFile, + cp, + mkdir, + mkdtemp, + readFile, + realpath, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, dirname, join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { startRpc } from "./package-rpc.mjs"; +import { packageSmokeTimeouts } from "../package-smoke-timeouts.mjs"; + +const installedRoot = await realpath(process.argv[2]); +const root = await realpath( + await mkdtemp(join(tmpdir(), "package deep % fixture-")), +); +const installedPlugin = join(installedRoot, "_bundled_plugin"); +try { + const detachedPlugin = join(root, "standalone plugin %", "codex-security"); + await cp(installedPlugin, detachedPlugin, { recursive: true }); + // Assert physical independence instead of merely changing the working directory. + for (let ancestor = detachedPlugin; ; ancestor = dirname(ancestor)) { + for (const dependency of ["node_modules", join("sdk", "typescript")]) { + await assert.rejects(stat(join(ancestor, dependency)), { + code: "ENOENT", + }); + } + if (ancestor === dirname(ancestor)) break; + } + for (const name of [ + "package-deep-codex.mjs", + "package-rpc.mjs", + "package-deep-spawn.mjs", + ]) { + await copyFile(new URL(name, import.meta.url), join(root, name)); + } + const executable = join( + root, + process.platform === "win32" + ? "package-codex.exe" + : "package-deep-codex.mjs", + ); + if (process.platform === "win32") + await copyFile(process.execPath, executable); + await chmod(executable, 0o700); + + await runInstalledSdk(installedPlugin, executable); + await runDetachedPlugin(detachedPlugin, executable); + console.log( + "Validated installed SDK and detached plugin: real Deep processes, bound artifact tools, checkpoints, reducer acceptance, restart before finalization, and sealed results.", + ); +} catch (error) { + for (const name of ["installed", "detached"]) { + try { + error.message += `\n${await readFile(join(root, name, "executions.jsonl"), "utf8")}`; + } catch (readError) { + if (readError.code !== "ENOENT") throw readError; + } + } + throw error; +} finally { + await rm(root, { + recursive: true, + force: true, + maxRetries: 10, + retryDelay: 100, + }); +} + +async function fixture(name, pluginRoot, executable) { + const directory = join(root, name); + const target = join(directory, "target with spaces %"); + const home = join(directory, "home"); + await mkdir(target, { recursive: true }); + await mkdir(join(home, "codex-security"), { recursive: true }); + await writeFile( + join(target, "fixture.py"), + "print('synthetic package fixture')\n", + ); + await writeFile( + join(home, "codex-security", "config.toml"), + "[deep_scan]\nworkers = 1\nsubagents = 0\nstop_after_no_new = 1\nmax_discovery_runs = 2\n", + ); + const env = Object.fromEntries( + [ + "PATH", + "Path", + "SystemRoot", + "WINDIR", + "ComSpec", + "PATHEXT", + "TMP", + "TEMP", + "TMPDIR", + ] + .filter((key) => process.env[key] !== undefined) + .map((key) => [key, process.env[key]]), + ); + Object.assign(env, { + HOME: home, + USERPROFILE: home, + CODEX_HOME: home, + CODEX_CLI_PATH: executable, + CODEX_SECURITY_PLUGIN_ROOT: pluginRoot, + CODEX_SECURITY_STATE_DIR: join(directory, "state"), + CODEX_SECURITY_SCAN_ROOT: join(directory, "scans"), + PYTHON: process.env.PYTHON || "python3", + OPENAI_API_KEY: "synthetic-package-deep-key", + ...(process.platform === "win32" + ? { + PACKAGE_DEEP_EXECUTABLE: executable, + NODE_OPTIONS: `--import=${pathToFileURL(join(root, "package-deep-spawn.mjs")).href}`, + } + : {}), + PACKAGE_DEEP_TRACE: join(directory, "executions.jsonl"), + }); + return { directory, target, home, env, pluginRoot }; +} + +function metadata(f, owner) { + return { + "openai/threadId": owner, + "codex/sandbox-state-meta": { + permissionProfile: { + type: "managed", + file_system: { + type: "restricted", + entries: [ + { + path: { type: "special", value: { kind: "root" } }, + access: "read", + }, + ], + }, + network: "restricted", + }, + sandboxCwd: pathToFileURL(f.target).href, + }, + "x-codex-turn-metadata": { model: "gpt-5.5", reasoning_effort: "high" }, + }; +} + +function server(f, env = f.env) { + return startRpc( + process.execPath, + [join(f.pluginRoot, "mcp", "server.mjs"), "--stdio"], + { + cwd: f.target, + env, + requestTimeoutMs: packageSmokeTimeouts().commandTimeoutMs, + }, + ); +} + +async function runDetachedPlugin(pluginRoot, executable) { + const f = await fixture("detached", pluginRoot, executable); + const owner = "package-detached-owner"; + f.env.PACKAGE_DEEP_HOLD = join(f.directory, "hold-second-worker"); + await writeFile(f.env.PACKAGE_DEEP_HOLD, "hold"); + let rpc = await server(f); + let scanId; + let scanDir; + let partial; + const handoffClaimToken = randomUUID(); + try { + const opened = await rpc.call( + "open_codex_security_workspace", + { + targetPath: f.target, + scope: ".", + mode: "deep", + }, + metadata(f, owner), + ); + const sessionId = opened.workspace.id; + await rpc.call( + "submit_codex_security_setup", + { + sessionId, + targetPath: f.target, + scope: ".", + mode: "deep", + }, + metadata(f, owner), + ); + const started = await rpc.call( + "start_codex_security_scan", + { sessionId }, + metadata(f, owner), + ); + ({ scanId, scanDir } = started.workspace.results); + await rpc.call( + "claim_codex_security_scan_handoff_delivery", + { + scanId, + claimToken: handoffClaimToken, + }, + metadata(f, owner), + ); + await rpc.call( + "attach_codex_security_scan_continuation_thread", + { + scanId, + claimToken: handoffClaimToken, + threadId: owner, + }, + metadata(f, owner), + ); + const pending = rpc + .call( + "start_codex_security_deep_scan", + { scanId, handoffClaimToken }, + metadata(f, owner), + ) + .catch((error) => error); + const deadline = Date.now() + 30_000; + while (!(await readExecutions(f)).some((entry) => entry.phase === "held")) { + assert.ok( + Date.now() < deadline, + "Second worker did not reach the interruption boundary.", + ); + await new Promise((resolve) => setTimeout(resolve, 50)); + } + partial = ( + await workbench(f, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + owner, + ]) + ).deepScan; + assert.equal( + partial.workers.filter( + (worker) => + worker.kind === "discovery" && worker.status === "succeeded", + ).length, + 1, + ); + await rpc.close(); + await pending; + } finally { + await rpc.close(); + } + // Simulate an expired owner lease without waiting for wall-clock expiry. Keep + // the real stored workers/results and use the production recovery path. + await rm( + join( + scanDir, + "artifacts", + "deep_discovery", + `coordinator-heartbeat-${partial.coordinatorGeneration}.json`, + ), + { force: true }, + ); + await promisify(execFile)( + f.env.PYTHON, + [ + "-c", + "import sqlite3,sys; c=sqlite3.connect(sys.argv[1]); c.execute('UPDATE deep_scan_runs SET updated_at = ? WHERE scan_id = ?', ('2000-01-01T00:00:00Z',sys.argv[2])); c.commit()", + join(f.env.CODEX_SECURITY_STATE_DIR, "workbench.sqlite3"), + scanId, + ], + { env: f.env }, + ); + await rm(f.env.PACKAGE_DEEP_HOLD); + rpc = await server(f); + try { + const result = await rpc.call( + "start_codex_security_deep_scan", + { scanId, handoffClaimToken }, + metadata(f, owner), + ); + await assertDraft(result.manifestPath); + const recovered = ( + await workbench(f, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + owner, + ]) + ).deepScan; + assert.equal( + recovered.coordinatorGeneration, + partial.coordinatorGeneration + 1, + ); + assert.equal(recovered.dispatchedCount, 2); + const retained = partial.workers.find( + (worker) => worker.status === "succeeded", + ); + assert.equal( + recovered.workers.find((worker) => worker.id === retained.id).status, + "succeeded", + ); + } finally { + await rpc.close(); + } + // Publication uses the saved aggregate after the recovered executor exits too. + rpc = await server(f); + try { + await rpc.call( + "complete_codex_security_scan", + { scanId, handoffClaimToken }, + metadata(f, owner), + ); + const completed = await rpc.call( + "get_codex_security_completed_scan", + { scanId, handoffClaimToken }, + metadata(f, owner), + ); + assert.equal(completed.manifest.scan.id, scanId); + assert.equal(completed.manifest.scan.status, "completed"); + assert.ok(completed.manifest.scan.sealedAt); + } finally { + await rpc.close(); + } + await assertSavedState(f, scanId, owner); + await assertExecutions(f, scanId, 4); +} + +async function workbench(f, args) { + const { stdout } = await promisify(execFile)( + f.env.PYTHON, + [join(f.pluginRoot, "scripts", "workbench_db.py"), ...args], + { + env: f.env, + cwd: f.target, + maxBuffer: 4 * 1024 * 1024, + }, + ); + return JSON.parse(stdout); +} + +async function runInstalledSdk(pluginRoot, executable) { + const f = await fixture("installed", pluginRoot, executable); + f.env.PACKAGE_DEEP_EMPTY_ONCE = join( + f.directory, + "missing-result-completion", + ); + const sdk = await import( + pathToFileURL(join(installedRoot, "dist", "index.js")).href + ); + const manifest = JSON.parse( + await readFile(join(pluginRoot, ".codex-plugin", "plugin.json"), "utf8"), + ); + const owner = "package-sdk-owner"; + const postScanPrompt = "Explain the completed synthetic scan."; + const prompts = []; + let threadCount = 0; + let manifestBeforeFollowUp; + let scanId; + const client = new sdk.CodexSecurity( + { pythonPath: f.env.PYTHON }, + { + environment: f.env, + prepareRuntime: async () => ({ + codexHome: f.home, + environment: f.env, + credentialsAvailable: true, + plugin: { + pluginRoot, + marketplaceRoot: pluginRoot, + installedRoot: pluginRoot, + marketplaceName: "codex-security-sdk", + name: manifest.name, + version: manifest.version, + }, + }), + // Replace only the parent model's tool choice. The installed SDK registers + // and finalizes the scan; the packaged MCP runs the real Deep lifecycle. + createCodex({ env, apiKey }) { + return { + startThread() { + threadCount += 1; + return { + id: owner, + async runStreamed(prompt) { + prompts.push(prompt); + return { + events: (async function* () { + yield { type: "thread.started", thread_id: owner }; + if (prompts.length > 1) { + assert.equal(prompt, postScanPrompt); + manifestBeforeFollowUp = await readFile( + join(env.CODEX_SECURITY_SCAN_DIR, "scan-manifest.json"), + "utf8", + ); + const completed = JSON.parse(manifestBeforeFollowUp); + assert.equal(completed.scan.status, "completed"); + assert.ok(completed.scan.sealedAt); + yield { + type: "turn.completed", + usage: { + input_tokens: 100_000, + cached_input_tokens: 0, + output_tokens: 100_000, + }, + }; + return; + } + scanId = env.CODEX_SECURITY_SCAN_ID; + // The pinned SDK maps its apiKey option to this child variable. + const rpc = await server(f, { + ...env, + ...(apiKey ? { CODEX_API_KEY: apiKey } : {}), + }); + try { + const result = await rpc.call( + "start_codex_security_deep_scan", + { scanId }, + metadata(f, owner), + ); + await assertDraft(result.manifestPath); + } finally { + await rpc.close(); + } + yield { + type: "turn.completed", + usage: { + input_tokens: 1, + cached_input_tokens: 0, + output_tokens: 1, + }, + }; + })(), + }; + }, + }; + }, + }; + }, + }, + ); + try { + const result = await client.run(f.target, { + mode: "deep", + auth: "api-key", + workers: 1, + subagents: 0, + maxDiscoveryRuns: 2, + stopAfterNoNew: 1, + postScanPrompt, + outputDir: join(f.directory, "output"), + }); + assert.equal(threadCount, 1); + assert.equal(prompts.length, 2); + assert.equal(prompts[1], postScanPrompt); + assert.equal(result.threadId, owner); + assert.equal(result.manifest.scan.status, "completed"); + assert.ok(result.manifest.scan.sealedAt); + assert.equal(result.manifest.scan.id, scanId); + assert.deepEqual(result.findings.findings, []); + assert.equal( + await readFile(result.manifestPath, "utf8"), + manifestBeforeFollowUp, + ); + assert.ok(result.cost === null || result.cost.inputTokens < 100_000); + assert.equal(result.toJSON().threadId, owner); + assert.ok( + (await readFile(join(f.directory, "output", "report.md"), "utf8")) + .length > 0, + ); + } finally { + await client.close(); + } + await assertSavedState(f, scanId, owner); + await assertExecutions(f, scanId, 4); +} + +async function assertSavedState(f, scanId, owner) { + const { deepScan } = await workbench(f, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + owner, + ]); + assert.equal(deepScan.status, "succeeded"); + if (deepScan.workflowVersion === "deep-security-scan/v2") { + const selected = deepScan.finalizationInput; + assert.ok( + selected, + "The completed v2 scan retains its finalization input.", + ); + assert.equal(selected.version, 1); + assert.equal(selected.terminalReason, deepScan.terminalReason); + assert.deepEqual(selected.omittedWorkerIds, []); + await assertDigest( + resolve(deepScan.scanDir, selected.resultPath), + selected.resultSha256, + ); + const workers = deepScan.workers.filter( + (worker) => worker.status === "succeeded", + ); + assert.equal(workers.length, 3); + for (const worker of workers) { + const attempt = deepScan.attempts.find( + (entry) => + entry.workerId === worker.id && entry.attempt === worker.attempt, + ); + assert.ok(attempt, "Each accepted worker retains its execution attempt."); + assert.equal(attempt.status, "succeeded"); + await assertDigest( + attempt.acceptedResultPath, + attempt.acceptedResultSha256, + ); + if (worker.kind === "dedup") { + assert.equal(selected.resultSha256, attempt.acceptedResultSha256); + } else { + const input = deepScan.dedupInputs.find( + (entry) => entry.discoveryWorkerId === worker.id, + ); + assert.ok(input, "The reducer retains each accepted discovery input."); + assert.equal(input.attempt, worker.attempt); + assert.equal(input.resultManifestSha256, attempt.acceptedResultSha256); + await assertDigest( + input.resultManifestPath, + input.resultManifestSha256, + ); + } + } + assert.equal(deepScan.dedupInputs.length, 2); + } + console.log( + JSON.stringify({ + fixture: basename(f.directory), + workflowVersion: deepScan.workflowVersion, + attempts: deepScan.attempts?.length ?? null, + selectedFinalization: deepScan.finalizationInput != null, + }), + ); +} + +async function assertDigest(path, expected) { + const bytes = await readFile(path); + assert.equal(createHash("sha256").update(bytes).digest("hex"), expected); +} + +async function assertDraft(path) { + const document = JSON.parse(await readFile(path, "utf8")); + const findings = JSON.parse( + await readFile(join(dirname(path), "findings.json"), "utf8"), + ); + assert.deepEqual(findings.findings, []); + assert.ok(document.scan.target); +} + +async function readExecutions(f) { + try { + return (await readFile(f.env.PACKAGE_DEEP_TRACE, "utf8")) + .trim() + .split("\n") + .filter(Boolean) + .map(JSON.parse); + } catch (error) { + if (error.code === "ENOENT") return []; + throw error; + } +} + +async function assertExecutions(f, scanId, preflights = 3) { + const executions = await readExecutions(f); + const workers = executions.filter((entry) => entry.phase === "worker"); + const reducers = executions.filter((entry) => entry.phase === "reducer"); + const incomplete = f.env.PACKAGE_DEEP_EMPTY_ONCE ? 1 : 0; + assert.equal(workers.length, 2 + incomplete); + assert.equal(workers.filter((entry) => !entry.complete).length, incomplete); + assert.equal(workers.filter((entry) => entry.resumed).length, incomplete); + assert.equal(reducers.length, 1); + assert.equal( + executions.filter((entry) => entry.phase === "preflight").length, + preflights, + ); + for (const execution of [...workers, ...reducers]) { + assert.equal(execution.scanId, scanId); + assert.equal(execution.home, f.home); + assert.equal(execution.hasApiKey, true); + assert.equal( + execution.args[execution.args.indexOf("--model") + 1], + "gpt-5.5", + ); + assert.ok(execution.args.includes('approval_policy="never"')); + } +} diff --git a/sdk/typescript/scripts/fixtures/package-deep-spawn.mjs b/sdk/typescript/scripts/fixtures/package-deep-spawn.mjs new file mode 100644 index 0000000000..e594405586 --- /dev/null +++ b/sdk/typescript/scripts/fixtures/package-deep-spawn.mjs @@ -0,0 +1,20 @@ +import childProcess from "node:child_process"; +import { syncBuiltinESMExports } from "node:module"; +import { win32 } from "node:path"; +import { fileURLToPath } from "node:url"; + +// Windows cannot execute the POSIX fixture's shebang. Preserve the selected +// native executable and its options, inserting only the deterministic protocol +// script, as in the worker launch tests. Every other child runs unchanged. +const executable = win32.toNamespacedPath(process.env.PACKAGE_DEEP_EXECUTABLE); +const script = fileURLToPath( + new URL("package-deep-codex.mjs", import.meta.url), +); +const spawn = childProcess.spawn; +childProcess.spawn = (command, args, options) => + spawn( + command, + win32.toNamespacedPath(command) === executable ? [script, ...args] : args, + options, + ); +syncBuiltinESMExports(); diff --git a/sdk/typescript/scripts/fixtures/package-rpc.mjs b/sdk/typescript/scripts/fixtures/package-rpc.mjs new file mode 100644 index 0000000000..ceb6156f2a --- /dev/null +++ b/sdk/typescript/scripts/fixtures/package-rpc.mjs @@ -0,0 +1,90 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { createInterface } from "node:readline"; + +// This client deliberately uses only Node builtins. Detached plugin tests must +// not resolve an MCP client or SDK from the checkout's node_modules. +export async function startRpc(command, args, options) { + const { requestTimeoutMs = 30_000, ...spawnOptions } = options; + const child = spawn(command, args, { + ...spawnOptions, + stdio: ["pipe", "pipe", "pipe"], + windowsHide: true, + }); + let sequence = 0; + let stderr = ""; + const pending = new Map(); + child.stderr.setEncoding("utf8").on("data", (chunk) => { + stderr += chunk; + }); + createInterface({ input: child.stdout }).on("line", (line) => { + const response = JSON.parse(line); + const waiter = pending.get(response.id); + if (!waiter) return; + pending.delete(response.id); + clearTimeout(waiter.timer); + if (response.error) + waiter.reject(new Error(JSON.stringify(response.error))); + else waiter.resolve(response.result); + }); + const exited = once(child, "exit"); + child.on("exit", (code, signal) => { + for (const waiter of pending.values()) { + clearTimeout(waiter.timer); + waiter.reject( + new Error(`Fixture RPC exited (${code}, ${signal}): ${stderr}`), + ); + } + pending.clear(); + }); + const client = { + child, + request(method, params = {}) { + return new Promise((resolve, reject) => { + const id = ++sequence; + const timer = setTimeout(() => { + pending.delete(id); + reject(new Error(`Fixture RPC timed out: ${method}\n${stderr}`)); + }, requestTimeoutMs); + pending.set(id, { resolve, reject, timer }); + child.stdin.write( + `${JSON.stringify({ jsonrpc: "2.0", id, method, params })}\n`, + ); + }); + }, + async call(name, args, meta) { + const result = await this.request("tools/call", { + name, + arguments: args, + ...(meta ? { _meta: meta } : {}), + }); + assert.notEqual(result.isError, true, JSON.stringify(result)); + return result.structuredContent ?? JSON.parse(result.content[0].text); + }, + async close() { + if (child.exitCode !== null || child.signalCode !== null) return; + child.stdin.end(); + const timeout = setTimeout(() => child.kill("SIGKILL"), 5_000); + try { + await exited; + } finally { + clearTimeout(timeout); + } + }, + }; + try { + await client.request("initialize", { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "installed-deep-fixture", version: "1.0.0" }, + }); + child.stdin.write( + `${JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" })}\n`, + ); + return client; + } catch (error) { + await client.close(); + throw error; + } +} diff --git a/sdk/typescript/scripts/smoke-package.mjs b/sdk/typescript/scripts/smoke-package.mjs index b610435b7c..65dff98927 100644 --- a/sdk/typescript/scripts/smoke-package.mjs +++ b/sdk/typescript/scripts/smoke-package.mjs @@ -822,6 +822,15 @@ try { ); await smokeNestedDeepScanWorker(installedRoot, consumer); + run( + process.execPath, + [ + join(packageRoot, "scripts", "fixtures", "package-deep-scan.mjs"), + installedRoot, + ], + { cwd: consumer }, + ); + console.log( `Validated installed ${packageManifest.name}@${packageManifest.version}: public import, NodeNext types, CLI, SDK lifecycle, credential locking, ${expectedPluginFiles.length} bundled plugin files, MCP initialization, bundled Codex version, dashboard assets, and a nested worker without global codex.`, ); diff --git a/sdk/typescript/src/accepted-audit.ts b/sdk/typescript/src/accepted-audit.ts new file mode 100644 index 0000000000..ee526a955f --- /dev/null +++ b/sdk/typescript/src/accepted-audit.ts @@ -0,0 +1,49 @@ +export interface ScanDraftInput { + scanId: string; + complete?: boolean; + handoffClaimToken?: string; + scope?: Record; + threatModel?: Record; + findings: Record[]; + coverage: Record; +} + +/** Accepted evidence may still describe partial or unknown source coverage. */ +export interface AuditEvidence { + checkpoint?: ScanDraftInput; + accepted?: ScanDraftInput; +} + +export type AuditOutcome = AuditEvidence & + ( + | { status: "accepted"; execution: Execution; accepted: ScanDraftInput } + | { status: "checkpoint"; execution: Execution } + ); + +/** One attempt; enclosing callers own retries and public completion. */ +export async function runAcceptedAudit(input: { + signal: AbortSignal; + execute: () => Promise; + accept: (execution: Execution) => Promise; +}): Promise> { + input.signal.throwIfAborted(); + const execution = await input.execute(); + input.signal.throwIfAborted(); + const evidence = await input.accept(execution); + input.signal.throwIfAborted(); + return evidence.accepted === undefined + ? { ...evidence, execution, status: "checkpoint" } + : { + ...evidence, + execution, + status: "accepted", + accepted: evidence.accepted, + }; +} + +/** Process completion alone does not accept an unfinished audit checkpoint. */ +export function auditEvidence(checkpoint: ScanDraftInput): AuditEvidence { + return checkpoint.complete === false + ? { checkpoint } + : { checkpoint, accepted: checkpoint }; +} diff --git a/sdk/typescript/src/api.ts b/sdk/typescript/src/api.ts index 0e230e9526..91162b47fa 100644 --- a/sdk/typescript/src/api.ts +++ b/sdk/typescript/src/api.ts @@ -1,6 +1,15 @@ /// -import { isSafeNonNegativeInteger as safeInteger } from "./value.js"; +import { scanPreflightCodexConfig } from "./preflight-config.js"; +import { captureOriginalReasoningSummary } from "./reasoning-summary.js"; +export { scanPreflightCodexConfig } from "./preflight-config.js"; +import { resumeSelectedDeepScan } from "./deep-scan-finalization.js"; +import { + auditEvidence, + runAcceptedAudit, + type ScanDraftInput, +} from "./accepted-audit.js"; +import { pathToFileURL } from "node:url"; import { chmod, lstat, @@ -24,7 +33,6 @@ import { sep, } from "node:path"; import { - Codex, type CodexOptions, type ThreadOptions, type TurnOptions, @@ -33,6 +41,13 @@ import { z } from "incur"; import { readThreatModelPath } from "./artifact-export.js"; import { isRecord } from "./record.js"; +import { + createCodexClient, + readCodexSessionTurn, + type CodexSessionClient as CodexClientLike, + type CodexSessionThread as CodexThreadLike, + type CodexSessionEvent as ScanEvent, +} from "./codex-session.js"; import { CODEX_AUTH_CONFIG_KEYS, NO_CREDENTIALS_MESSAGE, @@ -74,6 +89,7 @@ import { type ScanCost, type ScanSessionEvent, } from "./cost.js"; +import type { ScanExecutionAttribution } from "./scan-sessions.js"; import { DeepScanProgressTracker, type DeepScanProgress, @@ -230,24 +246,6 @@ import { type InspectedExecutable, } from "./trusted-executable.js"; -interface CodexThreadLike { - readonly id: string | null; - runStreamed( - input: string, - options: TurnOptions, - ): Promise<{ events: AsyncGenerator }>; -} - -interface ScanEvent { - readonly type: string; - readonly [key: string]: unknown; -} - -interface CodexClientLike { - startThread(options: ThreadOptions): CodexThreadLike; - resumeThread?(threadId: string, options: ThreadOptions): CodexThreadLike; -} - interface PreparedRuntime { codexHome: string; bootstrapWorkspace?: string; @@ -460,7 +458,7 @@ interface ClientDependencies { } const DEFAULT_DEPENDENCIES: ClientDependencies = { - createCodex: (options) => new Codex(options), + createCodex: createCodexClient, environment: process.env, }; @@ -1016,15 +1014,7 @@ export class CodexSecurity { `permissions.${POLICY_PERMISSION_PROFILE}.filesystem=${inlineToml(policyFilesystemPermissions(inputs.gitMetadataPaths))}`, ], ); - const reportCost = (current: Readonly): void => { - const total = addScanCosts(accumulatedCost, current); - if (completeCost) - notifyObserver( - "onCost", - options.onCost, - options.onObserverError, - total, - ); + const enforceCostLimit = (total: Readonly): void => { if ( options.maxCostUsd !== undefined && total.estimatedUsd > options.maxCostUsd @@ -1036,6 +1026,17 @@ export class CodexSecurity { ); } }; + const reportCost = (current: Readonly): void => { + const total = addScanCosts(accumulatedCost, current); + if (completeCost) + notifyObserver( + "onCost", + options.onCost, + options.onObserverError, + total, + ); + enforceCostLimit(total); + }; const outputSchema = securityPolicyStageOutputSchema(); const run = async ( stage: SecurityPolicyStage, @@ -1059,6 +1060,10 @@ export class CodexSecurity { options.onCost === undefined && options.maxCostUsd === undefined ? undefined : reportCost, + onCostLowerBound: + options.maxCostUsd === undefined + ? undefined + : (cost) => enforceCostLimit(addScanCosts(accumulatedCost, cost)), onError: (error) => { if (options.maxCostUsd !== undefined) budgetController.abort(error); else @@ -1225,14 +1230,35 @@ export class CodexSecurity { let preparedTargetWarnings: string[] = []; let runPostScan: (() => ReturnType) | null = null; + let selectedDeepFinalization = false; + let observedScanThreadId: string | undefined; let activeScan: { id: string; options: WorkbenchCommandOptions; + mode: ScanMode; } | null = null; const prepareArtifactRestorer = this.#dependencies.prepareScanArtifactRestorer ?? prepareScanArtifactRestorer; const workbench = this.#dependencies.runWorkbench ?? runWorkbench; + const recoverCompletedScan = async ( + commandOptions: WorkbenchCommandOptions, + scanId: string, + error: unknown, + completionArgs: readonly string[], + ): Promise => { + const saved = await workbench(commandOptions, [ + "get-scan", + "--scan-id", + scanId, + ]).catch(() => null); + const savedScan = saved?.["scan"]; + const progress = isRecord(savedScan) ? savedScan["progress"] : null; + if (!isRecord(progress) || progress["status"] !== "complete") throw error; + // A lost response can follow a durable seal. Only the existing normal + // completion command can validate and return that committed receipt. + return workbench(commandOptions, completionArgs); + }; try { const checkOpen = (): void => { this.#requireOpen(); @@ -1294,13 +1320,12 @@ export class CodexSecurity { const { runtime, runtimeHome, - effectiveConfig, - preflightConfig, modelProvider, authentication, approvalPolicy, python, } = session; + let { effectiveConfig, preflightConfig } = session; releaseCredentialHome = session.releaseCredentialHome; let git: InspectedExecutable = { executable: null, @@ -1363,6 +1388,47 @@ export class CodexSecurity { ); checkOpen(); + if (mode === "deep" && options.resumeScanId === undefined) { + const summary = await captureOriginalReasoningSummary({ + config: session.sessionConfig, + command: this.#codexCommand(), + cwd: scanDir, + environment: { + ...withoutOpenAiApiKeys( + this.#createSessionEnvironment(session, {}, options.auth), + ), + ...(session.externalProvider === null && session.apiKey !== null + ? { CODEX_API_KEY: session.apiKey } + : {}), + }, + signal, + }); + if (summary !== undefined) { + effectiveConfig = { + ...effectiveConfig, + model_reasoning_summary: summary, + }; + preflightConfig = scanPreflightCodexConfig(effectiveConfig); + session.effectiveConfig = effectiveConfig; + session.preflightConfig = preflightConfig; + session.sessionConfig = { + ...session.sessionConfig, + model_reasoning_summary: summary, + }; + if (runtime.configPath !== undefined) + await writeCodexConfig(runtime.configPath, { + ...preflightConfig, + ...(options.cyberAccessProgram === undefined + ? {} + : { + codex_security: { + cyber_access_program: options.cyberAccessProgram, + }, + }), + }); + } + } + const shellPluginRoot = runtime.plugin.pluginRoot; const canonicalShellPluginRoot = await realpath(shellPluginRoot); const pluginRelativeToHome = relative( @@ -1450,6 +1516,14 @@ export class CodexSecurity { `Could not track scan activity: ${errorMessage(error)}`, ); }; + const enforceCostLimit = (cost: Readonly): boolean => { + if (maxCostUsd === undefined || cost.estimatedUsd <= maxCostUsd) + return false; + costAbortController.abort( + new ScanCostLimitExceededError(maxCostUsd, cost, scanDir), + ); + return true; + }; const tracker = new ScanCostTracker({ codexHome: runtime.codexHome, model, @@ -1490,15 +1564,7 @@ export class CodexSecurity { cost, maxCostUsd, ); - if ( - maxCostUsd !== undefined && - cost.estimatedUsd > maxCostUsd - ) { - costAbortController.abort( - new ScanCostLimitExceededError(maxCostUsd, cost, scanDir), - ); - return; - } + if (enforceCostLimit(cost)) return; const request = options.onBudgetApproaching; if ( request === undefined || @@ -1563,6 +1629,8 @@ export class CodexSecurity { } }); }, + onCostLowerBound: + options.maxCostUsd === undefined ? undefined : enforceCostLimit, onError: reportTrackingError, }); costTracker = tracker; @@ -1732,7 +1800,23 @@ export class CodexSecurity { }, ); } - activeScan = { id: scanId, options: workbenchOptions }; + activeScan = { id: scanId, options: workbenchOptions, mode }; + if (mode === "deep") { + tracker.setAttributionReader(async () => { + const context = await workbench( + { ...workbenchOptions, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + const scan = context["scan"]; + if (isRecord(scan) && !("executionAttribution" in scan)) + return undefined; + return isRecord(scan) && isRecord(scan["executionAttribution"]) + ? (scan[ + "executionAttribution" + ] as unknown as ScanExecutionAttribution) + : null; + }); + } if (mode === "deep" && options.onDeepProgress !== undefined) { let progressWarningReported = false; deepProgressTracker = new DeepScanProgressTracker({ @@ -1909,6 +1993,7 @@ export class CodexSecurity { ); } thread = codex.resumeThread(resumeThreadId, threadOptions); + observedScanThreadId = resumeThreadId; tracker.start(resumeThreadId); if (budgetRecovery !== null) budgetRecovery.threadId = resumeThreadId; await tracker.refresh().catch(reportTrackingError); @@ -1936,14 +2021,66 @@ export class CodexSecurity { if (postScanPrompt?.trim()) { runPostScan = () => thread.runStreamed(postScanPrompt, turnOptions); } - const { events } = await thread.runStreamed(prompt, turnOptions); + const recoverSelectedCompletion = async () => { + const threadId = observedScanThreadId ?? thread.id; + if (mode !== "deep" || !threadId || signal.aborted) return null; + const saved = await workbench(workbenchOptions, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + threadId, + ]).catch(() => null); + const deep = saved?.["deepScan"]; + if ( + !isRecord(deep) || + !isRecord(deep["finalizationInput"]) || + (deep["status"] !== "running" && deep["status"] !== "succeeded") + ) + return null; + selectedDeepFinalization = true; + await resumeSelectedDeepScan({ + scanId, + threadId, + pluginRoot: runtime.plugin.installedRoot, + signal, + runWorkbench: (args) => workbench(workbenchOptions, args), + }); + return { + status: "completed" as const, + threadId, + finalResponse: "", + usage: null, + lastStreamError: null, + }; + }; + const savedCompletion = resumeThreadId + ? await recoverSelectedCompletion() + : null; + const events = (async function* () { + if (savedCompletion) return; + yield* (await thread.runStreamed(prompt, turnOptions)).events; + })(); checkOpen(); let result = await runScanEvents({ + savedCompletion: savedCompletion ?? undefined, + recoverCompletion: recoverSelectedCompletion, + reconcileCheckpoint: + mode === "deep" + ? undefined + : async () => { + await workbench(workbenchOptions, [ + "prepare-scan-completion", + "--scan-id", + scanId, + ]); + }, thread, events, signal, scanDir, + scanId, pluginRoot: runtime.plugin.installedRoot, pythonPath: session.python, protectedRoot, @@ -1953,6 +2090,7 @@ export class CodexSecurity { workbenchValidated: true, model, onThreadStarted: async (threadId) => { + observedScanThreadId = threadId; if (resumeThreadId !== undefined) { if (threadId !== resumeThreadId) { throw new CodexSecurityError( @@ -1981,6 +2119,7 @@ export class CodexSecurity { } }, onFinalize: async (usage) => { + await recoverSelectedCompletion(); if (options.validationPrompt !== undefined) { tracker.recordUsage(usage); await tracker.refresh().catch(reportTrackingError); @@ -2046,7 +2185,10 @@ export class CodexSecurity { return { usage, cost: estimateScanCost(model, usage) }; }); throwIfAborted(signal, scanDir); - if (options.maxCostUsd !== undefined && snapshot.cost === null) { + if ( + options.maxCostUsd !== undefined && + (snapshot.cost === null || snapshot.cost.coverage === "partial") + ) { notifyObserver( "onWarning", options.onWarning, @@ -2113,14 +2255,20 @@ export class CodexSecurity { onObserverError: options.onObserverError, }); checkOpen(); - const completion = await workbench(workbenchOptions, [ + const completionArgs = [ "complete-scan", "--scan-id", scanId, ...(completionCost === null ? [] : ["--cost-json", JSON.stringify(completionCost)]), - ]); + ]; + const completion = await workbench( + workbenchOptions, + completionArgs, + ).catch((error) => + recoverCompletedScan(workbenchOptions, scanId, error, completionArgs), + ); activeScan = null; const completedScan = completion["scan"]; if (isRecord(completedScan) && Array.isArray(completedScan["warnings"])) { @@ -2175,7 +2323,7 @@ export class CodexSecurity { let artifactRestorer: ScanArtifactRestorer | null = null; try { artifactRestorer = await prepareArtifactRestorer( - workbenchOptions, + { ...workbenchOptions, signal: undefined }, scanDir, ); const followUpResult = await runScanEvents({ @@ -2183,6 +2331,7 @@ export class CodexSecurity { events: (await followUp()).events, signal, scanDir, + scanId, pluginRoot: runtime.plugin.installedRoot, pythonPath: session.python, protectedRoot, @@ -2317,17 +2466,57 @@ export class CodexSecurity { options.signal?.aborted !== true ) { try { - const completion = await workbench( - { ...activeScan.options, signal: undefined }, - [ - "complete-budget-exhausted-scan", + const budgetScanId = activeScan.id; + const budgetCost = snapshot?.cost ?? { lowerBound: failure.cost }; + const completionSignal = AbortSignal.any([ + this.#abortController.signal, + ...(options.signal === undefined ? [] : [options.signal]), + ]); + const completionOptions = { + ...activeScan.options, + signal: completionSignal, + }; + const saved = await workbench(completionOptions, [ + "get-deep-scan", + "--scan-id", + activeScan.id, + "--thread-id", + budgetRecovery.threadId, + ]).catch(() => null); + const deep = saved?.["deepScan"]; + if ( + isRecord(deep) && + deep["status"] === "running" && + isRecord(deep["finalizationInput"]) + ) { + // Cost stops model work, but an already selected result can still + // finish through the local publisher. Caller cancellation remains live. + selectedDeepFinalization = true; + await resumeSelectedDeepScan({ + scanId: activeScan.id, + threadId: budgetRecovery.threadId, + pluginRoot: budgetRecovery.pluginRoot, + signal: completionSignal, + runWorkbench: (args) => workbench(completionOptions, args), + }); + } + const completion = await workbench(completionOptions, [ + "complete-budget-exhausted-scan", + "--scan-id", + budgetScanId, + "--cost-json", + JSON.stringify(budgetCost), + "--message", + failure.message.slice(0, 2400), + ]).catch((error) => + recoverCompletedScan(completionOptions, budgetScanId, error, [ + "complete-scan", "--scan-id", - activeScan.id, - "--cost-json", - JSON.stringify(snapshot?.cost ?? failure.cost), - "--message", - failure.message.slice(0, 2400), - ], + budgetScanId, + ...(snapshot?.cost + ? ["--cost-json", JSON.stringify(snapshot.cost)] + : []), + ]), ); activeScan = null; runPostScan = null; @@ -2341,10 +2530,7 @@ export class CodexSecurity { scanDir, budgetRecovery.pluginRoot, budgetRecovery.expectation, - AbortSignal.any([ - this.#abortController.signal, - ...(options.signal === undefined ? [] : [options.signal]), - ]), + completionSignal, true, budgetRecovery.pythonPath, budgetRecovery.protectedRoot, @@ -2385,9 +2571,32 @@ export class CodexSecurity { return result; } catch {} } - // A failed attachment must not turn a resumable coordinator into a terminal failure. - // Deep Scan orchestration persists its own terminal failures and cancellations. - if (activeScan !== null && options.resumeScanId === undefined) { + const callerCanceledDeepScan = + activeScan?.mode === "deep" && options.signal?.aborted; + if (activeScan !== null && callerCanceledDeepScan) { + const workbenchOptions = { ...activeScan.options, signal: undefined }; + // The workbench owns the running-state check and repeated cancellation. + // Selection may have committed before the SDK received its response. + await workbench(workbenchOptions, [ + "cancel-scan", + "--scan-id", + activeScan.id, + ...(observedScanThreadId === undefined + ? [] + : ["--thread-id", observedScanThreadId]), + ]).catch(() => undefined); + } + // Publication failures remain resumable. A cost stop or explicit client close + // still uses the existing failure path to retain partial results and stop work. + if ( + activeScan !== null && + !callerCanceledDeepScan && + ((options.resumeScanId === undefined && !selectedDeepFinalization) || + (selectedDeepFinalization && + !options.signal?.aborted && + (failure instanceof ScanCostLimitExceededError || + this.#abortController.signal.aborted))) + ) { if ( options.validationPrompt !== undefined && !customValidationComplete @@ -2629,14 +2838,12 @@ export class CodexSecurity { } } - #createSessionCodex( + #createSessionEnvironment( session: PreparedSession, runtimePaths: Record, auth: ScanAuthMode = "auto", git?: InspectedExecutable, - config?: JsonObject, - configOverrides: string[] = [], - ): { codex: CodexClientLike; environment: ProcessEnvironment } { + ): ProcessEnvironment { const { runtime, python, @@ -2675,6 +2882,25 @@ export class CodexSecurity { if (session.safetyIdentifier !== undefined) { environment[SAFETY_IDENTIFIER_ENV] = session.safetyIdentifier; } + return environment; + } + + #createSessionCodex( + session: PreparedSession, + runtimePaths: Record, + auth: ScanAuthMode = "auto", + git?: InspectedExecutable, + config?: JsonObject, + configOverrides: string[] = [], + ): { codex: CodexClientLike; environment: ProcessEnvironment } { + const { externalProvider, apiKey, sessionConfig } = session; + const commandAuth = hasCommandAuth(sessionConfig); + const environment = this.#createSessionEnvironment( + session, + runtimePaths, + auth, + git, + ); const sdkCodexConfig = { ...(config ?? sessionConfig) }; // Projects and permissions already live in generated TOML files; the SDK // cannot safely encode their path and selector keys as dotted overrides. @@ -3595,7 +3821,14 @@ async function removeTargetPathsFile(path: string | null): Promise { } interface ScanEventRunOptions extends Pick { - thread: Pick; + scanId?: string; + savedCompletion?: Awaited>; + recoverCompletion?: () => Promise + > | null>; + reconcileCheckpoint?: () => Promise; + thread: Pick & + Partial>; events: AsyncGenerator; signal: AbortSignal; scanDir: string; @@ -3626,116 +3859,182 @@ export async function runScanEvents( let scanStarted = false; let tacStatusReported = false; try { - const turn = await readCodexTurn({ - thread: options.thread, - events: options.events, - onEvent: async (event) => { - if ( - !tacStatusReported && - options.modelProvider !== "amazon-bedrock" && - options.authentication?.method !== "aws_credentials" - ) { - const tacStatus = trustedAccessStatusFromEvent(event); - if (tacStatus !== null) { - tacStatusReported = true; - notifyObserver( - "onTrustedAccessStatus", - options.onTrustedAccessStatus, - options.onObserverError, - tacStatus, - ); - if (tacStatus !== "granted") { + let completedTurn: + | (Awaited> & { + threadId: string; + status: "completed"; + }) + | undefined; + const execute = async () => { + const turn = + options.savedCompletion ?? + (await readCodexTurn({ + thread: options.thread, + events: options.events, + onEvent: async (event) => { + if ( + !tacStatusReported && + options.modelProvider !== "amazon-bedrock" && + options.authentication?.method !== "aws_credentials" + ) { + const tacStatus = trustedAccessStatusFromEvent(event); + if (tacStatus !== null) { + tacStatusReported = true; + notifyObserver( + "onTrustedAccessStatus", + options.onTrustedAccessStatus, + options.onObserverError, + tacStatus, + ); + if (tacStatus !== "granted") { + notifyObserver( + "onWarning", + options.onWarning, + options.onObserverError, + trustedAccessWarning(tacStatus, options.authentication), + ); + } + } + } + for (const activity of scanActivitiesFromEvent( + event, + options.expectation.repository, + )) { notifyObserver( - "onWarning", - options.onWarning, + "onActivity", + options.onActivity, options.onObserverError, - trustedAccessWarning(tacStatus, options.authentication), + activity, ); } - } - } - for (const activity of scanActivitiesFromEvent( - event, - options.expectation.repository, - )) { - notifyObserver( - "onActivity", - options.onActivity, - options.onObserverError, - activity, - ); - } - for (const progress of scanProgressUpdatesFromEvent(event)) { - if ( - options.expectedFilesTotal !== undefined && - progress.filesTotal !== options.expectedFilesTotal - ) { - continue; - } - notifyObserver( - "onProgress", - options.onProgress, - options.onObserverError, - progress, - ); - } - const workerStatus = workerStatusFromEvent(event); - if (workerStatus !== null) { - notifyObserver( - "onWorkerStatus", - options.onWorkerStatus, - options.onObserverError, - workerStatus, - ); - } - if (event.type === "thread.started") { - const startedThreadId = event["thread_id"]; - if (typeof startedThreadId === "string") { - await options.onThreadStarted?.(startedThreadId); - } - if (!scanStarted) { - scanStarted = true; + for (const progress of scanProgressUpdatesFromEvent(event)) { + if ( + options.expectedFilesTotal !== undefined && + progress.filesTotal !== options.expectedFilesTotal + ) { + continue; + } + notifyObserver( + "onProgress", + options.onProgress, + options.onObserverError, + progress, + ); + } + const workerStatus = workerStatusFromEvent(event); + if (workerStatus !== null) { + notifyObserver( + "onWorkerStatus", + options.onWorkerStatus, + options.onObserverError, + workerStatus, + ); + } + if (event.type === "thread.started") { + const startedThreadId = event["thread_id"]; + if (typeof startedThreadId === "string") { + await options.onThreadStarted?.(startedThreadId); + } + if (!scanStarted) { + scanStarted = true; + notifyObserver( + "onScanStarted", + options.onScanStarted, + options.onObserverError, + ); + } + } + }, + onReconnect: (message, reconnect) => { notifyObserver( - "onScanStarted", - options.onScanStarted, + "onReconnect", + options.onReconnect, options.onObserverError, + ...reconnect, + reconnectDetails(message), ); - } - } - }, - onReconnect: (message, reconnect) => { - notifyObserver( - "onReconnect", - options.onReconnect, - options.onObserverError, - ...reconnect, - reconnectDetails(message), + }, + })); + const { status, threadId, lastStreamError } = turn; + if (status !== "completed") { + throw new IncompleteScanError( + lastStreamError ?? + "Codex Security event stream ended before the turn completed.", ); - }, - }); - const { status, threadId, finalResponse, lastStreamError } = turn; - let { usage } = turn; - if (options.signal.aborted) { - throw new ScanInterruptedError( - `Codex Security scan was interrupted; partial output remains at ${options.scanDir}.`, - options.scanDir, - ); - } - if (status !== "completed") { - throw new IncompleteScanError( - lastStreamError ?? - "Codex Security event stream ended before the turn completed.", - ); + } + if (threadId === null) { + throw new IncompleteScanError( + "Codex Security did not report a thread ID.", + ); + } + return (completedTurn = { ...turn, threadId, status }); + }; + const accept = async () => { + // Matching, custom validation and the canonical seal remain with the caller. + const readDocuments = () => + Promise.all( + ["scan-manifest.json", "findings.json", "coverage.json"].map( + async (name) => + JSON.parse( + ( + await readScanFile( + options.scanDir, + name, + name, + options.signal, + ) + ).toString("utf8"), + ), + ), + ); + let [manifest, findings, coverage] = await readDocuments(); + if (manifest?.scan?.complete === false && options.reconcileCheckpoint) { + await options.reconcileCheckpoint(); + [manifest, findings, coverage] = await readDocuments(); + } + const helper = ( + await import( + pathToFileURL(join(await bundledPluginRoot(), "mcp/helpers.mjs")).href + ) + ).default; + const draft: ScanDraftInput = helper.parseCanonicalScanDraft({ + scanId: options.scanId ?? manifest.scan.id, + manifest, + findings, + coverage, + }); + return auditEvidence(draft); + }; + let audit; + try { + audit = await runAcceptedAudit({ + signal: options.signal, + execute, + accept, + }); + } catch (error) { + if (options.signal.aborted) throw error; + const saved = await options.recoverCompletion?.(); + if (!saved || saved.status !== "completed" || saved.threadId === null) + throw error; + // The enclosing scan publishes its saved selection; acceptance then reads it. + const recovered = completedTurn ?? { ...saved, threadId: saved.threadId }; + audit = await runAcceptedAudit({ + signal: options.signal, + execute: async () => recovered, + accept, + }); } - if (threadId === null) { + if (audit.status === "checkpoint") throw new IncompleteScanError( - "Codex Security did not report a thread ID.", + "Codex Security produced only an unfinished audit checkpoint.", ); - } + const { status, threadId, finalResponse } = audit.execution; + let { usage } = audit.execution; if (options.onFinalize !== undefined) { usage = (await options.onFinalize(usage)) ?? usage; } - const result = await collectResult( + return await collectResult( { status, finalResponse, @@ -3751,13 +4050,6 @@ export async function runScanEvents( options.pythonPath, options.protectedRoot, ); - if (options.signal.aborted) { - throw new ScanInterruptedError( - `Codex Security scan was interrupted; partial output remains at ${options.scanDir}.`, - options.scanDir, - ); - } - return result; } catch (error) { if (options.signal.reason instanceof ScanCostLimitExceededError) { throw options.signal.reason; @@ -3785,61 +4077,28 @@ async function readCodexTurn(options: { usage: unknown; lastStreamError: string | null; }> { - let threadId = options.thread.id; - let status: "in_progress" | "completed" = "in_progress"; - let finalResponse = ""; - let usage: unknown = null; - let lastStreamError: string | null = null; - for await (const event of eventsWithOptionalUsage(options.events)) { - await options.onEvent?.(event); - if ( - event.type === "thread.started" && - typeof event["thread_id"] === "string" - ) { - threadId = event["thread_id"]; - } else if ( - event.type === "item.completed" && - isRecord(event["item"]) && - event["item"]["type"] === "agent_message" && - typeof event["item"]["text"] === "string" - ) { - finalResponse = event["item"]["text"]; - } else if (event.type === "turn.completed") { - status = "completed"; - usage = event["usage"]; - } else if (event.type === "turn.failed") { - throw new CodexSecurityError(turnFailureMessage(event["error"])); - } else if (event.type === "error" && typeof event["message"] === "string") { - const message = event["message"]; - const classification = classifyConnectionFailure(message); - if (classification === "unauthorized" || classification === "forbidden") { - throw new CodexSecurityError(message); + return readCodexSessionTurn({ + ...options, + onEvent: async (event) => { + await options.onEvent?.(event); + if (event.type === "turn.failed") { + throw new CodexSecurityError(turnFailureMessage(event["error"])); } - const reconnect = reconnectAttempt(message); - if (reconnect === null) throw new CodexSecurityError(message); - lastStreamError = message; - options.onReconnect?.(message, reconnect); - } - } - return { threadId, status, finalResponse, usage, lastStreamError }; -} - -async function* eventsWithOptionalUsage( - events: AsyncGenerator, -): AsyncGenerator { - try { - yield* events; - } catch (error) { - if ( - error instanceof TypeError && - /\b(?:null|undefined)\b/u.test(error.message) && - /\bcache_write_input_tokens\b/u.test(error.message) - ) { - yield { type: "turn.completed", usage: null }; - return; - } - throw error; - } + if (event.type === "error" && typeof event["message"] === "string") { + const message = event["message"]; + const classification = classifyConnectionFailure(message); + if ( + classification === "unauthorized" || + classification === "forbidden" + ) { + throw new CodexSecurityError(message); + } + const reconnect = reconnectAttempt(message); + if (reconnect === null) throw new CodexSecurityError(message); + options.onReconnect?.(message, reconnect); + } + }, + }); } function trustedAccessStatusFromEvent( @@ -3949,7 +4208,7 @@ function scanPrompt( : []), ...(mode === "deep" ? [ - `The SDK has already registered this scan. Call start_codex_security_deep_scan with ${JSON.stringify({ scanId })}; never pass targetPath or create another scan.`, + `The SDK has already registered this scan. Call start_codex_security_deep_scan with ${JSON.stringify({ scanId })}; never pass targetPath or create another scan. Leave finalization to the SDK after this turn; do not call a completion tool.`, ] : skillName === "security-scan" || customValidation ? [ @@ -4010,7 +4269,7 @@ function scanPrompt( ] : skillName === "deep-security-scan" ? [ - "The Deep Scan coordinator already wrote the canonical scan artifacts. Call complete_codex_security_scan exactly once without submitting another semantic draft; the workbench owns authoritative metadata, finalization, report generation, and sealing.", + "The Deep Scan coordinator already wrote the canonical scan artifacts. Leave them unchanged and end the scan turn without calling a completion tool; the SDK accounts for the completed turn, enforces its budget, and owns finalization, report generation, and sealing.", ] : [ "Use record_codex_security_scan_draft and complete_codex_security_scan as directed by the selected skill; the workbench owns authoritative metadata, finalization, report generation, and sealing.", @@ -4142,6 +4401,19 @@ function addScanCosts( previous.cacheWriteInputTokens + current.cacheWriteInputTokens, outputTokens: previous.outputTokens + current.outputTokens, estimatedUsd: previous.estimatedUsd + current.estimatedUsd, + ...(previous.coverage === "partial" || current.coverage === "partial" + ? { coverage: "partial" as const } + : {}), + ...(previous.modelCosts || + current.modelCosts || + previous.model !== current.model + ? { + modelCosts: [ + ...(previous.modelCosts ?? [previous]), + ...(current.modelCosts ?? [current]), + ], + } + : {}), ...(previous.cacheWriteInputTokensReported === false || current.cacheWriteInputTokensReported === false ? { cacheWriteInputTokensReported: false } @@ -4636,152 +4908,6 @@ function sharedCredentialCodexConfig( return scanRuntimeCodexConfig(shared, credentialHome); } -export function scanPreflightCodexConfig(config: JsonObject): JsonObject { - const safeString = (value: unknown): value is string => - typeof value === "string" && - value.length > 0 && - !/[\u0000-\u001f\u007f]/u.test(value); - const safeProfileName = (value: unknown): value is string => - safeString(value) && /^[A-Za-z0-9_-]+$/u.test(value); - const capabilityFeatures = (value: unknown): JsonObject => { - if (!isRecord(value)) return {}; - const result: JsonObject = {}; - for (const key of [ - "goals", - "multi_agent", - "enable_fanout", - "api_key_cyber_access_programs", - "api_key_model_discovery", - ]) { - if (typeof value[key] === "boolean") result[key] = value[key]; - } - const multiAgent = value["multi_agent_v2"]; - if (typeof multiAgent === "boolean") { - result["multi_agent_v2"] = multiAgent; - } else if (isRecord(multiAgent)) { - const sanitized: JsonObject = {}; - if (typeof multiAgent["enabled"] === "boolean") { - sanitized["enabled"] = multiAgent["enabled"]; - } - const capacity = multiAgent["max_concurrent_threads_per_session"]; - if (safeInteger(capacity)) { - sanitized["max_concurrent_threads_per_session"] = capacity; - } - if (Object.keys(sanitized).length > 0) { - result["multi_agent_v2"] = sanitized; - } - } - return result; - }; - const executionConfig = (source: JsonObject): JsonObject => { - const result: JsonObject = {}; - for (const key of [ - "model", - "model_reasoning_effort", - "model_reasoning_summary", - "model_provider", - "service_tier", - ]) { - const value = source[key]; - if (safeString(value)) result[key] = value; - } - const features = capabilityFeatures(source["features"]); - if (Object.keys(features).length > 0) result["features"] = features; - const agents = source["agents"]; - if (isRecord(agents)) { - const sanitized: JsonObject = {}; - for (const key of ["max_threads", "max_depth"]) { - const value = agents[key]; - if (safeInteger(value)) sanitized[key] = value; - } - if (Object.keys(sanitized).length > 0) result["agents"] = sanitized; - } - const multiagent = source["multiagent_config"]; - if (isRecord(multiagent) && safeInteger(multiagent["max_concurrency"])) { - result["multiagent_config"] = { - max_concurrency: multiagent["max_concurrency"], - }; - } - return result; - }; - const result = executionConfig(config); - // Keep effective worker settings even when preflight filters the profile name. - const resolved = resolveCodexProfile(config); - for (const key of ["model_reasoning_summary", "service_tier"]) { - const value = resolved[key]; - if (safeString(value)) result[key] = value; - } - const resolvedFeatures = capabilityFeatures(resolved["features"]); - for (const key of [ - "api_key_cyber_access_programs", - "api_key_model_discovery", - ]) { - if (resolvedFeatures[key] !== undefined) { - result["features"] = { - ...(isRecord(result["features"]) ? result["features"] : {}), - [key]: resolvedFeatures[key], - }; - } - } - const selectedProfile = safeProfileName(config["profile"]) - ? config["profile"] - : undefined; - if (selectedProfile !== undefined) { - result["profile"] = selectedProfile; - } - const profiles = config["profiles"]; - if (isRecord(profiles)) { - const sanitized: JsonObject = {}; - for (const [name, profile] of Object.entries(profiles)) { - if (!safeProfileName(name) || !isRecord(profile)) continue; - const projected = executionConfig(profile as JsonObject); - if (Object.keys(projected).length === 0) continue; - sanitized[name] = projected; - } - if (Object.keys(sanitized).length > 0) result["profiles"] = sanitized; - } - const modelProvider = scanModelProvider(result); - if (isExternalModelProvider(modelProvider)) { - result["model_providers"] = { - [modelProvider]: { ...EXTERNAL_CODEX_PROVIDERS[modelProvider] }, - }; - } else if (modelProvider === "amazon-bedrock") { - const providers = config["model_providers"]; - const provider = isRecord(providers) ? providers[modelProvider] : undefined; - const aws = isRecord(provider) ? provider["aws"] : undefined; - if (isRecord(aws)) { - const sanitized: JsonObject = {}; - for (const key of ["region", "profile"]) { - const value = aws[key]; - if (safeString(value)) sanitized[key] = value; - } - if (Object.keys(sanitized).length > 0) { - result["model_providers"] = { - [modelProvider]: { aws: sanitized }, - }; - } - } - } - const rootMarkers = config["project_root_markers"]; - if (Array.isArray(rootMarkers)) { - result["project_root_markers"] = rootMarkers.filter(safeString); - } - const projects = config["projects"]; - if (isRecord(projects)) { - const sanitized: JsonObject = {}; - for (const [path, project] of Object.entries(projects)) { - if (!safeString(path) || !isAbsolute(path) || !isRecord(project)) { - continue; - } - const trust = project["trust_level"]; - if (trust !== "trusted" && trust !== "untrusted") continue; - sanitized[path] = { trust_level: trust }; - } - if (Object.keys(sanitized).length > 0) result["projects"] = sanitized; - } - return result; -} - async function pluginSupportsIsolatedDeepScanConfig( pluginRoot: string, ): Promise { diff --git a/sdk/typescript/src/codex-session.ts b/sdk/typescript/src/codex-session.ts new file mode 100644 index 0000000000..751aba8a5e --- /dev/null +++ b/sdk/typescript/src/codex-session.ts @@ -0,0 +1,94 @@ +import { + Codex, + type CodexOptions, + type ThreadOptions, + type TurnOptions, +} from "@openai/codex-sdk"; + +export interface CodexSessionEvent { + readonly type: string; + readonly [key: string]: unknown; +} + +export interface CodexSessionThread { + readonly id: string | null; + runStreamed( + input: string, + options: TurnOptions, + ): Promise<{ events: AsyncGenerator }>; +} + +export interface CodexSessionClient { + startThread(options: ThreadOptions): CodexSessionThread; + resumeThread?(threadId: string, options: ThreadOptions): CodexSessionThread; +} + +export const createCodexClient = (options: CodexOptions): CodexSessionClient => + new Codex(options); + +/** Reduce a single stream; callers retain error, retry and acceptance policy. */ +export async function readCodexSessionTurn(options: { + thread: Pick; + events: AsyncGenerator; + onEvent: (event: CodexSessionEvent) => Promise | void; + stopOnCompletion?: boolean; +}): Promise<{ + threadId: string | null; + status: "in_progress" | "completed"; + finalResponse: string; + usage: unknown; + lastStreamError: string | null; +}> { + let threadId = options.thread.id; + let status: "in_progress" | "completed" = "in_progress"; + let finalResponse = ""; + let usage: unknown = null; + let lastStreamError: string | null = null; + for await (const event of eventsWithOptionalUsage(options.events)) { + await options.onEvent(event); + if ( + event.type === "thread.started" && + typeof event["thread_id"] === "string" + ) { + threadId = event["thread_id"]; + } else if ( + event.type === "item.completed" && + isRecord(event["item"]) && + event["item"]["type"] === "agent_message" && + typeof event["item"]["text"] === "string" + ) { + finalResponse = event["item"]["text"]; + } else if (event.type === "turn.completed") { + status = "completed"; + usage = event["usage"] ?? null; + if (options.stopOnCompletion) break; + } else if (event.type === "error" && typeof event["message"] === "string") { + lastStreamError = event["message"]; + } + } + return { threadId, status, finalResponse, usage, lastStreamError }; +} + +async function* eventsWithOptionalUsage( + events: AsyncGenerator, +): AsyncGenerator { + try { + yield* events; + } catch (error) { + // The pinned SDK accesses this field before yielding a completion with + // absent usage. Preserve completion without inventing a zero-token receipt. + if ( + error instanceof TypeError && + /\b(?:null|undefined)\b/u.test(error.message) && + /\bcache_write_input_tokens\b/u.test(error.message) + ) { + yield { type: "turn.completed", usage: null }; + return; + } + throw error; + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} diff --git a/sdk/typescript/src/cost-model.ts b/sdk/typescript/src/cost-model.ts index dd0506d782..3852467553 100644 --- a/sdk/typescript/src/cost-model.ts +++ b/sdk/typescript/src/cost-model.ts @@ -10,6 +10,8 @@ export interface ScanCost { outputTokens: number; /** Short-context baseline retained for compatibility and spending limits. */ estimatedUsd: number; + coverage?: "partial"; + modelCosts?: readonly ScanCost[]; /** Standard token-cost bounds for the observed usage, not a billing total. */ estimatedUsdRange?: { min: number; @@ -179,6 +181,121 @@ export function tokenUsage(value: unknown): ScanTokenUsage | null { export function estimateScanCost( model: string | undefined, usage: unknown, +): ScanCost | null { + if (isRecord(usage) && Array.isArray(usage["modelUsage"])) { + const total = tokenUsage(usage); + if (total === null || usage["modelUsage"].length === 0) return null; + const costs: ScanCost[] = []; + for (const part of usage["modelUsage"]) { + if (!isRecord(part)) return null; + const tokens = tokenUsage(part); + if (tokens === null) return null; + const cost = + typeof part["model"] === "string" + ? estimateModelCost(part["model"], part) + : null; + if (cost === null) { + if (tokens.total_tokens === 0) continue; + return null; + } + costs.push(cost); + } + if (costs.length === 0) return null; + const sum = ( + key: + | "inputTokens" + | "cachedInputTokens" + | "cacheWriteInputTokens" + | "outputTokens" + | "estimatedUsd", + ) => costs.reduce((value, cost) => value + cost[key], 0); + if ( + sum("inputTokens") !== total.input_tokens || + sum("cachedInputTokens") !== total.cached_input_tokens || + sum("cacheWriteInputTokens") !== total.cache_write_input_tokens || + sum("outputTokens") !== total.output_tokens + ) + return null; + return { + model: model ?? costs[0]!.model, + inputTokens: total.input_tokens, + cachedInputTokens: total.cached_input_tokens, + cacheWriteInputTokens: total.cache_write_input_tokens, + ...(total.cache_write_input_tokens_reported === false + ? { cacheWriteInputTokensReported: false } + : {}), + outputTokens: total.output_tokens, + estimatedUsd: sum("estimatedUsd"), + estimatedUsdRange: { + min: sum("estimatedUsd"), + max: costs.some((cost) => cost.estimatedUsdRange?.max == null) + ? null + : costs.reduce( + (value, cost) => value + cost.estimatedUsdRange!.max!, + 0, + ), + context: "unknown", + }, + modelCosts: costs, + ...(costs.length === 1 ? { pricing: costs[0]!.pricing } : {}), + ...(usage["coverage"] === "partial" + ? { coverage: "partial" as const } + : {}), + }; + } + const cost = estimateModelCost(model, usage); + return cost && isRecord(usage) && usage["coverage"] === "partial" + ? { ...cost, coverage: "partial" } + : cost; +} + +// Internal budget enforcement only. The public estimate remains unavailable +// when some attributed usage has no price. +export function estimateScanCostLowerBound( + model: string | undefined, + usage: unknown, +): ScanCost | null { + if (!isRecord(usage) || !Array.isArray(usage["modelUsage"])) return null; + const total = tokenUsage(usage); + if (total === null) return null; + const keys = [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "reasoning_output_tokens", + ] as const; + const observed = Object.fromEntries(keys.map((key) => [key, 0])); + const priced = Object.fromEntries(keys.map((key) => [key, 0])); + const parts: Record[] = []; + let cacheWritesReported = true; + for (const part of usage["modelUsage"]) { + const normalized = tokenUsage(part); + if (!isRecord(part) || normalized === null) return null; + for (const key of keys) observed[key]! += normalized[key]; + if ( + typeof part["model"] !== "string" || + estimateModelCost(part["model"], part) === null + ) + continue; + parts.push(part); + for (const key of keys) priced[key]! += normalized[key]; + if (normalized.cache_write_input_tokens_reported === false) + cacheWritesReported = false; + } + // A malformed partition is not evidence of an enforceable lower bound. + if (keys.some((key) => observed[key] !== total[key])) return null; + return estimateScanCost(model, { + ...priced, + cache_write_input_tokens_reported: cacheWritesReported, + modelUsage: parts, + coverage: "partial", + }); +} + +function estimateModelCost( + model: string | undefined, + usage: unknown, ): ScanCost | null { if (model === undefined) return null; const bedrockPricing = BEDROCK_MODEL_PRICING[model]; diff --git a/sdk/typescript/src/cost.ts b/sdk/typescript/src/cost.ts index c5a2079996..7e6e28bdda 100644 --- a/sdk/typescript/src/cost.ts +++ b/sdk/typescript/src/cost.ts @@ -1,8 +1,10 @@ -import { open, readdir } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { open, readdir, realpath } from "node:fs/promises"; import { join } from "node:path"; import { isRecord } from "./record.js"; import { estimateScanCost, + estimateScanCostLowerBound, tokenUsage, type ScanCost, type ScanTokenUsage, @@ -12,9 +14,13 @@ import { type ScanActivity, } from "./scan-activity.js"; import { + attributedScanThreads, + isAttributedScanEvent, isScanArtifactDirectory, + recordedScanCodexHome, sessionParentThreadId, sessionStartedAt, + type ScanExecutionAttribution, } from "./scan-sessions.js"; import { scanProgressUpdatesFromText, @@ -49,15 +55,27 @@ interface SessionUsage { inheritedUsage: ScanTokenUsage | null; replaying: boolean; usage: ScanTokenUsage | null; + counterUsage: ScanTokenUsage | null; + model: string | null; + modelUsage: Map; + counterModelUsage: Map; + currentTurnId: string | null; + previousUsage: ScanTokenUsage | null; + responseIds: Set; + responseUsageObserved: boolean; + responseTokens: number; + expectedResponseTokens: number; + counterRegressed: boolean; calls: Map; - activities: ScanActivity[]; + activities: { index: number; activity: ScanActivity }[]; progress: ScanProgress[]; filesCompleted: number; filesTotal: number | null; prose: Set; reasoning: SessionReasoning | null; reasoningCount: number; - events?: Record[]; + eventIndex: number; + events?: { index: number; event: Record }[]; } interface ScanCostTrackerOptions { @@ -68,6 +86,8 @@ interface ScanCostTrackerOptions { maxCostUsd?: number; expectedFilesTotal?: number; onCost?: (cost: Readonly) => void; + // Only reported when the full public estimate is unavailable. + onCostLowerBound?: (cost: Readonly) => void; onActivity?: (activity: ScanActivity) => void; onProgress?: (progress: ScanProgress) => void; onSessionEvent?: (event: ScanSessionEvent) => void; @@ -95,6 +115,17 @@ function createSessionUsage(): SessionUsage { inheritedUsage: null, replaying: false, usage: null, + counterUsage: null, + model: null, + modelUsage: new Map(), + counterModelUsage: new Map(), + currentTurnId: null, + previousUsage: null, + responseIds: new Set(), + responseUsageObserved: false, + responseTokens: 0, + expectedResponseTokens: 0, + counterRegressed: false, calls: new Map(), activities: [], progress: [], @@ -103,6 +134,7 @@ function createSessionUsage(): SessionUsage { prose: new Set(), reasoning: null, reasoningCount: 0, + eventIndex: 0, }; } @@ -113,13 +145,19 @@ export class ScanCostTracker { readonly #workers = new Map(); readonly #workerProgress = new Map(); readonly #reportedProgress = new Set(); + readonly #reportedSessionEvents = new Map>(); + readonly #reportedActivities = new Map>(); #threadId: string | null = null; #timer: NodeJS.Timeout | null = null; #pending: Promise = Promise.resolve(); #snapshot: ScanCostSnapshot = { usage: null, cost: null }; #lastCost: string | null = null; + #lastCostLowerBound: string | null = null; #highestFilesCompleted = 0; #expectedFilesTotal: number | undefined; + #attribution: ScanExecutionAttribution | null = null; + #readAttribution: + (() => Promise) | undefined; public constructor(options: ScanCostTrackerOptions) { this.#options = options; @@ -130,10 +168,23 @@ export class ScanCostTracker { this.#expectedFilesTotal = filesTotal; } + public setAttributionReader( + reader: () => Promise, + ): void { + this.#readAttribution = reader; + } + public recordUsage(usage: unknown, threadId = this.#threadId): void { const normalized = tokenUsage(usage); if (threadId !== null) { - this.#receipts.set(threadId, normalized); + const previous = this.#receipts.get(threadId); + if ( + previous == null || + (normalized !== null && + normalized.total_tokens >= previous.total_tokens) + ) { + this.#receipts.set(threadId, normalized); + } } } @@ -143,6 +194,7 @@ export class ScanCostTracker { if ( this.#options.maxCostUsd === undefined && this.#options.onCost === undefined && + this.#options.onCostLowerBound === undefined && this.#options.onActivity === undefined && this.#options.onProgress === undefined && this.#options.onSessionEvent === undefined @@ -188,35 +240,81 @@ export class ScanCostTracker { this.#timer = null; if (fallbackUsage !== undefined) this.recordUsage(fallbackUsage); await this.refresh(); - if (this.#receipts.size > 0 || this.#snapshot.usage !== null) + if ( + this.#attribution !== null || + this.#receipts.size > 0 || + this.#snapshot.usage !== null + ) return this.#snapshot; const cost = estimateScanCost(this.#options.model, fallbackUsage); this.#snapshot = { usage: fallbackUsage ?? null, cost }; - this.#reportCost(cost); + this.#reportCost(cost, fallbackUsage); return this.#snapshot; } async #readSessions(): Promise { if (this.#threadId === null) return; - const unreadable: Array<{ session: SessionUsage; error: unknown }> = []; - for await (const path of sessionFiles( - join(this.#options.codexHome, "sessions"), - )) { - let session = this.#sessions.get(path); - if (session === undefined) { - session = createSessionUsage(); - this.#sessions.set(path, session); + let recordedAttribution = this.#attribution; + if (this.#readAttribution) { + const record = await this.#readAttribution(); + recordedAttribution = record ?? null; + const attribution = record == null || record.legacy ? null : record; + if ( + attribution && + (!this.#attribution || + attribution.workerCodexHome !== this.#attribution.workerCodexHome || + attribution.completedAt !== this.#attribution.completedAt) + ) { + this.#sessions.clear(); } + this.#attribution = attribution; + } + const unreadable: Array<{ session: SessionUsage; error: unknown }> = []; + const homes = new Set([this.#options.codexHome]); + if (this.#options.scanDirectory !== undefined) { + const home = await recordedScanCodexHome( + this.#options.scanDirectory, + recordedAttribution, + ); + if (home !== undefined) homes.add(home); + } + // Recovery restores workers to their recorded home; the SDK parent can + // continue in the current home. Apply the same scan membership to both. + const directories = new Set(); + for (const home of homes) { + let directory: string; try { - await readSessionUsage(path, session, this.#options.repository); + directory = await realpath(join(home, "sessions")); } catch (error) { - if (session.threadId === null) throw error; - unreadable.push({ session, error }); + if (isMissingFile(error)) continue; + throw error; + } + if (directories.has(directory)) continue; + directories.add(directory); + for await (const path of sessionFiles(directory)) { + let session = this.#sessions.get(path); + if (session === undefined) { + session = createSessionUsage(); + this.#sessions.set(path, session); + } + try { + await readSessionUsage( + path, + session, + this.#options.repository, + this.#attribution, + ); + } catch (error) { + if (session.threadId === null) throw error; + unreadable.push({ session, error }); + } } } - const included = new Set([this.#threadId, ...this.#receipts.keys()]); - if (this.#options.scanDirectory !== undefined) { + const included = this.#attribution + ? attributedScanThreads(this.#sessions.values(), this.#attribution) + : new Set([this.#threadId, ...this.#receipts.keys()]); + if (!this.#attribution && this.#options.scanDirectory !== undefined) { const scanStartedAt = [...this.#sessions.values()].find( (session) => session.threadId === this.#threadId, @@ -242,9 +340,9 @@ export class ScanCostTracker { } } } - let previousSize: number; - do { - previousSize = included.size; + let changed = this.#attribution === null; + while (changed) { + const previousSize = included.size; for (const session of this.#sessions.values()) { if ( session.threadId !== null && @@ -254,12 +352,28 @@ export class ScanCostTracker { included.add(session.threadId); } } - } while (included.size !== previousSize); + changed = included.size !== previousSize; + } for (const { session, error } of unreadable) { if (included.has(session.threadId!)) throw error; } - const usages = new Map(this.#receipts); + let incomplete = false; + const usages = new Map( + [...this.#receipts].filter( + ([threadId]) => + included.has(threadId) && + (!this.#attribution || + this.#attribution.executionThreadIds.includes(threadId)), + ), + ); + if (this.#attribution) { + for (const threadId of included) { + if (!usages.has(threadId)) usages.set(threadId, null); + } + } + const usageSessions = new Map(); + const counterSessions = new Map(); for (const [path, tracked] of this.#sessions) { const threadId = tracked.threadId; if (threadId === null || !included.has(threadId)) continue; @@ -271,7 +385,12 @@ export class ScanCostTracker { // Replay only newly associated sessions, including their early events. session = createSessionUsage(); session.events = []; - await readSessionUsage(path, session, this.#options.repository); + await readSessionUsage( + path, + session, + this.#options.repository, + this.#attribution, + ); this.#sessions.set(path, session); } let worker: number | undefined; @@ -279,7 +398,20 @@ export class ScanCostTracker { worker = this.#workers.get(threadId) ?? this.#workers.size + 1; this.#workers.set(threadId, worker); } - for (const event of session.events?.splice(0) ?? []) { + for (const { index, event } of session.events?.splice(0) ?? []) { + let reported = this.#reportedSessionEvents.get(threadId); + if (reported === undefined) { + reported = new Set(); + this.#reportedSessionEvents.set(threadId, reported); + } + // A physical copy keeps each event's position, including repeated + // identical events. Positions count unfiltered records so attribution + // changes can replay the same log without changing occurrence identity. + const identity = `${index}:${createHash("sha256") + .update(JSON.stringify(event)) + .digest("hex")}`; + if (reported.has(identity)) continue; + reported.add(identity); this.#options.onSessionEvent?.({ threadId, parentThreadId: session.parentThreadId, @@ -288,7 +420,17 @@ export class ScanCostTracker { }); } if (worker !== undefined) { - for (const activity of session.activities.splice(0)) { + for (const { index, activity } of session.activities.splice(0)) { + let reported = this.#reportedActivities.get(threadId); + if (reported === undefined) { + reported = new Set(); + this.#reportedActivities.set(threadId, reported); + } + const identity = `${index}:${createHash("sha256") + .update(JSON.stringify(activity)) + .digest("hex")}`; + if (reported.has(identity)) continue; + reported.add(identity); this.#options.onActivity?.({ ...activity, id: `${threadId}:${activity.id}`, @@ -297,6 +439,31 @@ export class ScanCostTracker { } this.#reportWorkerProgress(session); } + // A copied prefix must not supply model usage for a more complete log. + const previous = usageSessions.get(threadId); + if ( + previous === undefined || + (session.usage?.total_tokens ?? -1) > + (previous.usage?.total_tokens ?? -1) || + (session.usage?.total_tokens === previous.usage?.total_tokens && + session.pendingLineBytes === 0 && + previous.pendingLineBytes > 0) + ) { + usageSessions.set(threadId, session); + } + if ( + session.counterUsage && + session.counterUsage.total_tokens > + (usages.get(threadId)?.total_tokens ?? -1) + ) { + usages.set(threadId, session.counterUsage); + } + if ( + session.counterUsage && + session.counterUsage.total_tokens > + (counterSessions.get(threadId)?.counterUsage?.total_tokens ?? -1) + ) + counterSessions.set(threadId, session); const receipt = usages.get(threadId); if ( session.usage !== null && @@ -307,19 +474,118 @@ export class ScanCostTracker { ) { usages.set(threadId, session.usage); } + if (!usages.has(threadId)) usages.set(threadId, null); + } + for (const [threadId, session] of usageSessions) { + let selected = usages.get(threadId); + const counter = counterSessions.get(threadId)?.counterUsage; + if (selected && counter) + selected = addTokenUsage( + selected, + tokenUsageRemainder(counter, selected), + ); + if (selected && session.usage) { + const remainder = tokenUsageRemainder(selected, session.usage); + if (remainder.total_tokens > 0) { + usages.set(threadId, addTokenUsage(session.usage, remainder)); + if (session.responseUsageObserved) incomplete = true; + } + } + if ( + (session.counterRegressed && !session.responseUsageObserved) || + session.expectedResponseTokens > session.responseTokens + ) + incomplete = true; + if (session.pendingLineBytes > 0 && !this.#receipts.get(threadId)) + incomplete = true; } let usage: ScanTokenUsage | null = null; for (const value of usages.values()) { if (value === null) { + if (this.#attribution) { + incomplete = true; + continue; + } this.#snapshot = { usage: null, cost: null }; return; } usage = addTokenUsage(usage, value); } - if (usage === null) return; - const cost = estimateScanCost(this.#options.model, usage); - this.#snapshot = { usage, cost }; - this.#reportCost(cost); + if (usage === null) { + this.#snapshot = { usage: null, cost: null }; + return; + } + const modelUsage = new Map(); + const liveModelUsage = new Map(); + let observedModel = false; + for (const [threadId, value] of usages) { + if (value === null) continue; + const session = usageSessions.get(threadId); + for (const [model, tokens] of session?.modelUsage ?? []) { + if (model !== null) observedModel = true; + modelUsage.set( + model, + addTokenUsage(modelUsage.get(model) ?? null, tokens), + ); + } + const remainder = session?.usage + ? subtractTokenUsage(value, session.usage) + : value; + if (remainder !== null && remainder.total_tokens > 0) { + const model = + this.#attribution || (session?.modelUsage.size ?? 0) > 0 + ? null + : (session?.model ?? + (threadId === this.#threadId ? this.#options.model : null)); + modelUsage.set( + model, + addTokenUsage(modelUsage.get(model) ?? null, remainder), + ); + } + const counter = counterSessions.get(threadId); + const live = + (counter?.counterUsage?.total_tokens ?? -1) > + (session?.usage?.total_tokens ?? -1); + const liveTokens = live ? counter?.counterUsage : session?.usage; + const liveModels = live + ? counter?.counterModelUsage + : session?.modelUsage; + for (const [model, tokens] of liveModels ?? []) + liveModelUsage.set( + model, + addTokenUsage(liveModelUsage.get(model) ?? null, tokens), + ); + const liveRemainder = liveTokens + ? subtractTokenUsage(value, liveTokens) + : value; + if (liveRemainder !== null && liveRemainder.total_tokens > 0) + liveModelUsage.set( + null, + addTokenUsage(liveModelUsage.get(null) ?? null, liveRemainder), + ); + } + const reconciled = + observedModel || this.#attribution !== null + ? { + ...usage, + modelUsage: [...modelUsage].map(([model, tokens]) => ({ + model, + ...tokens, + })), + } + : usage; + const measured = incomplete + ? { ...reconciled, coverage: "partial" } + : reconciled; + const cost = estimateScanCost(this.#options.model, measured); + this.#snapshot = { usage: measured, cost }; + this.#reportCost(cost, measured, { + ...usage, + modelUsage: [...liveModelUsage].map(([model, tokens]) => ({ + model, + ...tokens, + })), + }); } #reportWorkerProgress(session: SessionUsage): void { @@ -342,9 +608,10 @@ export class ScanCostTracker { this.#workerProgress.set(session.threadId, progress.filesCompleted); const filesCompleted = Math.min( expectedFilesTotal ?? Number.MAX_SAFE_INTEGER, - this.#workerProgress - .values() - .reduce((total, reviewed) => total + reviewed, 0), + [...this.#workerProgress.values()].reduce( + (total, reviewed) => total + reviewed, + 0, + ), ); if (filesCompleted < this.#highestFilesCompleted) continue; const update = { @@ -361,8 +628,32 @@ export class ScanCostTracker { } } - #reportCost(cost: ScanCost | null): void { - if (cost === null) return; + #reportCost( + cost: ScanCost | null, + usage: unknown, + liveUsage?: unknown, + ): void { + if (cost === null) { + if (this.#options.onCostLowerBound === undefined) return; + const receiptCost = estimateScanCostLowerBound( + this.#options.model, + usage, + ); + const liveCost = estimateScanCostLowerBound( + this.#options.model, + liveUsage, + ); + const lowerBound = + liveCost && liveCost.estimatedUsd > (receiptCost?.estimatedUsd ?? -1) + ? liveCost + : receiptCost; + if (lowerBound === null) return; + const signature = JSON.stringify(lowerBound); + if (signature === this.#lastCostLowerBound) return; + this.#lastCostLowerBound = signature; + this.#options.onCostLowerBound(lowerBound); + return; + } const signature = JSON.stringify(cost); if (signature === this.#lastCost) return; this.#lastCost = signature; @@ -392,6 +683,7 @@ async function readSessionUsage( path: string, session: SessionUsage, repository?: string, + attribution: ScanExecutionAttribution | null = null, ): Promise { if (session.unreadable) return; let file; @@ -413,7 +705,12 @@ async function readSessionUsage( if (bytesRead === 0) return; session.offset += bytesRead; try { - readSessionChunk(buffer.subarray(0, bytesRead), session, repository); + readSessionChunk( + buffer.subarray(0, bytesRead), + session, + repository, + attribution, + ); } catch (error) { session.unreadable = true; session.pendingLine = []; @@ -430,6 +727,7 @@ function readSessionChunk( contents: Buffer, session: SessionUsage, repository?: string, + attribution: ScanExecutionAttribution | null = null, ): void { let lineStart = 0; while (lineStart < contents.length) { @@ -447,13 +745,19 @@ function readSessionChunk( } if (session.pendingLineBytes === 0) { - readSessionEvent(fragment.toString("utf8"), session, repository); + readSessionEvent( + fragment.toString("utf8"), + session, + repository, + attribution, + ); } else { if (fragment.length > 0) session.pendingLine.push(Buffer.from(fragment)); readSessionEvent( Buffer.concat(session.pendingLine, lineBytes).toString("utf8"), session, repository, + attribution, ); session.pendingLine = []; session.pendingLineBytes = 0; @@ -466,6 +770,7 @@ function readSessionEvent( line: string, session: SessionUsage, repository?: string, + attribution: ScanExecutionAttribution | null = null, ): void { if (line.length === 0) return; let event: unknown; @@ -476,10 +781,11 @@ function readSessionEvent( } if (!isRecord(event) || !isRecord(event["payload"])) return; const payload = event["payload"]; + const index = session.eventIndex++; if (event["type"] === "session_meta") { if (session.threadId !== null) { session.replaying = payload["id"] !== session.threadId; - if (!session.replaying) session.events?.push(event); + if (!session.replaying) session.events?.push({ index, event }); return; } if (typeof payload["id"] === "string") { @@ -488,9 +794,10 @@ function readSessionEvent( if (typeof payload["cwd"] === "string") { session.workingDirectory = payload["cwd"]; } + if (typeof payload["model"] === "string") session.model = payload["model"]; session.startedAt = sessionStartedAt(payload["timestamp"]); session.parentThreadId = sessionParentThreadId(payload); - session.events?.push(event); + session.events?.push({ index, event }); return; } if (session.replaying) { @@ -511,12 +818,87 @@ function readSessionEvent( : turnOrder !== null && turnOrder >= threadOrder; if (owned) { session.replaying = false; - session.events?.push(event); + session.events?.push({ index, event }); } } return; } - session.events?.push(event); + if ( + (event["type"] === "turn_context" || payload["type"] === "task_started") && + typeof payload["turn_id"] === "string" + ) { + session.currentTurnId = payload["turn_id"]; + } + if ( + event["type"] === "turn_context" && + typeof payload["model"] === "string" + ) { + session.model = payload["model"]; + } + if (event["type"] === "token_usage_record") { + const responseId = payload["response_id"]; + const usage = tokenUsage(payload["usage"]); + if ( + typeof responseId !== "string" || + usage === null || + (typeof payload["thread_id"] === "string" && + payload["thread_id"] !== session.threadId) || + session.responseIds.has(responseId) + ) + return; + session.responseIds.add(responseId); + const cumulative = tokenUsage(payload["thread_token_usage"]); + if (cumulative) + session.expectedResponseTokens = Math.max( + session.expectedResponseTokens, + cumulative.total_tokens, + ); + session.responseTokens += usage.total_tokens; + const turnId = + typeof payload["turn_id"] === "string" + ? payload["turn_id"] + : session.currentTurnId; + if ( + attribution && + !isAttributedScanEvent( + attribution, + session.threadId!, + turnId, + event["timestamp"], + ) + ) + return; + if (!session.responseUsageObserved) { + // Exact receipts include compaction and survive counter resets. Keep the + // legacy counter as an independent lower bound, never add it to receipts. + session.responseUsageObserved = true; + session.usage = null; + session.modelUsage.clear(); + } + const model = + typeof payload["model"] === "string" ? payload["model"] : session.model; + session.usage = addTokenUsage(session.usage, usage); + session.modelUsage.set( + model, + addTokenUsage(session.modelUsage.get(model) ?? null, usage), + ); + session.events?.push({ index, event }); + return; + } + const attributable = + attribution === null || + isAttributedScanEvent( + attribution, + session.threadId!, + session.currentTurnId, + event["timestamp"], + ); + if (attributable) session.events?.push({ index, event }); + if ( + !attributable && + !(event["type"] === "event_msg" && payload["type"] === "token_count") + ) + return; if (event["type"] === "response_item") { session.progress.push(...sessionProgressUpdates(payload)); if (repository === undefined) return; @@ -587,7 +969,7 @@ function readSessionEvent( if (activity.status === "running") { session.calls.set(activity.id, activity); } - session.activities.push(activity); + session.activities.push({ index: session.eventIndex - 1, activity }); return; } if ( @@ -598,8 +980,11 @@ function readSessionEvent( const call = session.calls.get(payload["call_id"]); if (call !== undefined) { session.activities.push({ - ...call, - status: payload["status"] === "failed" ? "failed" : "completed", + index: session.eventIndex - 1, + activity: { + ...call, + status: payload["status"] === "failed" ? "failed" : "completed", + }, }); session.calls.delete(call.id); } @@ -632,7 +1017,7 @@ function readSessionEvent( !session.prose.has(`${activity.kind}:${activity.description}`) ) { session.prose.add(`${activity.kind}:${activity.description}`); - session.activities.push(activity); + session.activities.push({ index: session.eventIndex - 1, activity }); } return; } @@ -649,7 +1034,51 @@ function readSessionEvent( session.inheritedUsage === null ? usage : subtractTokenUsage(usage, session.inheritedUsage); - if (ownUsage !== null) session.usage = ownUsage; + if (ownUsage !== null) { + const hadPreviousUsage = session.previousUsage !== null; + const delta = + session.previousUsage === null + ? ownUsage + : subtractTokenUsage(ownUsage, session.previousUsage); + if ( + session.previousUsage !== null && + ownUsage.total_tokens < session.previousUsage.total_tokens + ) { + session.counterRegressed = true; + return; + } + session.previousUsage = ownUsage; + if ( + attribution && + !isAttributedScanEvent( + attribution, + session.threadId!, + session.currentTurnId, + event["timestamp"], + ) + ) + return; + if (delta !== null) { + const model = + !hadPreviousUsage && session.responseUsageObserved + ? null + : session.model; + session.counterModelUsage.set( + model, + addTokenUsage(session.counterModelUsage.get(model) ?? null, delta), + ); + if (!session.responseUsageObserved) + session.modelUsage.set( + session.model, + addTokenUsage(session.modelUsage.get(session.model) ?? null, delta), + ); + } + session.counterUsage = + attribution && delta !== null + ? addTokenUsage(session.counterUsage, delta) + : ownUsage; + if (!session.responseUsageObserved) session.usage = session.counterUsage; + } } function uuid7Order(value: unknown): bigint | null { @@ -729,7 +1158,7 @@ function recordReasoningActivity( } reasoning.activity = activity; session.prose.add(`${activity.kind}:${activity.description}`); - session.activities.push(activity); + session.activities.push({ index: session.eventIndex - 1, activity }); } function sessionProgressUpdates( @@ -821,3 +1250,40 @@ function subtractTokenUsage( function isMissingFile(error: unknown): boolean { return isRecord(error) && error["code"] === "ENOENT"; } + +function tokenUsageRemainder( + usage: ScanTokenUsage, + inherited: ScanTokenUsage, +): ScanTokenUsage { + const input = Math.max(0, usage.input_tokens - inherited.input_tokens); + const cached = Math.min( + input, + Math.max(0, usage.cached_input_tokens - inherited.cached_input_tokens), + ); + const writes = Math.min( + input - cached, + Math.max( + 0, + usage.cache_write_input_tokens - inherited.cache_write_input_tokens, + ), + ); + const output = Math.max(0, usage.output_tokens - inherited.output_tokens); + return { + input_tokens: input, + cached_input_tokens: cached, + cache_write_input_tokens: writes, + output_tokens: output, + reasoning_output_tokens: Math.min( + output, + Math.max( + 0, + usage.reasoning_output_tokens - inherited.reasoning_output_tokens, + ), + ), + total_tokens: input + output, + ...(usage.cache_write_input_tokens_reported === false || + inherited.cache_write_input_tokens_reported === false + ? { cache_write_input_tokens_reported: false } + : {}), + }; +} diff --git a/sdk/typescript/src/custom-validation-prompt.ts b/sdk/typescript/src/custom-validation-prompt.ts index 49d6f96100..05016fb39e 100644 --- a/sdk/typescript/src/custom-validation-prompt.ts +++ b/sdk/typescript/src/custom-validation-prompt.ts @@ -9,7 +9,7 @@ import { PLUGIN_NAME } from "./runtime.js"; // the ordinary validation sequence with a custom-validation request. const SOURCES = { "references/core-scan.md": - "9c0eca53147383a89cd345eac7791ea75b954f7f083448ef25019d8f9ba30afe", + "6e88093ed18790a03c24f33b4efe52214426cf399eeef47debbe5a17960a9dff", "skills/security-scan/SKILL.md": "104e2f93fc965c34e91970f517a89e330d5b29dbb243dcea5d95d83f14d135cf", "skills/security-diff-scan/SKILL.md": diff --git a/sdk/typescript/src/deep-scan-finalization.ts b/sdk/typescript/src/deep-scan-finalization.ts new file mode 100644 index 0000000000..e18bf9f583 --- /dev/null +++ b/sdk/typescript/src/deep-scan-finalization.ts @@ -0,0 +1,26 @@ +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +type WorkbenchRunner = (args: string[]) => Promise>; + +/** Load the same publisher used by native Deep from the installed plugin. */ +export async function resumeSelectedDeepScan(input: { + scanId: string; + threadId: string; + pluginRoot: string; + runWorkbench: WorkbenchRunner; + signal: AbortSignal; +}): Promise { + const helper = ( + await import(pathToFileURL(join(input.pluginRoot, "mcp/helpers.mjs")).href) + ).default as { + resumeSelectedDeepScan: (input: { + scanId: string; + threadId: string; + pluginRoot: string; + runWorkbench: WorkbenchRunner; + signal: AbortSignal; + }) => Promise; + }; + await helper.resumeSelectedDeepScan(input); +} diff --git a/sdk/typescript/src/preflight-config.ts b/sdk/typescript/src/preflight-config.ts new file mode 100644 index 0000000000..c3d21c4b61 --- /dev/null +++ b/sdk/typescript/src/preflight-config.ts @@ -0,0 +1,156 @@ +import { isSafeNonNegativeInteger as safeInteger } from "./value.js"; +import { isRecord } from "./record.js"; +import { isAbsolute } from "node:path"; +import { + EXTERNAL_CODEX_PROVIDERS, + isExternalModelProvider, + resolveCodexProfile, + scanModelProvider, + type JsonObject, +} from "./config.js"; + +export function scanPreflightCodexConfig(config: JsonObject): JsonObject { + const safeString = (value: unknown): value is string => + typeof value === "string" && + value.length > 0 && + !/[\u0000-\u001f\u007f]/u.test(value); + const safeProfileName = (value: unknown): value is string => + safeString(value) && /^[A-Za-z0-9_-]+$/u.test(value); + const capabilityFeatures = (value: unknown): JsonObject => { + if (!isRecord(value)) return {}; + const result: JsonObject = {}; + for (const key of [ + "goals", + "multi_agent", + "enable_fanout", + "api_key_cyber_access_programs", + "api_key_model_discovery", + ]) { + if (typeof value[key] === "boolean") result[key] = value[key]; + } + const multiAgent = value["multi_agent_v2"]; + if (typeof multiAgent === "boolean") { + result["multi_agent_v2"] = multiAgent; + } else if (isRecord(multiAgent)) { + const sanitized: JsonObject = {}; + if (typeof multiAgent["enabled"] === "boolean") { + sanitized["enabled"] = multiAgent["enabled"]; + } + const capacity = multiAgent["max_concurrent_threads_per_session"]; + if (safeInteger(capacity)) { + sanitized["max_concurrent_threads_per_session"] = capacity; + } + if (Object.keys(sanitized).length > 0) { + result["multi_agent_v2"] = sanitized; + } + } + return result; + }; + const executionConfig = (source: JsonObject): JsonObject => { + const result: JsonObject = {}; + for (const key of [ + "model", + "model_reasoning_effort", + "model_reasoning_summary", + "model_provider", + "service_tier", + ]) { + const value = source[key]; + if (safeString(value)) result[key] = value; + } + const features = capabilityFeatures(source["features"]); + if (Object.keys(features).length > 0) result["features"] = features; + const agents = source["agents"]; + if (isRecord(agents)) { + const sanitized: JsonObject = {}; + for (const key of ["max_threads", "max_depth"]) { + const value = agents[key]; + if (safeInteger(value)) sanitized[key] = value; + } + if (Object.keys(sanitized).length > 0) result["agents"] = sanitized; + } + const multiagent = source["multiagent_config"]; + if (isRecord(multiagent) && safeInteger(multiagent["max_concurrency"])) { + result["multiagent_config"] = { + max_concurrency: multiagent["max_concurrency"], + }; + } + return result; + }; + const result = executionConfig(config); + // Keep effective worker settings even when preflight filters the profile name. + const resolved = resolveCodexProfile(config); + for (const key of ["model_reasoning_summary", "service_tier"]) { + const value = resolved[key]; + if (safeString(value)) result[key] = value; + } + const resolvedFeatures = capabilityFeatures(resolved["features"]); + for (const key of [ + "api_key_cyber_access_programs", + "api_key_model_discovery", + ]) { + if (resolvedFeatures[key] !== undefined) { + result["features"] = { + ...(isRecord(result["features"]) ? result["features"] : {}), + [key]: resolvedFeatures[key], + }; + } + } + const selectedProfile = safeProfileName(config["profile"]) + ? config["profile"] + : undefined; + if (selectedProfile !== undefined) { + result["profile"] = selectedProfile; + } + const profiles = config["profiles"]; + if (isRecord(profiles)) { + const sanitized: JsonObject = {}; + for (const [name, profile] of Object.entries(profiles)) { + if (!safeProfileName(name) || !isRecord(profile)) continue; + const projected = executionConfig(profile as JsonObject); + if (Object.keys(projected).length === 0) continue; + sanitized[name] = projected; + } + if (Object.keys(sanitized).length > 0) result["profiles"] = sanitized; + } + const modelProvider = scanModelProvider(result); + if (isExternalModelProvider(modelProvider)) { + result["model_providers"] = { + [modelProvider]: { ...EXTERNAL_CODEX_PROVIDERS[modelProvider] }, + }; + } else if (modelProvider === "amazon-bedrock") { + const providers = config["model_providers"]; + const provider = isRecord(providers) ? providers[modelProvider] : undefined; + const aws = isRecord(provider) ? provider["aws"] : undefined; + if (isRecord(aws)) { + const sanitized: JsonObject = {}; + for (const key of ["region", "profile"]) { + const value = aws[key]; + if (safeString(value)) sanitized[key] = value; + } + if (Object.keys(sanitized).length > 0) { + result["model_providers"] = { + [modelProvider]: { aws: sanitized }, + }; + } + } + } + const rootMarkers = config["project_root_markers"]; + if (Array.isArray(rootMarkers)) { + result["project_root_markers"] = rootMarkers.filter(safeString); + } + const projects = config["projects"]; + if (isRecord(projects)) { + const sanitized: JsonObject = {}; + for (const [path, project] of Object.entries(projects)) { + if (!safeString(path) || !isAbsolute(path) || !isRecord(project)) { + continue; + } + const trust = project["trust_level"]; + if (trust !== "trusted" && trust !== "untrusted") continue; + sanitized[path] = { trust_level: trust }; + } + if (Object.keys(sanitized).length > 0) result["projects"] = sanitized; + } + return result; +} diff --git a/sdk/typescript/src/reasoning-summary.ts b/sdk/typescript/src/reasoning-summary.ts new file mode 100644 index 0000000000..a313bc00c4 --- /dev/null +++ b/sdk/typescript/src/reasoning-summary.ts @@ -0,0 +1,69 @@ +import { + inlineToml, + resolveCodexProfile, + scanModelConfiguration, + type JsonObject, +} from "./config.js"; +import { scanPreflightCodexConfig } from "./preflight-config.js"; +import { + runCodexCommand, + type CodexCommand, + type ProcessEnvironment, +} from "./runtime.js"; + +/** Capture a future owner's selected model default; never infer historical settings. */ +export async function captureOriginalReasoningSummary(options: { + config: JsonObject; + command: CodexCommand; + cwd: string; + environment: ProcessEnvironment; + signal: AbortSignal; +}): Promise { + // Keep explicit selections and invalid values with their existing native validator. + if ( + scanPreflightCodexConfig(options.config)["model_reasoning_summary"] !== + undefined || + resolveCodexProfile(options.config)["model_reasoning_summary"] !== undefined + ) + return undefined; + const { model } = scanModelConfiguration(options.config); + // The same per-session overrides protect the lookup from concurrent home edits. + const config = JSON.parse(JSON.stringify(options.config)) as JsonObject; + const args = [ + "debug", + "models", + ...Object.entries(config).flatMap(([key, value]) => [ + "--config", + `${key}=${inlineToml(value)}`, + ]), + ]; + const result = await runCodexCommand( + options.command, + args, + options.environment, + undefined, + options.signal, + options.cwd, + ); + // Older native executables and models absent from their catalog provide no + // recoverable value. Preserve omission instead of choosing another default. + if (!result.success) return undefined; + let catalog: unknown; + try { + catalog = JSON.parse(result.stdout); + } catch { + return undefined; + } + if (!isRecord(catalog) || !Array.isArray(catalog["models"])) return undefined; + const selected = catalog["models"].find( + (entry: unknown) => isRecord(entry) && entry["slug"] === model, + ); + return isRecord(selected) && + typeof selected["default_reasoning_summary"] === "string" + ? selected["default_reasoning_summary"] + : undefined; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/sdk/typescript/src/runtime.ts b/sdk/typescript/src/runtime.ts index 873cc88373..ee2d419599 100644 --- a/sdk/typescript/src/runtime.ts +++ b/sdk/typescript/src/runtime.ts @@ -2862,12 +2862,14 @@ export async function runCodexCommand( environment: ProcessEnvironment, input?: string | Uint8Array, signal?: AbortSignal, + cwd?: string, ): Promise { const child = spawn(executablePathForSpawn(command.command), [...args], { env: environment, stdio: ["pipe", "pipe", "pipe"], windowsHide: true, signal, + ...(cwd === undefined ? {} : { cwd }), }); let stdout = ""; let stderr = ""; diff --git a/sdk/typescript/src/scan-logs.ts b/sdk/typescript/src/scan-logs.ts index efa6553b19..1a77d4edbb 100644 --- a/sdk/typescript/src/scan-logs.ts +++ b/sdk/typescript/src/scan-logs.ts @@ -7,9 +7,13 @@ import { sessionFiles } from "./cost.js"; import { CodexSecurityError } from "./errors.js"; import type { JsonObject } from "./config.js"; import { + attributedScanThreads, + isAttributedScanEvent, isScanArtifactDirectory, + recordedScanCodexHome, sessionParentThreadId, sessionStartedAt, + type ScanExecutionAttribution, } from "./scan-sessions.js"; interface ScanLogOptions { @@ -21,6 +25,7 @@ interface ScanLogOptions { scanDirectory?: string; completedAt?: string | null; allowMissingRoot?: boolean; + executionAttribution?: ScanExecutionAttribution | null; } export type ScanLogSource = JsonObject & { @@ -28,6 +33,7 @@ export type ScanLogSource = JsonObject & { continuationThreadId?: string; threadIds?: string[]; executionThreadIds?: string[]; + executionAttribution?: ScanExecutionAttribution | null; mode?: string; scanDir?: string; progress?: { status?: string; updatedAt?: string }; @@ -49,6 +55,7 @@ export function readSavedScanLogs( threadId: threadId ?? scan.threadIds?.[0], threadIds: scan.threadIds, executionThreadIds: scan.executionThreadIds ?? [], + executionAttribution: scan.executionAttribution, codexHome, allowMissingRoot: options.allowMissingRoot, scanDirectory: scan.mode === "deep" ? scan.scanDir : undefined, @@ -113,6 +120,14 @@ export async function readScanLogs(options: ScanLogOptions) { ? [options.codexHome] : options.codexHome, ); + if (options.scanDirectory !== undefined) { + const home = await recordedScanCodexHome( + options.scanDirectory, + options.executionAttribution, + ); + if (home !== undefined) homes.add(home); + } + // Recovered workers retain their original home; the parent can use the current home. for (const directory of ["sessions", "archived_sessions"]) { for (const home of homes) { for await (const session of scanSessions(home, directory)) { @@ -130,15 +145,24 @@ export async function readScanLogs(options: ScanLogOptions) { ); } - const included = new Set([ - ...(options.threadId ? [options.threadId] : []), - ...(options.threadIds ?? []), - ...(options.executionThreadIds ?? []), - ]); + const attribution = options.executionAttribution?.legacy + ? null + : options.executionAttribution; + const included = attribution + ? attributedScanThreads( + Array.from(logs.values(), ([session]) => session), + attribution, + ) + : new Set([ + ...(options.threadId ? [options.threadId] : []), + ...(options.threadIds ?? []), + ...(options.executionThreadIds ?? []), + ]); // A Desktop owner can contain other work. Include its log without treating // the whole conversation tree as part of this scan. const traversed = new Set(options.executionThreadIds ?? included); - for (const parentId of traversed) { + const pending = attribution ? [] : traversed; + for (const parentId of pending) { const parent = logs.get(parentId)?.[0]; for (const [session] of logs.values()) { if ( @@ -160,31 +184,20 @@ export async function readScanLogs(options: ScanLogOptions) { if (copies === undefined) continue; let session = copies[0]; for (const copy of copies.slice(1)) { - if (await extendsSessionLog(session.path, copy.path)) session = copy; + if ( + await extendsSessionLog(session.path, copy.path, session, attribution) + ) + session = copy; } sessions.push(session); } const events: Record[] = []; for (const session of sessions) { - let replaying = false; - for await (const event of sessionEvents(session.path)) { - const payload = event["payload"]; - if (event["type"] === "session_meta" && isRecord(payload)) { - replaying = payload["id"] !== session.threadId; - } - if (replaying) { - if ( - event["type"] !== "event_msg" || - !isRecord(payload) || - payload["type"] !== "task_started" || - typeof payload["started_at"] !== "number" || - session.startedAt === null || - payload["started_at"] < Math.floor(session.startedAt / 1_000) - ) { - continue; - } - replaying = false; - } + for await (const event of attributedSessionEvents( + session.path, + session, + attribution, + )) { events.push({ threadId: session.threadId, event }); } } @@ -201,6 +214,58 @@ export async function readScanLogs(options: ScanLogOptions) { }; } +async function* attributedSessionEvents( + path: string, + session: SessionLog, + attribution: ScanExecutionAttribution | null | undefined, +): AsyncGenerator> { + let replaying = false; + let turnId: string | null = null; + for await (const event of sessionEvents(path)) { + const payload = event["payload"]; + if ( + isRecord(payload) && + (event["type"] === "turn_context" || + payload["type"] === "task_started") && + typeof payload["turn_id"] === "string" + ) { + turnId = payload["turn_id"]; + } + if (event["type"] === "session_meta" && isRecord(payload)) { + replaying = payload["id"] !== session.threadId; + } + if (replaying) { + if ( + event["type"] !== "event_msg" || + !isRecord(payload) || + payload["type"] !== "task_started" || + typeof payload["started_at"] !== "number" || + session.startedAt === null || + payload["started_at"] < Math.floor(session.startedAt / 1_000) + ) { + continue; + } + replaying = false; + } + if ( + !attribution || + event["type"] === "session_meta" || + isAttributedScanEvent( + attribution, + session.threadId, + event["type"] === "token_usage_record" && + isRecord(payload) && + typeof payload["turn_id"] === "string" + ? payload["turn_id"] + : turnId, + event["timestamp"], + ) + ) { + yield event; + } + } +} + function belongsToScan( session: SessionLog, root: SessionLog, @@ -250,10 +315,16 @@ function belongsToScan( async function extendsSessionLog( previousPath: string, path: string, + session: SessionLog, + attribution: ScanExecutionAttribution | null | undefined, ): Promise { - const events = sessionEvents(path); + const selectedEvents = (source: string) => + attribution + ? attributedSessionEvents(source, session, attribution) + : sessionEvents(source); + const events = selectedEvents(path); try { - for await (const previous of sessionEvents(previousPath)) { + for await (const previous of selectedEvents(previousPath)) { const next = await events.next(); if (next.done || !isDeepStrictEqual(previous, next.value)) return false; } diff --git a/sdk/typescript/src/scan-sessions.ts b/sdk/typescript/src/scan-sessions.ts index f34d53270c..e074210533 100644 --- a/sdk/typescript/src/scan-sessions.ts +++ b/sdk/typescript/src/scan-sessions.ts @@ -1,6 +1,78 @@ import { isAbsolute, join, relative, sep } from "node:path"; import { isRecord } from "./record.js"; +export async function recordedScanCodexHome( + _scanDirectory: string, + attribution?: ScanExecutionAttribution | null, +): Promise { + return attribution?.workerCodexHome ?? undefined; +} + +export interface ScanExecutionAttribution { + formatVersion: 1; + legacy?: true; + workerCodexHome?: string | null; + executionThreadIds: string[]; + owner: { + threadId: string | null; + turnId: string | null; + startedAt: string; + dedicated?: boolean; + }; + startedAt: string; + completedAt: string | null; +} + +export function attributedScanThreads( + sessions: Iterable<{ + threadId: string | null; + parentThreadId: string | null; + }>, + attribution: ScanExecutionAttribution, +): Set { + const included = new Set(attribution.executionThreadIds); + const pending = [...included]; + const all = [...sessions]; + for (const parent of pending) { + for (const session of all) { + if ( + session.threadId !== null && + session.parentThreadId === parent && + !included.has(session.threadId) + ) { + included.add(session.threadId); + pending.push(session.threadId); + } + } + } + if (attribution.owner.threadId) included.add(attribution.owner.threadId); + return included; +} + +export function isAttributedScanEvent( + attribution: ScanExecutionAttribution, + threadId: string, + turnId: string | null, + timestamp: unknown, +): boolean { + const time = sessionStartedAt(timestamp); + if ( + time === null || + time < Date.parse(attribution.startedAt) || + (attribution.completedAt !== null && + time > Date.parse(attribution.completedAt)) + ) + return false; + if ( + threadId !== attribution.owner.threadId || + attribution.executionThreadIds.includes(threadId) + ) + return true; + return ( + attribution.owner.turnId !== null && turnId === attribution.owner.turnId + ); +} + export function sessionStartedAt(timestamp: unknown): number | null { const startedAt = typeof timestamp === "string" ? Date.parse(timestamp) : Number.NaN; diff --git a/sdk/typescript/tests-ts/api-audit-admission.test.ts b/sdk/typescript/tests-ts/api-audit-admission.test.ts new file mode 100644 index 0000000000..f547442ad5 --- /dev/null +++ b/sdk/typescript/tests-ts/api-audit-admission.test.ts @@ -0,0 +1,662 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { afterEach, expect, test } from "bun:test"; +import { build } from "esbuild"; +import { runScanEvents } from "../src/api.js"; +import { runWorkbench, type WorkbenchCommandOptions } from "../src/runtime.js"; +import { TestClient } from "./support/api-client.js"; +import { preparedRuntime } from "./support/api-events.js"; +import { runNodePython } from "./support/python-probe.js"; +import type { ScanDraftInput } from "../src/accepted-audit.js"; +import { PLUGIN_ROOT } from "./plugin-root.js"; +import { + completedEvents, + createApiTestFixtures, +} from "./support/api-events.js"; + +const { temporaryDirectory, copyCompletedScan, cleanup } = + createApiTestFixtures(); +afterEach(cleanup); + +const bundle = await build({ + stdin: { + resolveDir: fileURLToPath( + new URL("../../../plugins/codex-security/mcp-app/", import.meta.url), + ), + contents: `export * from "./src/artifact-scan-draft.ts"; + export * from "./src/deep-scan/artifacts.ts"; + export * from "./src/deep-scan/artifact-validation.ts"; + export * from "./src/deep-scan/worker-runner.ts";`, + }, + bundle: true, + format: "esm", + platform: "node", + loader: { ".md": "text" }, + write: false, +}); +const bundlePath = join(await temporaryDirectory(), "deep-admission.mjs"); +await writeFile(bundlePath, bundle.outputFiles[0]!.contents); +const { + createDeepScanArtifacts, + parseCanonicalScanDraft, + recordCodexSecurityScanDraft, + recordCodexSecurityScanDraftViaWorkbench, + readDiscoveryAuditDraft, + DeepScanWorkerRunner, +} = await import(pathToFileURL(bundlePath).href); + +const scanId = "811aef98-3709-4c2d-8b7a-742977521865"; +type Mutation = + | "missing-findings" + | "contradictory-coverage" + | "inverted-lines" + | "wrong-scan" + | "legacy-details"; +const cases: { + name: string; + coverage: "complete" | "partial" | "unknown"; + complete?: boolean; + mutation?: Mutation; + accepted: boolean; +}[] = [ + { + name: "complete coverage", + coverage: "complete", + complete: true, + accepted: true, + }, + { + name: "partial coverage", + coverage: "partial", + complete: true, + accepted: true, + }, + { + name: "unknown coverage", + coverage: "unknown", + complete: true, + accepted: true, + }, + { name: "omitted completion marker", coverage: "partial", accepted: true }, + { + name: "persisted legacy details", + coverage: "partial", + mutation: "legacy-details", + accepted: true, + }, + { + name: "unfinished checkpoint", + coverage: "partial", + complete: false, + accepted: false, + }, + { + name: "missing findings", + coverage: "partial", + mutation: "missing-findings", + accepted: false, + }, + { + name: "complete coverage with deferred work", + coverage: "partial", + mutation: "contradictory-coverage", + accepted: false, + }, + { + name: "inverted finding lines", + coverage: "partial", + mutation: "inverted-lines", + accepted: false, + }, + { + name: "mismatched canonical scan ID", + coverage: "partial", + mutation: "wrong-scan", + accepted: false, + }, +]; + +for (const scenario of cases) { + test(`Standard and Deep production admission: ${scenario.name}`, async () => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const standardRoot = join(root, "standard"); + const deepRoot = join(root, "deep"); + await Promise.all([ + mkdir(repository), + mkdir(standardRoot, { mode: 0o700 }), + mkdir(deepRoot, { mode: 0o700 }), + ]); + const semantic: ScanDraftInput = { + scanId, + ...(scenario.complete === undefined + ? {} + : { complete: scenario.complete }), + scope: { summary: "Archive extraction." }, + threatModel: { summary: "An untrusted caller supplies archive entries." }, + findings: [ + { + ruleId: "path-traversal.archive", + title: "Unsafe archive extraction", + summary: "An archive entry reaches a filesystem write.", + severity: { level: "high" }, + confidence: { level: "high", rationale: "Source review." }, + taxonomy: { category: "path-traversal", cwe: ["CWE-22"] }, + locations: [{ path: "extract.py", startLine: 4, endLine: 7 }], + remediation: "Validate the resolved output path before writing.", + provenance: { source: "local_plugin", candidateId: "archive-entry" }, + }, + ], + coverage: { + completeness: scenario.coverage, + surfaces: [{ label: "Archive extraction", disposition: "reported" }], + explicitExclusions: [], + deferred: + scenario.coverage === "complete" + ? [] + : [ + { + id: "deployment", + reason: "Deployment controls remain unverified.", + }, + ], + }, + }; + await recordCodexSecurityScanDraft( + { + root: standardRoot, + repoRoot: repository, + layout: "scan", + scanId, + mode: "standard", + status: "running", + scope: ".", + targetContract: { + target: { + allowedKinds: ["directory_snapshot"], + targetId: "target_example", + displayName: "example", + requiredSnapshotDigest: `codex-security-snapshot/v1:sha256:${"a".repeat(64)}`, + }, + scope: { requiredIncludePaths: ["."], requiredExcludePaths: [] }, + diffTarget: null, + }, + }, + semantic, + ); + const submitted = mutateDraft(semantic, scenario.mutation); + const findings = { + scanId: submitted.scanId, + findings: submitted.findings?.map((finding) => ({ + ...finding, + findingId: "finding_example", + occurrenceId: "occurrence_example", + fingerprints: { identity: "synthetic" }, + })), + }; + const coverage = JSON.parse( + await readFile(join(standardRoot, "coverage.json"), "utf8"), + ); + Object.assign(coverage, submitted.coverage); + await Promise.all([ + writeFile(join(standardRoot, "findings.json"), JSON.stringify(findings)), + writeFile(join(standardRoot, "coverage.json"), JSON.stringify(coverage)), + ]); + const standard = await observeStandardAdmission( + repository, + standardRoot, + scanId, + ); + + const artifacts = createDeepScanArtifacts(deepRoot); + const acceptedPaths: string[] = []; + let executions = 0; + const runner = new DeepScanWorkerRunner({ + run: { + scanId, + scanDir: deepRoot, + targetPath: repository, + scope: ".", + config: { subagents: 0 }, + }, + artifacts, + pluginRoot: PLUGIN_ROOT, + signal: new AbortController().signal, + retryDelaysMs: [], + random: () => 0, + log: () => {}, + clock: { now: () => Date.now(), sleep: async () => {} }, + executor: { + async run(request: { + artifactContext: { root: string }; + onThreadStarted?: (id: string) => Promise; + }) { + executions++; + await request.onThreadStarted?.("deep-thread"); + await writeFile( + join(request.artifactContext.root, "result.json"), + JSON.stringify(submitted), + ); + return { threadId: "deep-thread" }; + }, + }, + store: { + async updateWorker(update: { + status: string; + resultManifestPath?: string; + }) { + if (update.status === "succeeded") { + acceptedPaths.push(update.resultManifestPath!); + return { ...update, completionSequence: 1 }; + } + return update; + }, + }, + }); + const deepResult = await runner.runDiscoveryWorker( + "worker-1", + "discovery-1", + ); + expect(executions).toBe(1); + if (scenario.accepted) { + expect(standard.error).toBe(standard.finalization); + expect(standard.finalizations).toBe(1); + expect(standard.drafts).toHaveLength(1); + expect(deepResult.status).toBe("succeeded"); + expect(acceptedPaths).toEqual([deepResult.worker.resultPath]); + const deepDraft: ScanDraftInput = await readDiscoveryAuditDraft( + artifacts, + deepResult.worker.resultPath, + scanId, + ); + expect(standard.drafts[0]!.findings).toEqual(deepDraft.findings); + expect(standard.drafts[0]!.coverage).toEqual(deepDraft.coverage); + expect(standard.drafts[0]!.scope).toEqual(deepDraft.scope); + expect(standard.drafts[0]!.threatModel).toEqual(deepDraft.threatModel); + if (scenario.mutation === "legacy-details") { + expect(deepDraft.findings[0]!["validation"]).toEqual({ + limitations: ["Legacy persisted limitation."], + }); + } + } else { + expect(standard.error).toBeInstanceOf(Error); + expect(standard.error).not.toBe(standard.finalization); + expect(standard.finalizations).toBe(0); + expect(deepResult.status).toBe("failed"); + expect(acceptedPaths).toHaveLength(0); + } + const manifest = JSON.parse( + await readFile(join(standardRoot, "scan-manifest.json"), "utf8"), + ); + expect(manifest.scan.sealedAt).toBeUndefined(); + await expect(readFile(join(standardRoot, "report.md"))).rejects.toThrow(); + }); +} + +test("Standard admission preserves existing canonical scan IDs", async () => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + await mkdir(repository); + const scanDir = await copyCompletedScan(root); + const manifest = JSON.parse( + await readFile(join(scanDir, "scan-manifest.json"), "utf8"), + ); + const standard = await observeStandardAdmission(repository, scanDir); + expect(manifest.scan.id).toBe("scan_example_001"); + expect(standard.error).toBe(standard.finalization); + expect(standard.finalizations).toBe(1); + expect(standard.drafts).toHaveLength(1); + expect(standard.drafts[0]!.scanId).toBe(manifest.scan.id); +}); + +test.each(["http", "HTTP"])( + "Standard admission preserves file-authored canonical surface ID %s", + async (surfaceId) => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + await mkdir(repository); + const scanDir = await copyCompletedScan(root); + const coveragePath = join(scanDir, "coverage.json"); + const coverage = JSON.parse(await readFile(coveragePath, "utf8")); + coverage.surfaces[0].id = surfaceId; + await writeFile(coveragePath, JSON.stringify(coverage)); + const standard = await observeStandardAdmission(repository, scanDir); + expect(standard.error).toBe(standard.finalization); + expect(standard.finalizations).toBe(1); + expect( + (standard.drafts[0]!.coverage["surfaces"] as { id: string }[])[0]!.id, + ).toBe(surfaceId); + expect(JSON.parse(await readFile(coveragePath, "utf8"))).toEqual(coverage); + }, +); + +test.each(["current", "legacy"])( + "Standard admission retains selected %s plugin helper compatibility", + async (kind) => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + await mkdir(repository); + const scanDir = await copyCompletedScan(root); + const pluginRoot = join(root, "selected-plugin"); + await mkdir(join(pluginRoot, "mcp"), { recursive: true }); + await writeFile( + join(pluginRoot, "mcp", "helpers.mjs"), + kind === "legacy" + ? "process.exitCode = 2;\n" + : `export { default } from ${JSON.stringify(pathToFileURL(join(PLUGIN_ROOT, "mcp", "helpers.mjs")).href)};\n`, + ); + const finalization = new Error("Finalization receives the admitted scan."); + let finalizations = 0; + const previousExitCode = process.exitCode; + try { + const error = await runScanEvents({ + thread: { id: "standard-thread" }, + events: completedEvents("standard-thread"), + signal: new AbortController().signal, + scanDir, + pluginRoot, + expectation: { + repository, + repositoryRevision: null, + target: { kind: "repository", paths: [] }, + mode: "standard", + pluginVersion: "0.1.95", + }, + onFinalize: async () => { + finalizations++; + throw finalization; + }, + }).catch((error: unknown) => error); + expect(error).toBe(finalization); + expect(finalizations).toBe(1); + expect(process.exitCode).toBe(previousExitCode); + } finally { + process.exitCode = previousExitCode; + } + }, +); + +async function observeStandardAdmission( + repository: string, + scanDir: string, + scanId?: string, +) { + const drafts: ScanDraftInput[] = []; + const finalization = new Error("The enclosing finalizer owns the next step."); + let finalizations = 0; + const error = await runScanEvents({ + scanId, + thread: { + id: "standard-thread", + async runStreamed() { + return { events: completedEvents("standard-thread") }; + }, + }, + events: completedEvents("standard-thread"), + signal: new AbortController().signal, + scanDir, + pluginRoot: PLUGIN_ROOT, + expectation: { + repository, + repositoryRevision: null, + target: { kind: "repository", paths: [] }, + mode: "standard", + pluginVersion: "0.1.0", + }, + onFinalize: async () => { + finalizations++; + const [manifest, findings, coverage] = await Promise.all( + ["scan-manifest.json", "findings.json", "coverage.json"].map( + async (name) => + JSON.parse(await readFile(join(scanDir, name), "utf8")), + ), + ); + drafts.push( + parseCanonicalScanDraft({ + scanId: scanId ?? manifest.scan.id, + manifest, + findings, + coverage, + }), + ); + throw finalization; + }, + }).catch((error: unknown) => error); + return { + error, + finalization, + finalizations, + drafts, + }; +} + +function mutateDraft( + input: ScanDraftInput, + mutation?: Mutation, +): ScanDraftInput { + const draft = structuredClone(input); + if (mutation === "missing-findings") + return { ...draft, findings: undefined } as unknown as ScanDraftInput; + if (mutation === "wrong-scan") + draft.scanId = "553a0c18-dcdf-4a3b-8e39-2751a8187bce"; + if (mutation === "contradictory-coverage") + draft.coverage["completeness"] = "complete"; + if (mutation === "inverted-lines") { + const locations = draft.findings[0]!["locations"] as Record< + string, + unknown + >[]; + locations[0]!["endLine"] = 1; + } + if (mutation === "legacy-details") + draft.findings[0]!["validation"] = { + method: null, + limitations: "Legacy persisted limitation.", + }; + return draft; +} + +for (const scenario of ["published", "interrupted", "unfinished"] as const) { + test(`Standard admission reconciles its committed terminal checkpoint: ${scenario}`, async () => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const home = join(root, "codex-home"); + const scanDir = join(root, "scan"); + await mkdir(repository); + await writeFile(join(repository, "extract.py"), "# Synthetic source\n"); + await mkdir(home); + await mkdir(scanDir, { mode: 0o700 }); + const environment = { + ...process.env, + CODEX_HOME: home, + CODEX_SECURITY_STATE_DIR: join(root, "state"), + }; + const python = process.env["PYTHON"] ?? Bun.which("python3")!; + let registration: Record; + let savedOptions: WorkbenchCommandOptions; + const commands: string[] = []; + const runtime = preparedRuntime(home); + runtime.plugin.version = JSON.parse( + await readFile(join(PLUGIN_ROOT, ".codex-plugin", "plugin.json"), "utf8"), + ).version; + const client = new TestClient( + {}, + { + environment, + prepareRuntime: async () => ({ ...runtime, environment }), + resolvePluginPython: async () => python, + prepareOutputDir: async () => scanDir, + runWorkbench: async (options, args, input) => { + savedOptions = options; + commands.push(args[0]!); + const value = await runWorkbench(options, args, input); + if (args[0] === "register-cli-scan") registration = value; + return value; + }, + createCodex: () => ({ + startThread: () => ({ + id: null, + async runStreamed() { + const id = String(registration!["scanId"]); + const context = { + root: scanDir, + repoRoot: repository, + layout: "scan", + scanId: id, + mode: "standard", + status: "running", + scope: ".", + targetContract: registration!["contract"], + }; + const draft: ScanDraftInput = { + scanId: id, + complete: false, + scope: { summary: "Archive extraction." }, + threatModel: { + summary: "An untrusted caller supplies archive entries.", + }, + findings: [ + { + ruleId: "path-traversal.archive", + title: "Unsafe archive extraction", + summary: "An archive entry reaches a filesystem write.", + severity: { level: "high" }, + confidence: { level: "high", rationale: "Source review." }, + taxonomy: { category: "path-traversal", cwe: ["CWE-22"] }, + locations: [ + { path: "extract.py", startLine: 1, endLine: 1 }, + ], + remediation: + "Validate the resolved output path before writing.", + provenance: { + source: "local_plugin", + candidateId: "archive-entry", + }, + }, + ], + coverage: { + completeness: "partial", + surfaces: [ + { + id: "archive", + label: "Archive extraction", + disposition: "reported", + }, + ], + explicitExclusions: [], + deferred: [ + { + id: "deployment", + reason: "Deployment controls remain unverified.", + }, + ], + }, + }; + const publish = (args: readonly string[]) => + runWorkbench(savedOptions!, args); + await recordCodexSecurityScanDraftViaWorkbench( + context, + draft, + publish, + ); + if (scenario !== "unfinished") { + const terminal = { ...draft, complete: true }; + if (scenario === "interrupted") { + await expect( + recordCodexSecurityScanDraftViaWorkbench( + context, + terminal, + async (args: readonly string[]) => { + const result = runNodePython( + python, + [ + "-c", + [ + "import sys", + "sys.path.insert(0, sys.argv[1])", + "import workbench_db as db", + "original = db.saved_results.write_scan_local_bytes", + "def fail_manifest(root, relative, contents):", + " if relative == 'scan-manifest.json': raise OSError('Synthetic interrupted canonical manifest publication')", + " return original(root, relative, contents)", + "db.saved_results.write_scan_local_bytes = fail_manifest", + "sys.argv = ['workbench_db.py', *sys.argv[2:]]", + "db.main()", + ].join("\n"), + join(PLUGIN_ROOT, "scripts"), + ...args, + ], + { + env: { + ...process.env, + ...savedOptions!.environment, + }, + }, + ); + expect(result.status).toBe(1); + expect(result.stderr).toContain( + "Synthetic interrupted canonical manifest publication", + ); + throw new Error( + "Synthetic interrupted canonical manifest publication", + ); + }, + ), + ).rejects.toThrow( + "Synthetic interrupted canonical manifest publication", + ); + const head = JSON.parse( + await readFile( + join(scanDir, "checkpoint-head.json"), + "utf8", + ), + ); + const checkpoint = JSON.parse( + await readFile( + join(scanDir, "checkpoints", head.checkpoint), + "utf8", + ), + ); + expect(checkpoint.complete).not.toBe(false); + expect( + JSON.parse( + await readFile( + join(scanDir, "scan-manifest.json"), + "utf8", + ), + ).scan.complete, + ).toBe(false); + } else + await recordCodexSecurityScanDraftViaWorkbench( + context, + terminal, + publish, + ); + } + return { events: completedEvents("standard-checkpoint-thread") }; + }, + }), + }), + }, + ); + try { + const pending = client.run(repository); + if (scenario === "unfinished") { + await expect(pending).rejects.toThrow(); + expect(commands).not.toContain("complete-scan"); + } else { + const result = await pending; + expect(result.findings.findings).toHaveLength(1); + expect(result.findings.findings[0]!.title).toBe( + "Unsafe archive extraction", + ); + expect(commands).toContain("prepare-scan-completion"); + expect(commands).toContain("complete-scan"); + expect(commands).not.toContain("fail-scan"); + } + } finally { + await client.close(); + } + }); +} diff --git a/sdk/typescript/tests-ts/api-events.test.ts b/sdk/typescript/tests-ts/api-events.test.ts index d3bc1c2203..218799d229 100644 --- a/sdk/typescript/tests-ts/api-events.test.ts +++ b/sdk/typescript/tests-ts/api-events.test.ts @@ -1,6 +1,6 @@ import { copyCompletedScan } from "./plugin-root.js"; import { once } from "node:events"; -import { mkdir, stat } from "node:fs/promises"; +import { mkdir, rm, stat } from "node:fs/promises"; import { existsSync } from "node:fs"; import { join } from "node:path"; import { @@ -507,7 +507,8 @@ describe("one-shot scan events", () => { test("lets the workbench seal artifacts before validating completed scans", async () => { const root = await temporaryDirectory(); - const scanDir = join(root, "scan"); + const scanDir = await copyCompletedScan(root); + await rm(join(scanDir, "report.md")); const events = completedEvents(); let finalized = false; @@ -520,7 +521,8 @@ describe("one-shot scan events", () => { cache_write_input_tokens: 0, output_tokens: 3, }); - expect(existsSync(join(scanDir, "scan-manifest.json"))).toBe(false); + expect(existsSync(join(scanDir, "scan-manifest.json"))).toBe(true); + expect(existsSync(join(scanDir, "report.md"))).toBe(false); await copyCompletedScan(root); finalized = true; }, diff --git a/sdk/typescript/tests-ts/api-policy.test.ts b/sdk/typescript/tests-ts/api-policy.test.ts index 741492e844..9e824bdd49 100644 --- a/sdk/typescript/tests-ts/api-policy.test.ts +++ b/sdk/typescript/tests-ts/api-policy.test.ts @@ -1536,6 +1536,93 @@ describe("CodexSecurity policy API", () => { await f.security.close(); }); + test.each(["architecture", "threat_model"] as const)( + "enforces priced policy usage despite an unpriced remainder in %s", + async (crossingStage) => { + const costs: number[] = []; + const f = await setup({ + config: { codexOverrides: { model: "gpt-5.6-sol" } }, + stream: async function* (stage, signal) { + if (stage !== crossingStage) { + yield* events(stage); + return; + } + const directory = join(f.root, "codex-home", "sessions"); + await mkdir(directory, { recursive: true }); + const thread = `policy-${stage}`; + const input = stage === "architecture" ? 1_200 : 1_000; + for (const [id, model, parent, inputTokens] of [ + [thread, "gpt-5.6-sol", undefined, input], + ["unpriced-policy-worker", "synthetic-unpriced-model", thread, 100], + ] as const) { + await writeFile( + join(directory, `${id}.jsonl`), + [ + JSON.stringify({ + type: "session_meta", + payload: { + id, + ...(parent === undefined + ? {} + : { parent_thread_id: parent }), + }, + }), + JSON.stringify({ type: "turn_context", payload: { model } }), + JSON.stringify({ + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: inputTokens, + output_tokens: 0, + }, + }, + }, + }), + "", + ].join("\n"), + ); + } + yield { type: "thread.started", thread_id: thread }; + await new Promise((resolve) => { + if (signal.aborted) resolve(); + else + signal.addEventListener("abort", () => resolve(), { once: true }); + }); + throw signal.reason; + }, + }); + const keepAlive = setTimeout(() => {}, 10_000); + try { + await expect( + f.security.generatePolicy(f.repository, { + outputDir: f.outputDir, + maxCostUsd: 0.0045, + signal: AbortSignal.timeout(5_000), + onCost: (cost) => costs.push(cost.estimatedUsd), + }), + ).rejects.toThrow("exceeded its $0.0045 cost limit"); + expect(f.threads).toHaveLength( + crossingStage === "architecture" ? 1 : 2, + ); + if (crossingStage === "architecture") expect(costs).toEqual([]); + else { + expect(costs.length).toBeGreaterThan(0); + for (const cost of costs) expect(cost).toBeCloseTo(0.0006, 12); + } + if (crossingStage === "threat_model") { + expect( + await readFile(join(f.outputDir, "project-spec.md"), "utf8"), + ).toContain("src/service.ts:1"); + } + } finally { + clearTimeout(keepAlive); + await f.security.close(); + } + }, + ); + test("enforces one cost budget across stages and preserves completed evidence", async () => { const f = await setup(); await expect( diff --git a/sdk/typescript/tests-ts/api.test.ts b/sdk/typescript/tests-ts/api.test.ts index 59ea106da9..3541717c73 100644 --- a/sdk/typescript/tests-ts/api.test.ts +++ b/sdk/typescript/tests-ts/api.test.ts @@ -15,7 +15,7 @@ import { } from "node:fs/promises"; import * as fsPromises from "node:fs/promises"; import { execFileSync } from "node:child_process"; -import { hash } from "node:crypto"; +import { createHash, hash } from "node:crypto"; import { existsSync } from "node:fs"; import { basename, delimiter, dirname, join, relative, win32 } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; @@ -2552,7 +2552,7 @@ describe("CodexSecurity orchestration", () => { ).toBeUndefined(); expect(prompt).toContain("The SDK has already registered this scan."); expect(prompt).toContain("never call a scan-start or completion tool"); - expect(prompt).toContain("do not finalize or seal them"); + expect(prompt).toContain("leave finalization to the SDK"); expect(prompt).toContain( "This Standard scan authorizes its independent baseline auditor and focused investigators", ); @@ -4391,6 +4391,183 @@ describe("CodexSecurity orchestration", () => { }, ); + test.each([ + ["standard", false], + ["deep", false], + ["standard", true], + ["deep", true], + ] as const)( + "enforces priced usage with an unpriced remainder (%s, raised limit: %s)", + async (mode, raised) => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const codexHome = join(root, "codex-home"); + const scanDir = join(root, "scan"); + await Promise.all([ + mkdir(repository), + mkdir(codexHome), + mkdir(scanDir, { mode: 0o700 }), + ]); + const commands: Array = []; + const costs: number[] = []; + let turns = 0; + let releaseIncrease!: () => void; + const increased = new Promise((resolve) => { + releaseIncrease = resolve; + }); + const knownUsage = { + input_tokens: raised ? 2_500 : 1_250, + cached_input_tokens: 200, + output_tokens: 30, + }; + const expectedCost = estimateScanCost("gpt-5.6-sol", knownUsage)!; + const writePricedUsage = async (usage: Record) => { + const path = await writeUsageSession(codexHome, "scan-thread", usage); + const lines = (await readFile(path, "utf8")).split("\n"); + lines.splice( + 1, + 0, + JSON.stringify({ + type: "turn_context", + payload: { model: "gpt-5.6-sol" }, + }), + ); + await writeFile(path, lines.join("\n")); + }; + const writeUnpricedUsage = async () => { + const path = await writeUsageSession( + codexHome, + "unpriced-worker", + { input_tokens: 100, output_tokens: 10 }, + { parent: "scan-thread" }, + ); + const lines = (await readFile(path, "utf8")).split("\n"); + lines.splice( + 1, + 0, + JSON.stringify({ + type: "turn_context", + payload: { model: "synthetic-unpriced-model" }, + }), + ); + await writeFile(path, lines.join("\n")); + }; + const client = new TestClient( + {}, + { + environment: {}, + prepareRuntime: async () => preparedRuntime(codexHome), + resolvePluginPython: async () => "/managed/python", + prepareOutputDir: async () => scanDir, + repositoryRevision: async () => "deadbeef", + runWorkbench: async (_options, args, input) => { + commands.push(args); + if (args[0] === "get-scan") + return { scan: { id: "scan_example_001" } }; + if (args[0] === "complete-budget-exhausted-scan") + throw new Error("Synthetic canonical output is not ready"); + return mockWorkbench(args, input); + }, + createCodex: () => ({ + startThread: () => ({ + id: null, + async runStreamed( + _input: string, + options: { signal: AbortSignal }, + ) { + turns++; + async function* events(): AsyncGenerator { + if (raised) { + await writePricedUsage({ + input_tokens: 800, + output_tokens: 0, + }); + } else { + await writePricedUsage(knownUsage); + await writeUnpricedUsage(); + } + yield { type: "thread.started", thread_id: "scan-thread" }; + if (raised) { + await increased; + await writeUnpricedUsage(); + const path = join( + codexHome, + "sessions", + "2026", + "07", + "26", + "rollout-scan-thread.jsonl", + ); + await appendFile( + path, + JSON.stringify({ + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: knownUsage }, + }, + }) + "\n", + ); + } + await new Promise((resolve) => { + if (options.signal.aborted) resolve(); + else + options.signal.addEventListener( + "abort", + () => resolve(), + { once: true }, + ); + }); + throw new DOMException("aborted", "AbortError"); + } + return { events: events() }; + }, + }), + }), + }, + ); + const keepAlive = setTimeout(() => {}, 10_000); + try { + const failure = await client + .run(repository, { + mode, + maxCostUsd: 0.004, + signal: AbortSignal.timeout(5_000), + postScanPrompt: "No model work after the budget stop.", + ...(raised ? { onBudgetApproaching: () => 0.008 } : {}), + onCost: (cost, limit) => { + costs.push(cost.estimatedUsd); + if (limit === 0.008) releaseIncrease(); + }, + }) + .catch((error: unknown) => error); + expect(failure).toBeInstanceOf(ScanCostLimitExceededError); + expect(failure).toMatchObject({ + maxCostUsd: raised ? 0.008 : 0.004, + cost: { + estimatedUsd: expectedCost.estimatedUsd, + inputTokens: knownUsage.input_tokens, + coverage: "partial", + }, + }); + expect(costs.every((cost) => raised && cost === 0.0032)).toBe(true); + expect(turns).toBe(1); + expect( + commands.some((args) => args[0] === "complete-budget-exhausted-scan"), + ).toBe(mode === "deep"); + if (raised) + expect( + commands + .filter((args) => args[0] === "set-scan-cost-limit") + .map((args) => args.at(-1)), + ).toEqual(["0.008"]); + } finally { + clearTimeout(keepAlive); + await client.close(); + } + }, + ); + test("stops and records a scan as soon as its live cost exceeds the limit", async () => { const { repository, codexHome, scanDir } = await scanDirectories(); const commands: Array = []; @@ -4526,6 +4703,10 @@ describe("CodexSecurity orchestration", () => { input?: string, ): Promise => { commands.push(args); + if (args[0] === "get-scan") { + // Older workbench readers return a scan without execution attribution. + return { scan: { id: "scan_example_001" } }; + } if (args[0] !== "complete-budget-exhausted-scan") { return mockWorkbench(args, input); } @@ -5837,7 +6018,8 @@ describe("CodexSecurity orchestration", () => { ); expect(prompt()).toContain("$codex-security:deep-security-scan"); expect(prompt()).not.toContain("record_codex_security_scan_draft"); - expect(prompt()).toContain("complete_codex_security_scan"); + expect(prompt()).not.toContain("complete_codex_security_scan"); + expect(prompt()).toContain("Leave finalization to the SDK after this turn"); expect(prompt()).not.toContain("do not finalize or seal them"); expect(prompt()).toContain( 'start_codex_security_deep_scan with {"scanId":"scan_example_001"}', @@ -6604,6 +6786,227 @@ if ([basename(process.argv[1]), ...process.argv.slice(2)].join(" ") !== "login s expect(scanSignal?.aborted).toBe(false); }); + test.each([ + ["standard", false, false], + ["deep", false, false], + ["deep", false, true], + ["deep", true, true], + ] as const)( + "isolates concurrent managed %s sessions at the Codex child boundary (capture=%s, program=%s)", + async (mode, captureSummary, selectedProgram) => { + const clients: TestClient[] = []; + try { + const outcomes = await Promise.allSettled( + ["first", "second"].map(async (name) => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const codexHome = join(root, "codex-home"); + const scanDir = join(root, "scan"); + const preload = join(root, "fake-codex.mjs"); + const marker = join(root, "invocation.jsonl"); + await Promise.all([ + mkdir(repository), + mkdir(codexHome), + mkdir(scanDir, { mode: 0o700 }), + ]); + const model = `fixture-${name}-model`; + const summary = name === "first" ? "none" : "concise"; + const configPath = join(codexHome, "config-preflight.toml"); + let recipeConfig: JsonObject | undefined; + await writeFile( + preload, + [ + 'import { appendFileSync } from "node:fs";', + 'let prompt = ""; for await (const chunk of process.stdin) prompt += chunk;', + `appendFileSync(${JSON.stringify(marker)}, JSON.stringify({args:process.argv, executable:process.execPath, cwd:process.cwd(), home:process.env.CODEX_HOME, key:process.env.CODEX_API_KEY, value:process.env.FIXTURE_SCAN_VALUE, prompt}) + "\\n");`, + `if (process.argv.includes("models")) { console.log(JSON.stringify({models:[{slug:${JSON.stringify(model)},default_reasoning_summary:${JSON.stringify(summary)}}]})); process.exit(0); }`, + `console.log(JSON.stringify({type:"thread.started",thread_id:${JSON.stringify(`fixture-${name}-thread`)}}));`, + 'console.log(JSON.stringify({type:"item.completed",item:{type:"agent_message",text:"scan complete"}}));', + 'console.log(JSON.stringify({type:"turn.completed",usage:null}));', + "process.exit(0);", + ].join("\n"), + ); + const fake = nodeCodex(preload); + const provider = `fixture-${name}-provider`; + const client = new TestClient( + { + codexOverrides: { + model, + model_provider: provider, + model_reasoning_effort: "ultra", + // JavaScript callers can omit the merged default with undefined. + model_reasoning_summary: captureSummary + ? (undefined as unknown as string) + : summary, + service_tier: name === "first" ? "flex" : "fast", + features: { + multi_agent_v2: { max_concurrent_threads_per_session: 4 }, + }, + }, + }, + { + environment: { + OPENAI_API_KEY: `synthetic-${name}-key`, + CODEX_CLI_PATH: fake.command.command, + }, + prepareRuntime: async () => ({ + ...preparedRuntime(codexHome), + configPath, + environment: { + ...fake.environment, + FIXTURE_SCAN_VALUE: name, + }, + }), + resolvePluginPython: async () => "/managed/python", + prepareOutputDir: async () => scanDir, + repositoryRevision: async () => "deadbeef", + runWorkbench: async (_options, args, input) => { + if (args[0] === "register-cli-scan") + recipeConfig = JSON.parse(input!).recipe.config; + return mockWorkbench(args, input); + }, + createCodex: (options: CodexOptions) => { + const codex = new Codex(options); + return { + startThread: (threadOptions: ThreadOptions) => { + const thread = codex.startThread(threadOptions); + return { + get id() { + return thread.id; + }, + runStreamed: async ( + ...args: Parameters + ) => { + if (thread.id === null) { + await copyCompletedScan(root); + if (mode === "deep") { + const coveragePath = join( + scanDir, + "coverage.json", + ); + const coverage = JSON.parse( + await readFile(coveragePath, "utf8"), + ); + coverage.mode = "deep_repository"; + const coverageBytes = JSON.stringify(coverage); + await writeFile(coveragePath, coverageBytes); + const manifestPath = join( + scanDir, + "scan-manifest.json", + ); + const manifest = JSON.parse( + await readFile(manifestPath, "utf8"), + ); + manifest.scan.artifacts.find( + (artifact: { path: string }) => + artifact.path === "coverage.json", + ).sha256 = createHash("sha256") + .update(coverageBytes) + .digest("hex"); + await writeFile( + manifestPath, + JSON.stringify(manifest), + ); + } + } + return thread.runStreamed(...args); + }, + }; + }, + }; + }, + }, + ); + clients.push(client); + const postScanPrompt = "Summarize the completed synthetic scan."; + const program = name === "first" ? "daybreak_blue" : "daybreak_red"; + const result = await client.run(repository, { + mode, + postScanPrompt, + ...(selectedProgram ? { cyberAccessProgram: program } : {}), + }); + expect(result.threadId).toBe(`fixture-${name}-thread`); + expect(result.turnResult.usage).toBeNull(); + const invocations = (await readFile(marker, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + const lookups = invocations.filter((child) => + child.args.includes("models"), + ); + const children = invocations.filter( + (child) => !child.args.includes("models"), + ); + expect(recipeConfig?.["model_reasoning_summary"]).toBe(summary); + expect(lookups).toHaveLength(captureSummary ? 1 : 0); + for (const lookup of lookups) { + expect(await realpath(lookup.cwd)).toBe(await realpath(scanDir)); + expect(lookup.home).toBe(codexHome); + expect(lookup.key).toBe(`synthetic-${name}-key`); + expect(lookup.value).toBe(name); + expect(lookup.args).toContain(`model=${JSON.stringify(model)}`); + } + const preflight = await readFile(configPath, "utf8"); + expect(parseToml(preflight)["model_reasoning_summary"]).toBe( + summary, + ); + expect(preflight).not.toContain(`synthetic-${name}-key`); + if (selectedProgram) + expect(parseToml(preflight)).toMatchObject({ + codex_security: { cyber_access_program: program }, + }); + else expect(parseToml(preflight)["codex_security"]).toBeUndefined(); + expect(children).toHaveLength(2); + expect(children[1].prompt).toBe(postScanPrompt); + expect(children[1].args).toContain("resume"); + expect(children[1].args).toContain(`fixture-${name}-thread`); + for (const child of children) { + expect(child.executable).toBe( + process.platform === "win32" + ? win32.toNamespacedPath(fake.command.command) + : fake.command.command, + ); + expect(child.home).toBe(codexHome); + expect(child.key).toBe(`synthetic-${name}-key`); + expect(child.value).toBe(name); + if (selectedProgram) { + const programIndex = child.args.indexOf( + "--cyber-access-program", + ); + expect( + child.args.slice(programIndex, programIndex + 2), + ).toEqual(["--cyber-access-program", program]); + } else expect(child.args).not.toContain("--cyber-access-program"); + expect(child.args).toContain(`model=${JSON.stringify(model)}`); + expect(child.args).toContain( + `model_provider=${JSON.stringify(provider)}`, + ); + expect(child.args).toContain('model_reasoning_effort="ultra"'); + expect(child.args).toContain( + `model_reasoning_summary=${JSON.stringify(name === "first" ? "none" : "concise")}`, + ); + expect(child.args).toContain( + `service_tier=${JSON.stringify(name === "first" ? "flex" : "fast")}`, + ); + expect(child.args).toContain( + "features.multi_agent_v2.max_concurrent_threads_per_session=4", + ); + expect(child.args).toContain( + 'default_permissions="codex_security_scan"', + ); + expect(child.args).toContain('approval_policy="on-request"'); + } + }), + ); + for (const outcome of outcomes) { + if (outcome.status === "rejected") throw outcome.reason; + } + } finally { + await Promise.all(clients.map((client) => client.close())); + } + }, + ); + test("closes a real Codex subprocess cleanly after a streamed terminal failure", async () => { const root = await temporaryDirectory(); const repository = join(root, "repository"); diff --git a/sdk/typescript/tests-ts/build-plugin.test.ts b/sdk/typescript/tests-ts/build-plugin.test.ts index 02eb832f70..b36b76d503 100644 --- a/sdk/typescript/tests-ts/build-plugin.test.ts +++ b/sdk/typescript/tests-ts/build-plugin.test.ts @@ -1,6 +1,7 @@ import { execFile } from "node:child_process"; import { chmod, + copyFile, cp, mkdir, readFile, @@ -9,11 +10,12 @@ import { symlink, writeFile, } from "node:fs/promises"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { basename, delimiter, join } from "node:path"; -import { fileURLToPath } from "node:url"; import { brotliDecompressSync } from "node:zlib"; import { promisify } from "node:util"; import { afterEach, describe, expect, test } from "bun:test"; +import { transform } from "esbuild"; import { buildBundledPlugin } from "../scripts/build-plugin.mjs"; import { assertGeneratedPluginUntracked } from "../scripts/check-plugin-source.mjs"; @@ -62,6 +64,275 @@ async function snapshot(root: string) { afterEach(temporaryDirectories.cleanup); describe("bundled plugin build", () => { + test("bundles the native policy proof with only SDK dependencies", async () => { + const root = await temporaryDirectory(); + const plugin = join(root, "plugins", "codex-security"); + const native = join(plugin, "native"); + const sdk = join(root, "sdk", "typescript"); + const source = new URL("../../../plugins/codex-security/", import.meta.url); + await mkdir(native, { recursive: true }); + await mkdir(sdk, { recursive: true }); + await symlink( + fileURLToPath(new URL("../node_modules", import.meta.url)), + join(sdk, "node_modules"), + process.platform === "win32" ? "junction" : "dir", + ); + for (const name of ["schemas", "mcp-app/src"]) { + await cp(new URL(name, source), join(plugin, name), { recursive: true }); + } + await copyFile( + new URL("mcp-app/helpers-main.ts", source), + join(plugin, "mcp-app", "helpers-main.ts"), + ); + for (const name of [ + "binding", + "platform", + "windows-binding", + "windows-flags", + "windows-files", + "proof-policy-windows", + ]) { + const compiled = await transform( + await readFile(new URL(`native/${name}.mts`, source), "utf8"), + { loader: "ts", format: "esm", target: "node20" }, + ); + await writeFile(join(native, `${name}.mjs`), compiled.code); + } + const { nativeTarget } = await import( + pathToFileURL(join(native, "platform.mjs")).href + ); + const binary = process.platform === "win32" ? "windows.node" : "unix.node"; + // The build copies this artifact; this portable test does not load native code. + await writeFixture( + native, + `dist/${nativeTarget}/${binary}`, + "native fixture", + ); + await expect( + stat(join(plugin, "mcp-app", "node_modules")), + ).rejects.toMatchObject({ + code: "ENOENT", + }); + await execFileAsync( + "node", + [join(native, "proof-policy-windows.mjs"), "build"], + { + cwd: native, + env: { ...process.env, NODE_PATH: "" }, + }, + ); + const proof = join(native, "dist", nativeTarget, "policy-proof"); + expect( + await readFile( + join(proof, "native", nativeTarget, "windows.node"), + "utf8", + ), + ).toBe("native fixture"); + const helper = join(root, "helpers.cjs"); + await copyFile(join(proof, "helpers.cjs"), helper); + await execFileAsync("node", [ + "--eval", + "require('node:fs').unlinkSync(process.argv[1])", + join(sdk, "node_modules"), + ]); + await expect(stat(join(sdk, "node_modules"))).rejects.toMatchObject({ + code: "ENOENT", + }); + const result = await execFileAsync( + "node", + [ + "--eval", + ` + const assert = require("node:assert/strict"); + const helper = require(process.argv.pop()); + const input = { + scanId: "synthetic-scan", + manifest: { scan: {} }, + findings: { findings: [] }, + coverage: { + completeness: "complete", surfaces: [], explicitExclusions: [], deferred: [], + }, + }; + assert.equal(helper.parseCanonicalScanDraft(input).scanId, input.scanId); + assert.throws(() => helper.parseCanonicalScanDraft({ + ...input, coverage: { ...input.coverage, completeness: "invalid" }, + })); + console.log("Bundled parser accepted valid input and rejected invalid coverage."); + `, + helper, + ], + { cwd: root, env: { ...process.env, NODE_PATH: "" } }, + ); + expect(result.stdout).toBe( + "Bundled parser accepted valid input and rejected invalid coverage.\n", + ); + expect(result.stderr).toBe(""); + }); + + test("builds the MCP runtime with only MCP dependencies and no npm launcher", async () => { + const root = await temporaryDirectory(); + const plugin = join(root, "plugins", "codex-security"); + const mcp = join(plugin, "mcp-app"); + const sdk = join(root, "sdk", "typescript"); + const source = new URL("../../../plugins/codex-security/", import.meta.url); + await mkdir(mcp, { recursive: true }); + await mkdir(sdk, { recursive: true }); + for (const name of [ + "package.json", + "tsconfig.json", + "main.ts", + "artifact-writer-main.ts", + "helpers-main.ts", + "server.ts", + "src", + "scripts", + "templates", + "tests/support/temporary-directories.ts", + "tests/support/json.ts", + ]) { + await cp(new URL(`mcp-app/${name}`, source), join(mcp, name), { + recursive: true, + }); + } + for (const name of [ + "schemas", + "native/prebuilt", + "plugin-files.json", + "scripts/reserved_artifact_paths.json", + ]) { + await cp(new URL(name, source), join(plugin, name), { recursive: true }); + } + for (const name of await readdir(new URL("native/", source))) { + if (/\.(?:mjs|mts)$/.test(name)) { + await copyFile( + new URL(`native/${name}`, source), + join(plugin, "native", name), + ); + } + } + for (const name of ["src", "package.json", "tsconfig.json"]) { + await cp(new URL(`../${name}`, import.meta.url), join(sdk, name), { + recursive: true, + }); + } + await symlink( + fileURLToPath(new URL("mcp-app/node_modules", source)), + join(mcp, "node_modules"), + process.platform === "win32" ? "junction" : "dir", + ); + await expect(stat(join(sdk, "node_modules"))).rejects.toMatchObject({ + code: "ENOENT", + }); + const bin = join(root, "bin"); + const launcher = process.platform === "win32" ? "npm.cmd" : "npm"; + await writeFixture( + bin, + launcher, + process.platform === "win32" ? "@exit /b 91\r\n" : "#!/bin/sh\nexit 91\n", + ); + if (process.platform !== "win32") await chmod(join(bin, launcher), 0o755); + + const destination = join(root, "mcp"); + await execFileAsync( + "node", + [join(mcp, "scripts", "build_mcp_app.mjs"), "--output", destination], + { + env: { + ...process.env, + NODE_PATH: "", + PATH: [bin, process.env["PATH"]].filter(Boolean).join(delimiter), + }, + }, + ); + + await execFileAsync("node", [ + "--eval", + "require('node:fs').unlinkSync(process.argv[1])", + join(mcp, "node_modules"), + ]); + await expect(stat(join(mcp, "node_modules"))).rejects.toMatchObject({ + code: "ENOENT", + }); + const contract = JSON.parse( + await readFile( + new URL( + "../../../plugins/codex-security/plugin-files.json", + import.meta.url, + ), + "utf8", + ), + ) as { shippedExact: string[] }; + expect(await files(destination)).toEqual( + contract.shippedExact + .filter((path) => path.startsWith("mcp/")) + .map((path) => path.slice(4)) + .sort(), + ); + const helper = await execFileAsync("node", [ + join(destination, "helpers.mjs"), + "resolve-security-md", + "--repo", + root, + "--list", + ]); + expect(helper.stdout).toBe("[]\n"); + expect(helper.stderr).toBe(""); + + const repository = await temporaryDirectory(); + const policy = "Preserve this synthetic inherited security policy."; + await writeFixture( + repository, + "SECURITY.md", + `# Synthetic policy\n${policy}\n`, + ); + const alias = join(await temporaryDirectory(), "plugin link"); + await symlink( + root, + alias, + process.platform === "win32" ? "junction" : "dir", + ); + await mkdir(join(root, "scripts"), { recursive: true }); + await copyFile( + new URL("scripts/launch_codex_security_mcp", source), + join(root, "scripts", "launch_codex_security_mcp"), + ); + const node = ( + await execFileAsync("node", ["--print", "process.execPath"]) + ).stdout.trim(); + for (const pluginPath of [root, alias]) { + const linkedHelper = join(pluginPath, "mcp", "helpers.mjs"); + const list = await execFileAsync( + process.platform === "win32" ? node : "/bin/sh", + [ + ...(process.platform === "win32" + ? [linkedHelper] + : [ + join(pluginPath, "scripts", "launch_codex_security_mcp"), + "--helper", + ]), + "resolve-security-md", + "--repo", + repository, + "--list", + ], + { env: { ...process.env, CODEX_MCP_NODE_PATH: node, NODE_PATH: "" } }, + ); + expect(list.stdout).toBe('["SECURITY.md"]\n'); + expect(list.stderr).toBe(""); + const guidance = await execFileAsync(node, [ + linkedHelper, + "resolve-security-md", + "--repo", + repository, + "--scope", + repository, + "--out", + "-", + ]); + expect(guidance.stdout).toContain(policy); + expect(guidance.stderr).toBe(""); + } + }); test.each(["missing", "stale"])( "builds the MCP runtime from source with %s native wrappers and no npm launcher", async (emitted) => { @@ -69,7 +340,17 @@ describe("bundled plugin build", () => { const plugin = fileURLToPath( new URL("../../../plugins/codex-security/", import.meta.url), ); - const source = join(root, "plugin"); + const source = join(root, "plugins", "codex-security"); + const sdk = join(root, "sdk", "typescript"); + await mkdir(sdk, { recursive: true }); + for (const name of ["src", "package.json", "tsconfig.json"]) { + await cp(new URL(`../${name}`, import.meta.url), join(sdk, name), { + recursive: true, + }); + } + await expect(stat(join(sdk, "node_modules"))).rejects.toMatchObject({ + code: "ENOENT", + }); await cp(join(plugin, "mcp-app"), join(source, "mcp-app"), { recursive: true, filter: (path) => diff --git a/sdk/typescript/tests-ts/cli-scan-logs.test.ts b/sdk/typescript/tests-ts/cli-scan-logs.test.ts index 4c458ec4d3..9348921683 100644 --- a/sdk/typescript/tests-ts/cli-scan-logs.test.ts +++ b/sdk/typescript/tests-ts/cli-scan-logs.test.ts @@ -1,4 +1,3 @@ -import { writeJsonLines } from "./support/json.js"; import { execFile } from "node:child_process"; import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -6,21 +5,19 @@ import { join } from "node:path"; import { Writable } from "node:stream"; import { promisify } from "node:util"; import { pathToFileURL } from "node:url"; -import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { describe, expect, test } from "bun:test"; import { Cli, Formatter, z } from "incur"; import { main } from "../src/cli.js"; import { scanLogsJson } from "../src/cli-scan-logs-json.js"; import { readSavedScanLogs } from "../src/scan-logs.js"; import { VERSION } from "../src/version.js"; import { capture, dependencies } from "./cli-fixtures.js"; -import { throwing } from "./support/errors.js"; -import { createCliTest } from "./support/cli-run.js"; -async function fixture() { +async function fixture(attributedOwner = false) { const state = await realpath(await mkdtemp(join(tmpdir(), "saved-logs-"))); const home = join(state, "codex-home"); await mkdir(join(home, "sessions"), { recursive: true }); - const events = [ + const events: Record[] = [ { type: "session_meta", payload: { id: "thread-1" } }, { type: "event_msg", @@ -30,15 +27,112 @@ async function fixture() { }, }, ]; - await writeJsonLines(join(home, "sessions", "rollout.jsonl"), events); - const scan = { scanId: "scan-1", continuationThreadId: "thread-1" }; - const logs = await readSavedScanLogs(scan, home); + const timestamp = "2026-08-11T12:01:00.000Z"; + if (attributedOwner) { + events.splice(1, 0, { + type: "turn_context", + timestamp, + payload: { turn_id: "scan-turn" }, + }); + Object.assign(events.at(-1)!, { timestamp }); + } + await writeFile( + join(home, "sessions", "rollout.jsonl"), + events.map((event) => JSON.stringify(event)).join("\n"), + ); + const originalHome = join(state, "original-home"); + const scanDir = join(state, "scan"); + const settingsDirectory = join(scanDir, "artifacts", "deep_discovery"); + await mkdir(settingsDirectory, { recursive: true }); + await mkdir(join(originalHome, "sessions"), { recursive: true }); + let ownerEvents = events; + if (attributedOwner) { + const repeated = { + type: "event_msg", + timestamp, + payload: { message: "repeated scan occurrence" }, + }; + ownerEvents = [ + ...events, + repeated, + repeated, + { + type: "event_msg", + timestamp, + payload: { message: "recorded non-usage suffix" }, + }, + ]; + await writeFile( + join(home, "sessions", "rollout.jsonl"), + [ + ...events, + { type: "turn_context", timestamp, payload: { turn_id: "other-turn" } }, + { + type: "event_msg", + timestamp, + payload: { message: "unrelated first-copy suffix" }, + }, + ] + .map((event) => JSON.stringify(event)) + .join("\n"), + ); + await writeFile( + join(originalHome, "sessions", "owner.jsonl"), + ownerEvents.map((event) => JSON.stringify(event)).join("\n"), + ); + } + await writeFile( + join(settingsDirectory, "execution-settings.json"), + JSON.stringify({ + version: 1, + settings: { + codexHome: originalHome, + codexPath: join(originalHome, "codex"), + }, + }), + ); + await writeFile( + join(home, "sessions", "worker.jsonl"), + JSON.stringify({ type: "session_meta", payload: { id: "worker" } }) + "\n", + ); + await writeFile( + join(originalHome, "sessions", "worker.jsonl"), + [ + { type: "session_meta", payload: { id: "worker" } }, + { type: "event_msg", payload: { message: "recorded worker suffix" } }, + ] + .map((event) => JSON.stringify(event) + "\n") + .join(""), + ); + const scan = { + scanId: "scan-1", + continuationThreadId: "thread-1", + mode: "deep", + scanDir, + executionThreadIds: ["worker"], + executionAttribution: { + formatVersion: 1 as const, + ...(attributedOwner ? {} : { legacy: true as const }), + workerCodexHome: originalHome, + executionThreadIds: ["worker"], + owner: { + threadId: "thread-1", + turnId: attributedOwner ? "scan-turn" : null, + startedAt: timestamp, + }, + startedAt: timestamp, + completedAt: timestamp, + }, + }; + const logs = await readSavedScanLogs(scan, [home, originalHome]); const deps = dependencies({ environment: { CODEX_SECURITY_STATE_DIR: state }, onWorkbench: () => ({ scan }), }); - deps.createSecurity = throwing("Reading logs must not start Codex"); - return { state, logs, deps }; + deps.createSecurity = () => { + throw new Error("Reading logs must not start Codex"); + }; + return { state, logs, deps, ownerEvents }; } async function referenceOutput(args: string[], logs: unknown) { @@ -65,24 +159,68 @@ function withoutDuration(text: string) { } describe("saved logs JSON output", () => { - let dataHome: string; - let previousDataHome: string | undefined; - - beforeEach(async () => { - previousDataHome = process.env["XDG_DATA_HOME"]; - dataHome = await mkdtemp(join(tmpdir(), "saved-logs-data-")); - process.env["XDG_DATA_HOME"] = dataHome; + test("selects the recorded owner suffix after scan attribution through the saved logs command", async () => { + const f = await fixture(true); + try { + const stdout = capture(); + expect( + await main( + ["scans", "logs", "scan-1", "--json"], + stdout.stream, + capture().stream, + f.deps, + ), + ).toBe(0); + const result = JSON.parse(stdout.text()); + expect( + result.sessions.map(({ threadId }: { threadId: string }) => threadId), + ).toEqual(["worker", "thread-1"]); + expect( + result.events + .filter( + ({ threadId }: { threadId: string }) => threadId === "thread-1", + ) + .map(({ event }: { event: unknown }) => event), + ).toEqual(f.ownerEvents); + } finally { + await rm(f.state, { recursive: true, force: true }); + } }); - afterEach(async () => { - if (previousDataHome === undefined) delete process.env["XDG_DATA_HOME"]; - else process.env["XDG_DATA_HOME"] = previousDataHome; - await rm(dataHome, { recursive: true, force: true }); + test("loads the same-thread recorded worker suffix through the saved logs command", async () => { + const f = await fixture(); + try { + const stdout = capture(); + expect( + await main( + ["scans", "logs", "scan-1", "--json"], + stdout.stream, + capture().stream, + f.deps, + ), + ).toBe(0); + expect( + JSON.parse(stdout.text()).sessions.map( + ({ threadId }: { threadId: string }) => threadId, + ), + ).toEqual(["thread-1", "worker"]); + expect(JSON.parse(stdout.text()).events).toContainEqual({ + threadId: "worker", + event: { + type: "event_msg", + payload: { message: "recorded worker suffix" }, + }, + }); + } finally { + await rm(f.state, { recursive: true, force: true }); + } }); test("preserves the stale installed-skills CTA after saved logs", async () => { const f = await fixture(); + const previousDataHome = process.env["XDG_DATA_HOME"]; try { + const dataHome = join(f.state, "data"); const skillPath = join(f.state, "skills", "codex-security-scans"); await mkdir(join(dataHome, "incur"), { recursive: true }); await mkdir(skillPath, { recursive: true }); @@ -98,16 +236,22 @@ describe("saved logs JSON output", () => { paths: [skillPath], }), ); + process.env["XDG_DATA_HOME"] = dataHome; for (const args of [ ["--json"], ["--format", "json"], ["--format=json"], ]) { - const { stdout, stderr, runCli } = createCliTest(main); - - expect(await runCli(["scans", "logs", "scan-1", ...args], f.deps)).toBe( - 0, - ); + const stdout = capture(); + const stderr = capture(); + expect( + await main( + ["scans", "logs", "scan-1", ...args], + stdout.stream, + stderr.stream, + f.deps, + ), + ).toBe(0); const expected = await referenceOutput(["--json"], f.logs); expect(Object.keys(JSON.parse(expected))).toEqual([ "scanId", @@ -120,6 +264,8 @@ describe("saved logs JSON output", () => { expect(stderr.text()).toBe(""); } } finally { + if (previousDataHome === undefined) delete process.env["XDG_DATA_HOME"]; + else process.env["XDG_DATA_HOME"] = previousDataHome; await rm(f.state, { recursive: true, force: true }); } }); @@ -153,11 +299,16 @@ describe("saved logs JSON output", () => { )("preserves Incur output for %j", async (args) => { const f = await fixture(); try { - const { stdout, stderr, runCli } = createCliTest(main); - - expect(await runCli(["scans", "logs", "scan-1", ...args], f.deps)).toBe( - 0, - ); + const stdout = capture(); + const stderr = capture(); + expect( + await main( + ["scans", "logs", "scan-1", ...args], + stdout.stream, + stderr.stream, + f.deps, + ), + ).toBe(0); const expected = await referenceOutput( args.flatMap((arg) => arg === "--format=json" ? ["--format", "json"] : [arg], @@ -186,27 +337,42 @@ describe("saved logs JSON output", () => { ["--json", "--filter-output"], ].map((args) => [args]), )("preserves invalid-option failure for %j", async (args) => { - const { stdout, stderr, runCli } = createCliTest(main); - + const stdout = capture(); + const stderr = capture(); expect( - await runCli(["scans", "logs", "scan-1", ...args], dependencies()), + await main( + ["scans", "logs", "scan-1", ...args], + stdout.stream, + stderr.stream, + dependencies(), + ), ).toBe(2); expect(stdout.text()).toBe(""); expect(stderr.text()).not.toBe(""); }); - test("formats empty session and event lists", async () => { - const logs = { - scanId: "scan-1", - threadId: "thread-1", - sessions: [], - events: [], - }; - const chunks = []; - for await (const chunk of scanLogsJson(logs)) chunks.push(chunk); - expect(Buffer.concat(chunks).toString()).toBe( - `${Formatter.format(logs, "json")}\n`, - ); + test("preserves JSON conversion and empty arrays", async () => { + const f = await fixture(); + try { + f.logs.events.push({ + threadId: "thread-1", + event: { + bigint: 42n, + missing: undefined, + values: [NaN, Infinity, -0], + }, + }); + for (const logs of [f.logs, { ...f.logs, sessions: [], events: [] }]) { + const chunks = []; + for await (const chunk of scanLogsJson(logs)) + chunks.push(Buffer.from(chunk)); + expect(Buffer.concat(chunks).toString()).toBe( + `${Formatter.format(logs, "json")}\n`, + ); + } + } finally { + await rm(f.state, { recursive: true, force: true }); + } }); test("waits for output backpressure and leaves the stream open", async () => { diff --git a/sdk/typescript/tests-ts/cost.test.ts b/sdk/typescript/tests-ts/cost.test.ts index 021656058e..13dbaaa4e2 100644 --- a/sdk/typescript/tests-ts/cost.test.ts +++ b/sdk/typescript/tests-ts/cost.test.ts @@ -1,6 +1,14 @@ import { jsonLines } from "./support/json.js"; import { spawnSync } from "node:child_process"; -import { appendFile, writeFile } from "node:fs/promises"; +import { + appendFile, + cp, + mkdir, + readFile, + unlink, + symlink, + writeFile, +} from "node:fs/promises"; import { join, parse, sep } from "node:path"; import { Codex } from "@openai/codex-sdk"; import { afterEach, describe, expect, test } from "bun:test"; @@ -8,9 +16,14 @@ import { estimateScanCost, ScanCostTracker, type ScanSessionEvent, + type ScanCost, } from "../src/cost.js"; import type { ScanActivity } from "../src/scan-activity.js"; -import { formatTokenUsage, tokenUsage } from "../src/cost-model.js"; +import { + estimateScanCostLowerBound, + formatTokenUsage, + tokenUsage, +} from "../src/cost-model.js"; import { readScanLogs } from "../src/scan-logs.js"; import { sessionParentThreadId } from "../src/scan-sessions.js"; import type { ScanProgress } from "../src/worker-progress.js"; @@ -2116,3 +2129,1014 @@ describe("live scan cost tracking", () => { }, ); }); + +describe("recorded Deep worker homes", () => { + test("only enforces a priced subtotal from a valid attributed usage partition", () => { + const known = { + model: "gpt-5.6-sol", + input_tokens: 1_000, + output_tokens: 0, + }; + const unknown = { model: null, input_tokens: 100, output_tokens: 0 }; + const usage = { + input_tokens: 1_100, + output_tokens: 0, + modelUsage: [known, unknown], + }; + expect(estimateScanCostLowerBound("gpt-5.6-sol", usage)?.estimatedUsd).toBe( + 0.004, + ); + expect(estimateScanCost("gpt-5.6-sol", usage)).toBeNull(); + for (const invalid of [ + { ...usage, input_tokens: 999 }, + { ...usage, modelUsage: [known, known, unknown] }, + { ...usage, modelUsage: [known, { ...unknown, input_tokens: -1 }] }, + { ...usage, modelUsage: [{ ...known, model: null }, unknown] }, + ]) + expect(estimateScanCostLowerBound("gpt-5.6-sol", invalid)).toBeNull(); + }); + + test.each(["gpt-5.6-sol", "synthetic-unpriced-model"])( + "prices live worker counters after an earlier response receipt with model %s", + async (currentModel) => { + const home = await codexHome(); + const at = "2026-09-01T00:00:02Z"; + await writeSession(home, "owner", {}); + const worker = await writeSession(home, "worker", {}); + const receipt = (id: string, model: string, input: number) => ({ + type: "token_usage_record", + timestamp: at, + payload: { + thread_id: "worker", + turn_id: "worker-turn", + response_id: id, + model, + usage: { input_tokens: input, output_tokens: 0 }, + }, + }); + await appendFile( + worker, + jsonLines([ + { + type: "turn_context", + timestamp: at, + payload: { turn_id: "worker-turn", model: "gpt-5.6-sol" }, + }, + { + type: "event_msg", + timestamp: at, + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: 100, + output_tokens: 0, + }, + }, + }, + }, + receipt("earlier", "gpt-5.6-sol", 100), + ]) + "\n", + ); + const costUpdates: Readonly[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + maxCostUsd: 0.002, + onCost: (cost) => costUpdates.push(cost), + onCostLowerBound: (cost) => costUpdates.push(cost), + }); + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + executionThreadIds: ["worker"], + owner: { threadId: "owner", turnId: "turn", startedAt: at }, + startedAt: at, + completedAt: null, + })); + tracker.start("owner"); + try { + await tracker.refresh(); + expect(costUpdates.at(-1)?.estimatedUsd).toBeCloseTo(0.0004, 10); + await appendFile( + worker, + jsonLines([ + { + type: "turn_context", + timestamp: at, + payload: { turn_id: "worker-turn", model: currentModel }, + }, + { + type: "event_msg", + timestamp: at, + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: 1_100, + output_tokens: 0, + }, + }, + }, + }, + ]) + "\n", + ); + const running = await tracker.refresh(); + expect(tokenUsage(running.usage)?.input_tokens).toBe(1_100); + expect(running.cost).toBeNull(); + expect(costUpdates.at(-1)?.estimatedUsd).toBeCloseTo( + currentModel === "gpt-5.6-sol" ? 0.0044 : 0.0004, + 10, + ); + if (currentModel === "gpt-5.6-sol") + expect(costUpdates.at(-1)!.estimatedUsd).toBeGreaterThan(0.002); + const completedReceipt = receipt("current", currentModel, 1_000); + await appendFile( + worker, + jsonLines([completedReceipt, completedReceipt]) + "\n", + ); + const completed = await tracker.refresh(); + expect(tokenUsage(completed.usage)?.input_tokens).toBe(1_100); + expect(costUpdates.at(-1)?.estimatedUsd).toBeCloseTo( + currentModel === "gpt-5.6-sol" ? 0.0044 : 0.0004, + 10, + ); + } finally { + await tracker.stop(); + } + }, + ); + + test.each([null, "synthetic-unpriced-model"])( + "reports an internal priced lower bound with model %p without inventing a total", + async (unknownModel) => { + const home = await codexHome(); + const at = "2026-09-01T00:00:02Z"; + const known = await writeSession(home, "owner", {}); + await appendFile( + known, + JSON.stringify({ + type: "token_usage_record", + timestamp: at, + payload: { + thread_id: "owner", + turn_id: "turn", + response_id: "known-response", + model: "gpt-5.6-sol", + usage: { input_tokens: 1_000, output_tokens: 0 }, + }, + }) + "\n", + ); + const unknown = await writeSession(home, "worker", {}); + await appendFile( + unknown, + JSON.stringify({ + type: "turn_context", + timestamp: at, + payload: { + turn_id: "worker-turn", + ...(unknownModel === null ? {} : { model: unknownModel }), + }, + }) + + "\n" + + JSON.stringify({ + type: "event_msg", + timestamp: at, + payload: { + type: "token_count", + info: { + total_token_usage: { input_tokens: 100, output_tokens: 0 }, + }, + }, + }) + + "\n", + ); + const lowerBounds: Readonly[] = []; + const publicCosts: Readonly[] = []; + const options = { + codexHome: home, + model: "gpt-5.6-sol", + maxCostUsd: 0.003, + onCost: (cost: Readonly) => publicCosts.push(cost), + onCostLowerBound: (cost: Readonly) => lowerBounds.push(cost), + }; + const tracker = new ScanCostTracker(options); + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + executionThreadIds: ["worker"], + owner: { threadId: "owner", turnId: "turn", startedAt: at }, + startedAt: at, + completedAt: null, + })); + tracker.start("owner"); + try { + const snapshot = await tracker.refresh(); + expect(tokenUsage(snapshot.usage)?.input_tokens).toBe(1_100); + expect(snapshot.cost).toBeNull(); + expect(publicCosts).toEqual([]); + expect(lowerBounds).toHaveLength(1); + expect(lowerBounds[0]).toMatchObject({ + inputTokens: 1_000, + estimatedUsd: 0.004, + coverage: "partial", + }); + expect(lowerBounds[0]!.estimatedUsd).toBeGreaterThan( + options.maxCostUsd, + ); + await tracker.refresh(); + expect(lowerBounds).toHaveLength(1); + } finally { + await tracker.stop(); + } + }, + ); + + test.each(["identical", "prefix-first", "prefix-last"] as const)( + "forwards each event occurrence once from copied logs: %s", + async (copy) => { + const home = await codexHome(); + const recordedHome = await codexHome(); + const scanDirectory = join(home, "scan"); + const settings = join(scanDirectory, "artifacts", "deep_discovery"); + await mkdir(settings, { recursive: true }); + await writeFile( + join(settings, "execution-settings.json"), + JSON.stringify({ + version: 1, + settings: { codexHome: recordedHome }, + }), + ); + await mkdir(join(home, "sessions")); + await mkdir(join(recordedHome, "sessions")); + const first = join(home, "sessions", "worker.jsonl"); + const second = join(recordedHome, "sessions", "worker-copy.jsonl"); + const repeated = { + timestamp: "2026-09-01T00:00:02Z", + type: "event_msg", + payload: { type: "agent_message", message: "Reviewing source." }, + }; + const expected = [ + { + timestamp: "2026-09-01T00:00:00Z", + type: "session_meta", + payload: { id: "worker", model: "gpt-5.6-sol" }, + }, + repeated, + repeated, + { + timestamp: "2026-09-01T00:00:03Z", + type: "token_usage_record", + payload: { + thread_id: "worker", + turn_id: "turn", + response_id: "response", + model: "gpt-5.6-sol", + usage: { input_tokens: 100, output_tokens: 0 }, + }, + }, + ]; + const contents = expected.map((event) => JSON.stringify(event) + "\n"); + await writeFile( + first, + contents.slice(0, copy === "prefix-first" ? 2 : 4).join(""), + ); + await writeFile( + second, + contents.slice(0, copy === "prefix-last" ? 2 : 4).join(""), + ); + const events: ScanSessionEvent[] = []; + const options = { + codexHome: home, + scanDirectory, + model: "gpt-5.6-sol", + onSessionEvent: (event: ScanSessionEvent) => events.push(event), + }; + const recordedOwner = async () => ({ + formatVersion: 1 as const, + legacy: true as const, + workerCodexHome: recordedHome, + executionThreadIds: [], + owner: { + threadId: "worker", + turnId: null, + startedAt: "2026-09-01T00:00:00Z", + }, + startedAt: "2026-09-01T00:00:00Z", + completedAt: null, + }); + const tracker = new ScanCostTracker(options); + tracker.setAttributionReader(recordedOwner); + tracker.start("worker"); + try { + expect((await tracker.refresh()).cost?.inputTokens).toBe(100); + expect(events.map((event) => event.event)).toEqual(expected); + await tracker.refresh(); + expect(events).toHaveLength(expected.length); + // Both logs catch up, then a genuine repeated occurrence is copied later. + await writeFile(first, contents.join("")); + await writeFile(second, contents.join("")); + await appendFile(first, JSON.stringify(repeated) + "\n"); + await tracker.refresh(); + expect(events.map((event) => event.event)).toEqual([ + ...expected, + repeated, + ]); + await appendFile(second, JSON.stringify(repeated) + "\n"); + await tracker.stop(); + expect(events.map((event) => event.event)).toEqual([ + ...expected, + repeated, + ]); + // Re-reading after the owner interval becomes available filters early + // events, but must not renumber the surviving source occurrences. + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + workerCodexHome: recordedHome, + executionThreadIds: ["worker"], + owner: { + threadId: "worker", + turnId: "turn", + startedAt: "2026-09-01T00:00:03Z", + }, + startedAt: "2026-09-01T00:00:03Z", + completedAt: "2026-09-01T00:00:04Z", + })); + await tracker.refresh(); + expect(events.map((event) => event.event)).toEqual([ + ...expected, + repeated, + ]); + events.length = 0; + const reconstructed = new ScanCostTracker(options); + reconstructed.setAttributionReader(recordedOwner); + reconstructed.start("worker"); + await reconstructed.stop(); + expect(events.map((event) => event.event)).toEqual([ + ...expected, + repeated, + ]); + } finally { + await tracker.stop(); + } + }, + ); + + test("keeps resumed worker usage and current parent usage isolated per scan", async () => { + const currentHome = await codexHome(); + const firstHome = await codexHome(); + const secondHome = await codexHome(); + const at = "2026-09-01T00:00:02Z"; + const trackers: ScanCostTracker[] = []; + const fixture = async (home: string, id: string, count: number) => { + const path = await writeSession(home, id, {}); + await appendFile( + path, + [ + { + type: "turn_context", + timestamp: at, + payload: { turn_id: "scan-turn", model: "gpt-5.6-sol" }, + }, + { + type: "token_usage_record", + timestamp: at, + payload: { + thread_id: id, + turn_id: "scan-turn", + response_id: `${id}-response`, + model: "gpt-5.6-sol", + usage: { input_tokens: count, output_tokens: 0 }, + }, + }, + ] + .map((event) => JSON.stringify(event)) + .join("\n") + "\n", + ); + return path; + }; + try { + const cases = [ + { id: "one", home: firstHome, parent: 10, discovery: 20, reducer: 30 }, + { id: "two", home: secondHome, parent: 11, discovery: 21, reducer: 31 }, + ]; + for (const row of cases) { + const scanDirectory = join(currentHome, "scans", row.id); + const settingsDirectory = join( + scanDirectory, + "artifacts", + "deep_discovery", + ); + await mkdir(settingsDirectory, { recursive: true }); + await writeFile( + join(settingsDirectory, "execution-settings.json"), + JSON.stringify({ + version: 1, + settings: { codexHome: row.home }, + }), + ); + await fixture(currentHome, `${row.id}-parent`, row.parent); + const discovery = await fixture( + row.home, + `${row.id}-discovery`, + row.discovery, + ); + await fixture(row.home, `${row.id}-reducer`, row.reducer); + await fixture(row.home, `${row.id}-unrelated`, 10_000); + // Repeated receipt identity after reconnect must remain one charge. + const duplicate = (await readFile(discovery, "utf8")) + .trim() + .split("\n") + .at(-1)!; + await appendFile(discovery, duplicate + "\n"); + const attribution = { + formatVersion: 1 as const, + workerCodexHome: row.home, + executionThreadIds: [`${row.id}-discovery`, `${row.id}-reducer`], + owner: { + threadId: `${row.id}-parent`, + turnId: "scan-turn", + startedAt: at, + }, + startedAt: at, + completedAt: null, + }; + const tracker = new ScanCostTracker({ + codexHome: currentHome, + scanDirectory, + model: "gpt-5.6-sol", + maxCostUsd: 0.0002, + }); + tracker.setAttributionReader(async () => attribution); + tracker.start(`${row.id}-parent`); + trackers.push(tracker); + } + const initial = await Promise.all( + trackers.map((tracker) => tracker.refresh()), + ); + expect( + initial.map((snapshot) => tokenUsage(snapshot.usage)?.input_tokens), + ).toEqual([60, 63]); + expect(initial.map((snapshot) => snapshot.cost?.inputTokens)).toEqual([ + 60, 63, + ]); + expect(initial[0]!.cost!.estimatedUsd).toBeGreaterThan(0.0002); + const firstDirectory = join(currentHome, "scans", "one"); + await writeFile( + join( + firstDirectory, + "artifacts", + "deep_discovery", + "execution-settings.json", + ), + JSON.stringify({ version: 1, settings: { codexHome: secondHome } }), + ); + await unlink( + join( + currentHome, + "scans", + "two", + "artifacts", + "deep_discovery", + "execution-settings.json", + ), + ); + expect( + (await Promise.all(trackers.map((tracker) => tracker.refresh()))).map( + (snapshot) => snapshot.cost?.inputTokens, + ), + ).toEqual([60, 63]); + const rebuilt = new ScanCostTracker({ + codexHome: currentHome, + scanDirectory: firstDirectory, + model: "gpt-5.6-sol", + maxCostUsd: 0.0002, + }); + rebuilt.setAttributionReader(async () => ({ + formatVersion: 1, + workerCodexHome: firstHome, + executionThreadIds: [ + "one-discovery", + "one-reducer", + "one-missing-attempt", + ], + owner: { threadId: "one-parent", turnId: "scan-turn", startedAt: at }, + startedAt: at, + completedAt: null, + })); + rebuilt.start("one-parent"); + trackers.push(rebuilt); + expect((await rebuilt.refresh()).usage).toMatchObject({ + input_tokens: 60, + coverage: "partial", + }); + expect((await rebuilt.refresh()).cost!.estimatedUsd).toBeGreaterThan( + 0.0002, + ); + await fixture(firstHome, "one-missing-attempt", 7); + expect((await rebuilt.refresh()).cost?.inputTokens).toBe(67); + expect((await trackers[1]!.refresh()).cost?.inputTokens).toBe(63); + } finally { + await Promise.all(trackers.map((tracker) => tracker.stop())); + } + }); + + test("reads a recorded directory alias only once", async () => { + const home = await codexHome(); + const alias = join(await codexHome(), "recorded-home"); + await symlink( + home, + alias, + process.platform === "win32" ? "junction" : "dir", + ); + const scanDirectory = join(home, "scan"); + const directory = join(scanDirectory, "artifacts", "deep_discovery"); + await mkdir(directory, { recursive: true }); + await writeFile( + join(directory, "execution-settings.json"), + JSON.stringify({ version: 1, settings: { codexHome: alias } }), + ); + await writeSession(home, "worker", { input_tokens: 100, output_tokens: 0 }); + const events: ScanSessionEvent[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + scanDirectory, + model: "gpt-5.6-sol", + onSessionEvent: (event) => events.push(event), + }); + tracker.start("worker"); + try { + expect((await tracker.stop()).cost?.inputTokens).toBe(100); + expect(events).toHaveLength(2); + } finally { + await tracker.stop(); + } + }); + + test.each([ + ["identical", false], + ["identical", true], + ["prefix-first", false], + ["prefix-first", true], + ["prefix-last", false], + ["prefix-last", true], + ["truncated-first", false], + ["truncated-first", true], + ["truncated-last", false], + ["truncated-last", true], + ["tail-first", false], + ["tail-first", true], + ["tail-last", false], + ["tail-last", true], + ] as const)( + "prices copied response records (%s, attribution: %s)", + async (copy, attributed) => { + const home = await codexHome(); + const recordedHome = await codexHome(); + const scanDirectory = join(home, "scan"); + const directory = join(scanDirectory, "artifacts", "deep_discovery"); + await mkdir(directory, { recursive: true }); + await writeFile( + join(directory, "execution-settings.json"), + JSON.stringify({ version: 1, settings: { codexHome: recordedHome } }), + ); + const path = await writeSession(home, "worker", {}); + for (const [id, model, input, output] of [ + ["response-one", "gpt-5.6-sol", 100, 10], + ["response-two", "gpt-6-astra", 50, 5], + ] as const) { + await appendFile( + path, + JSON.stringify({ + type: "token_usage_record", + timestamp: "2026-09-01T00:00:02Z", + payload: { + thread_id: "worker", + turn_id: "turn", + response_id: id, + model, + usage: { input_tokens: input, output_tokens: output }, + }, + }) + "\n", + ); + } + await mkdir(join(recordedHome, "sessions")); + const copiedPath = join(recordedHome, "sessions", "copied-worker.jsonl"); + await cp(path, copiedPath); + const prefix = + (await readFile(path, "utf8")) + .trimEnd() + .split("\n") + .slice(0, -1) + .join("\n") + "\n"; + if (copy === "prefix-first") await writeFile(path, prefix); + if (copy === "prefix-last") await writeFile(copiedPath, prefix); + const truncated = prefix + '{"type":"token_usage_record"'; + if (copy === "truncated-first") await writeFile(path, truncated); + if (copy === "truncated-last") await writeFile(copiedPath, truncated); + if (copy === "tail-first") await appendFile(path, '{"type":"event_msg"'); + if (copy === "tail-last") + await appendFile(copiedPath, '{"type":"event_msg"'); + const tracker = new ScanCostTracker({ + codexHome: home, + scanDirectory, + model: "gpt-5.6-sol", + }); + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + ...(attributed ? {} : { legacy: true as const }), + workerCodexHome: recordedHome, + executionThreadIds: ["worker"], + owner: { + threadId: null, + turnId: null, + startedAt: "2026-09-01T00:00:00Z", + }, + startedAt: "2026-09-01T00:00:00Z", + completedAt: null, + })); + tracker.start("worker"); + try { + const snapshot = await tracker.stop(); + expect(snapshot.usage).toMatchObject({ + input_tokens: 150, + output_tokens: 15, + total_tokens: 165, + }); + expect(snapshot.usage).not.toMatchObject({ coverage: "partial" }); + expect( + Object.fromEntries( + snapshot.cost!.modelCosts!.map((part) => [ + part.model, + [part.inputTokens, part.outputTokens], + ]), + ), + ).toEqual({ + "gpt-5.6-sol": [100, 10], + "gpt-6-astra": [50, 5], + }); + } finally { + await tracker.stop(); + } + }, + ); +}); + +test.each([ + [150_000, 15_000, 100_000, 200_000], + [150_000, 15_000, 100_000, 20_000], + [100_000, 50_000, 110_000, 10_000], + [100_000, 50_000, 200_000, 10_000], + [100_000, 10_000, 100_000, 10_000], +] as const)( + "preserves receipt pricing across divergent counters %s/%s vs %s/%s", + async (input, output, counterInput, counterOutput) => { + const home = await codexHome(); + const path = await writeSession(home, "worker", {}); + const counts = (input: number, output: number) => ({ + input_tokens: input, + output_tokens: output, + cached_input_tokens: 0, + cache_write_input_tokens: 0, + reasoning_output_tokens: 0, + total_tokens: input + output, + }); + await appendFile( + path, + JSON.stringify({ + type: "token_usage_record", + timestamp: "2026-09-01T00:00:02Z", + payload: { + thread_id: "worker", + turn_id: "turn", + response_id: "receipt", + model: "gpt-5.6-sol", + usage: counts(input, output), + thread_token_usage: counts(counterInput, counterOutput), + }, + }) + + "\n" + + JSON.stringify({ + type: "event_msg", + timestamp: "2026-09-01T00:00:02Z", + payload: { + type: "token_count", + info: { total_token_usage: counts(counterInput, counterOutput) }, + }, + }) + + "\n", + ); + const lowerBounds: Readonly[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + maxCostUsd: 0.8, + onCost: (cost) => lowerBounds.push(cost), + onCostLowerBound: (cost) => lowerBounds.push(cost), + }); + tracker.start("worker"); + try { + const snapshot = await tracker.stop(); + expect(snapshot.usage).toMatchObject({ + input_tokens: Math.max(input, counterInput), + output_tokens: Math.max(output, counterOutput), + }); + if (counterInput > input || counterOutput > output) + expect(snapshot.usage).toMatchObject({ coverage: "partial" }); + else expect(snapshot.usage).not.toMatchObject({ coverage: "partial" }); + const measured = tokenUsage(snapshot.usage); + expect(measured).not.toBeNull(); + expect(lowerBounds.length).toBeGreaterThan(0); + expect(lowerBounds.at(-1)!.estimatedUsd).toBeGreaterThanOrEqual( + estimateScanCost("gpt-5.6-sol", counts(input, output))!.estimatedUsd, + ); + } finally { + await tracker.stop(); + } + }, +); + +test.each(["standard", "pending-deep", "legacy-deep", "bound-deep"] as const)( + "uses bound log homes rather than draft settings: %s", + async (kind) => { + const home = await codexHome(); + const foreign = await codexHome(); + const scanDirectory = join(home, "scan"); + const settings = join(scanDirectory, "artifacts", "deep_discovery"); + await mkdir(settings, { recursive: true }); + await writeFile( + join(settings, "execution-settings.json"), + JSON.stringify({ version: 1, settings: { codexHome: foreign } }), + ); + for (const [directory, input] of [ + [home, 100], + [foreign, 900_000], + ] as const) { + const path = await writeSession( + directory, + "scan-thread", + { + input_tokens: input, + output_tokens: 0, + cached_input_tokens: 0, + cache_write_input_tokens: 0, + }, + { timestamp: "2026-09-01T00:00:02Z" }, + ); + const lines = (await readFile(path, "utf8")) + .trimEnd() + .split("\n") + .map((line) => ({ + ...JSON.parse(line), + timestamp: "2026-09-01T00:00:02Z", + })); + await writeFile(path, jsonLines(lines) + "\n"); + } + const tracker = new ScanCostTracker({ + codexHome: home, + scanDirectory, + model: "gpt-5.6-sol", + }); + if (kind !== "standard") + tracker.setAttributionReader(async () => + kind === "pending-deep" + ? null + : { + formatVersion: 1, + ...(kind === "legacy-deep" + ? { legacy: true as const } + : { workerCodexHome: foreign }), + executionThreadIds: ["scan-thread"], + owner: { + threadId: null, + turnId: null, + startedAt: "2026-09-01T00:00:00Z", + }, + startedAt: "2026-09-01T00:00:00Z", + completedAt: null, + }, + ); + tracker.start("scan-thread"); + try { + expect((await tracker.stop()).usage).toMatchObject({ + input_tokens: kind === "bound-deep" ? 900_000 : 100, + }); + } finally { + await tracker.stop(); + } + }, +); + +test.each(["no-reader", "pending-reader"] as const)( + "enforces dedicated parent cost before orchestration: %s", + async (kind) => { + const home = await codexHome(); + await writeSession(home, "new-parent", { + input_tokens: 100_000, + output_tokens: 0, + cached_input_tokens: 0, + cache_write_input_tokens: 0, + }); + const costs: Readonly[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + maxCostUsd: 0.003, + onCost: (cost) => costs.push(cost), + onCostLowerBound: (cost) => costs.push(cost), + }); + if (kind === "pending-reader") + tracker.setAttributionReader(async () => null); + tracker.start("new-parent"); + try { + const snapshot = await tracker.stop(); + expect(snapshot.cost?.estimatedUsd).toBeGreaterThan(0.003); + expect(costs.length).toBeGreaterThan(0); + } finally { + await tracker.stop(); + } + }, +); + +test.each(["single", "identical", "prefix-first", "prefix-last"] as const)( + "forwards copied worker activities without replaying completed calls: %s", + async (copy) => { + const { home, worker } = await workerSessionFixture(); + const recordedHome = await codexHome(); + await mkdir(join(recordedHome, "sessions")); + const second = join(recordedHome, "sessions", "worker-copy.jsonl"); + const call = (id: string) => ({ + type: "response_item", + payload: { + type: "function_call", + name: "exec_command", + call_id: id, + arguments: JSON.stringify({ cmd: "rg -n source routes/login.ts" }), + }, + }); + const done = (id: string) => ({ + type: "response_item", + payload: { type: "function_call_output", call_id: id }, + }); + const prefix = + (await readFile(worker, "utf8")) + jsonLines([call("first")]) + "\n"; + const complete = prefix + jsonLines([done("first")]) + "\n"; + await writeFile(worker, copy === "prefix-first" ? prefix : complete); + if (copy !== "single") + await writeFile(second, copy === "prefix-last" ? prefix : complete); + const activities: ScanActivity[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + scanDirectory: join(home, "scan"), + model: "gpt-5.6-sol", + repository: "/code/juice-shop", + onActivity: (activity) => activities.push(activity), + }); + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + legacy: true, + workerCodexHome: recordedHome, + executionThreadIds: [], + owner: { + threadId: "scan-thread", + turnId: null, + startedAt: "2026-09-01T00:00:00Z", + }, + startedAt: "2026-09-01T00:00:00Z", + completedAt: null, + })); + tracker.start("scan-thread"); + try { + await tracker.refresh(); + expect(activities.map(({ id, status }) => [id, status])).toEqual([ + ["worker-thread:first", "running"], + ["worker-thread:first", "completed"], + ]); + await writeFile(worker, complete); + if (copy !== "single") await writeFile(second, complete); + await tracker.refresh(); + expect(activities).toHaveLength(2); + const later = jsonLines([call("second"), done("second")]) + "\n"; + await appendFile(worker, later); + await tracker.refresh(); + if (copy !== "single") await appendFile(second, later); + await tracker.stop(); + expect(activities.map(({ id, status }) => [id, status])).toEqual([ + ["worker-thread:first", "running"], + ["worker-thread:first", "completed"], + ["worker-thread:second", "running"], + ["worker-thread:second", "completed"], + ]); + } finally { + await tracker.stop(); + } + }, +); + +test.each([0, 100])( + "ignores only zero unpriced counter buckets: %i", + async (unpricedTokens) => { + const home = await codexHome(); + const path = await writeSession(home, "owner", { + input_tokens: unpricedTokens, + output_tokens: 0, + }); + await appendFile( + path, + jsonLines([ + { + type: "turn_context", + timestamp: "2026-09-01T00:00:02Z", + payload: { turn_id: "turn", model: "gpt-5.6-sol" }, + }, + { + type: "event_msg", + timestamp: "2026-09-01T00:00:02Z", + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: unpricedTokens + 1000, + output_tokens: 0, + }, + }, + }, + }, + ]) + "\n", + ); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + tracker.start("owner"); + try { + const snapshot = await tracker.refresh(); + expect(tokenUsage(snapshot.usage)?.input_tokens).toBe( + unpricedTokens + 1000, + ); + if (unpricedTokens === 0) + expect(snapshot.cost?.estimatedUsd).toBeCloseTo(0.004, 10); + else expect(snapshot.cost).toBeNull(); + } finally { + await tracker.stop(); + } + }, +); + +test.each(["other turn", "outside window"])( + "keeps owned legacy model before an unrelated first receipt: %s", + async (kind) => { + const home = await codexHome(); + const path = await writeSession(home, "owner", {}); + const at = "2026-09-01T00:00:02Z"; + await appendFile( + path, + jsonLines([ + { + type: "turn_context", + timestamp: at, + payload: { turn_id: "turn", model: "gpt-5.6-sol" }, + }, + { + type: "event_msg", + timestamp: at, + payload: { + type: "token_count", + info: { + total_token_usage: { input_tokens: 1000, output_tokens: 0 }, + }, + }, + }, + { + type: "token_usage_record", + timestamp: kind === "outside window" ? "2026-09-01T00:00:04Z" : at, + payload: { + thread_id: "owner", + turn_id: kind === "other turn" ? "other" : "turn", + response_id: "unrelated-response", + model: "gpt-6-astra", + usage: { input_tokens: 50, output_tokens: 0 }, + }, + }, + ]) + "\n", + ); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + executionThreadIds: [], + owner: { threadId: "owner", turnId: "turn", startedAt: at }, + startedAt: at, + completedAt: "2026-09-01T00:00:03Z", + })); + tracker.start("owner"); + try { + const snapshot = await tracker.refresh(); + expect(tokenUsage(snapshot.usage)?.input_tokens).toBe(1000); + expect(snapshot.cost?.estimatedUsd).toBeCloseTo(0.004, 10); + expect(snapshot.usage).not.toMatchObject({ coverage: "partial" }); + } finally { + await tracker.stop(); + } + }, +); diff --git a/sdk/typescript/tests-ts/deep-finalization.test.ts b/sdk/typescript/tests-ts/deep-finalization.test.ts new file mode 100644 index 0000000000..9588a3a5f1 --- /dev/null +++ b/sdk/typescript/tests-ts/deep-finalization.test.ts @@ -0,0 +1,1167 @@ +import { execFileSync, spawn } from "node:child_process"; +import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { afterEach, expect, test } from "bun:test"; +import type { ThreadEvent } from "@openai/codex-sdk"; +import { + ScanCostLimitExceededError, + ScanInterruptedError, +} from "../src/errors.js"; +import { + prepareScanArtifactRestorer, + runWorkbench, + type WorkbenchCommandOptions, +} from "../src/runtime.js"; +import { resumeSelectedDeepScan } from "../src/deep-scan-finalization.js"; +import { TestClient } from "./support/api-client.js"; +import { + completedEvents, + createApiTestFixtures, + preparedRuntime, +} from "./support/api-events.js"; +import { PLUGIN_ROOT } from "./plugin-root.js"; + +const { temporaryDirectory, cleanup } = createApiTestFixtures(); +afterEach(cleanup); +const threadId = "1af317a1-c9ed-4c73-b428-cb0d160cf8e8"; +const followUp = "Explain the selected finding."; + +for (const boundary of ["registration", "stream-start"] as const) { + test(`SDK cancels registered Deep Scan before first thread event: ${boundary}`, async () => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const scanDir = join(root, "scan"); + const codexHome = join(root, "codex-home"); + await Promise.all([ + mkdir(repository), + mkdir(scanDir, { mode: 0o700 }), + mkdir(codexHome), + ]); + await writeFile(join(repository, "source.py"), "# Synthetic source\n"); + const environment = { + ...process.env, + CODEX_HOME: codexHome, + CODEX_SECURITY_STATE_DIR: join(root, "state"), + }; + const cancellation = new AbortController(); + const reason = new Error( + "Synthetic cancellation before first thread event", + ); + const commands: string[][] = []; + let scanId = ""; + let savedOptions: WorkbenchCommandOptions; + let startedTurns = 0; + const client = new TestClient( + {}, + { + environment, + prepareRuntime: async () => ({ + ...preparedRuntime(codexHome), + environment, + }), + resolvePluginPython: async () => "python3", + prepareOutputDir: async () => scanDir, + runWorkbench: async (options, args, input) => { + savedOptions = options; + commands.push([...args]); + const result = await runWorkbench(options, args, input); + if (args[0] === "register-cli-scan") { + scanId = result["scanId"] as string; + if (boundary === "registration") cancellation.abort(reason); + } + return result; + }, + createCodex: () => ({ + startThread: () => ({ + id: null, + runStreamed: async () => { + startedTurns++; + cancellation.abort(reason); + throw reason; + }, + }), + }), + }, + ); + try { + const error = await client + .run(repository, { + mode: "deep", + signal: cancellation.signal, + postScanPrompt: followUp, + }) + .catch((error: unknown) => error); + expect(error).toBeInstanceOf(ScanInterruptedError); + expect((error as ScanInterruptedError).cause).toBe(reason); + const stopped = await runWorkbench( + { ...savedOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + expect(stopped["scan"]).toMatchObject({ + progress: { status: "canceled" }, + }); + expect(commands.filter((args) => args[0] === "cancel-scan")).toEqual([ + ["cancel-scan", "--scan-id", scanId], + ]); + expect(commands.some((args) => args[0] === "fail-scan")).toBe(false); + expect(commands.some((args) => args[0] === "set-scan-thread")).toBe( + false, + ); + expect(startedTurns).toBe(boundary === "registration" ? 0 : 1); + } finally { + await client.close(); + } + }, 30_000); +} + +const outcomes = [ + "failed", + "completed", + "completed-owner-suffix", + "completed-owner-explicit", + "restart", + "canceled-before-publication", + "canceled-during-publication", + "canceled-during-resumed-publication", + "published-before-cancellation", + "closed-during-publication", + "budget-during-publication", + "budget-after-deep-finish", + "budget-during-resumed-publication", + "closed-during-resumed-publication", + "lost-completion-response", + "lost-completion-response-followup-canceled", + "completion-before-commit-fails", + "followup-canceled", +] as const; +type BudgetCompletionFault = "lost" | "before-commit" | "lost-and-canceled"; +const cases: { + outcome: (typeof outcomes)[number]; + budgetCompletionFault?: BudgetCompletionFault; + initialResumeUsage?: boolean; + unpricedUsage?: boolean; + cancellationFault?: "status-read" | "deep-state-read" | "cancel-response"; +}[] = [ + ...outcomes.map((outcome) => ({ outcome })), + ...( + [ + "budget-during-publication", + "budget-after-deep-finish", + "budget-during-resumed-publication", + ] as const + ).flatMap((outcome) => [ + { outcome, unpricedUsage: true }, + { outcome, unpricedUsage: true, budgetCompletionFault: "lost" as const }, + ]), + ...( + [ + "budget-during-publication", + "budget-after-deep-finish", + "budget-during-resumed-publication", + ] as const + ).map((outcome) => ({ outcome, budgetCompletionFault: "lost" as const })), + { + outcome: "budget-after-deep-finish", + budgetCompletionFault: "before-commit", + }, + { + outcome: "budget-after-deep-finish", + budgetCompletionFault: "lost-and-canceled", + }, + { + outcome: "canceled-during-publication", + cancellationFault: "status-read", + }, + { + outcome: "canceled-during-publication", + cancellationFault: "cancel-response", + }, + { + outcome: "canceled-during-publication", + cancellationFault: "deep-state-read", + }, + { + outcome: "canceled-during-resumed-publication", + cancellationFault: "deep-state-read", + }, + { + outcome: "canceled-before-publication", + cancellationFault: "deep-state-read", + }, + { + outcome: "canceled-during-resumed-publication", + initialResumeUsage: true, + }, +]; +for (const { + outcome, + budgetCompletionFault, + cancellationFault, + initialResumeUsage, + unpricedUsage, +} of cases) { + const resumedStop = outcome.includes("-resumed-"); + const restart = outcome === "restart" || resumedStop; + const closed = outcome.startsWith("closed-"); + const budgeted = outcome.startsWith("budget-"); + const canceledFollowUp = outcome.endsWith("followup-canceled"); + const loseCompletionResponse = outcome.startsWith("lost-completion-response"); + const name = + outcome === "followup-canceled" + ? "SDK preserves a selected aggregate when its follow-up is canceled" + : `SDK handles selected aggregate: ${outcome}${budgetCompletionFault ? ` (budget completion ${budgetCompletionFault})` : ""}${cancellationFault ? ` (cancellation ${cancellationFault})` : ""}${initialResumeUsage ? " (initial resume cost)" : ""}${unpricedUsage ? " (unpriced remainder)" : ""}`; + const runCase = async () => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const scanDir = join(root, "scan"); + const codexHome = join(root, "codex-home"); + const stateDir = join(root, "state"); + await Promise.all([ + mkdir(repository), + mkdir(scanDir, { mode: 0o700 }), + mkdir(codexHome), + ]); + await writeFile(join(repository, "extract.py"), "# Synthetic source\n"); + const environment = { + ...process.env, + CODEX_HOME: codexHome, + CODEX_SECURITY_STATE_DIR: stateDir, + }; + const cancellation = new AbortController(); + const parentError = new Error( + "Parent turn ended before its final completion tool call", + ); + let scanId = ""; + let workbenchOptions: WorkbenchCommandOptions; + let publicationFails = restart; + let completionReceiptLost = loseCompletionResponse; + let budgetReceiptLost = + budgetCompletionFault === "lost" || + budgetCompletionFault === "lost-and-canceled"; + let budgetTriggered = false; + let cancellationReadLost = false; + let lostCancellationDeepState: unknown; + const ownerSuffix = + outcome === "completed-owner-suffix" || + outcome === "completed-owner-explicit"; + let nativeOwnerStatus: unknown; + let originalFinalizationInput: unknown; + let selectedPath = ""; + let selectedBytes: Buffer; + let originalResumeSignal: AbortSignal | undefined; + let acceptedReport = ""; + let completedArtifacts: Buffer[] = []; + const modelInputs: string[] = []; + const commands: string[] = []; + const reportedCosts: number[] = []; + const usagePath = join( + codexHome, + "sessions", + "2026", + "01", + "01", + `rollout-${threadId}.jsonl`, + ); + const recordBudgetUsage = ( + inputTokens = 1_250, + outputTokens = 30, + timestamp = new Date().toISOString(), + ) => + appendFile( + usagePath, + JSON.stringify({ + timestamp, + type: "turn_context", + payload: { + turn_id: "synthetic-scan-turn", + model: "gpt-5.6-sol", + }, + }) + + "\n" + + JSON.stringify({ + timestamp, + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: inputTokens, + cached_input_tokens: 200, + output_tokens: outputTokens, + }, + }, + }, + }) + + "\n" + + (unpricedUsage + ? [ + JSON.stringify({ + timestamp, + type: "turn_context", + payload: { + turn_id: "synthetic-scan-turn", + model: "synthetic-unpriced-model", + }, + }), + JSON.stringify({ + timestamp, + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: 1_350, + cached_input_tokens: 200, + output_tokens: outputTokens, + }, + }, + }, + }), + "", + ].join("\n") + : ""), + ); + let closePromise: Promise | undefined; + const makeClient = () => + new TestClient( + {}, + { + environment, + prepareScanArtifactRestorer, + prepareRuntime: async () => { + const runtime = preparedRuntime(codexHome); + const manifest = JSON.parse( + await readFile( + join(PLUGIN_ROOT, ".codex-plugin/plugin.json"), + "utf8", + ), + ); + return { + ...runtime, + environment, + persistentCredentialHome: true, + plugin: { ...runtime.plugin, version: manifest.version }, + }; + }, + resolvePluginPython: async () => "python3", + prepareOutputDir: async () => scanDir, + runWorkbench: async (options, args, input) => { + workbenchOptions = options; + if (args[0] === "get-cli-scan-resume") + originalResumeSignal = options.signal; + commands.push(args[0]!); + if ( + args[0] === "get-scan" && + cancellation.signal.aborted && + cancellationFault === "status-read" + ) { + throw new Error("Synthetic lost cancellation status response"); + } + if (args[0] === "write-scan-draft" && publicationFails) { + publicationFails = false; + throw new Error("Synthetic publication write failure"); + } + if ( + args[0] === "complete-scan" && + outcome === "completion-before-commit-fails" + ) + throw new Error("Synthetic completion failure before commit"); + if ( + args[0] === "complete-budget-exhausted-scan" && + budgetCompletionFault === "before-commit" + ) + throw new Error( + "Synthetic budget completion failure before commit", + ); + const result = await runWorkbench(options, args, input); + if ( + args[0] === "get-deep-scan" && + cancellation.signal.aborted && + cancellationFault === "deep-state-read" && + !cancellationReadLost + ) { + cancellationReadLost = true; + lostCancellationDeepState = result["deepScan"]; + throw new Error( + "Synthetic lost cancellation Deep state response", + ); + } + if ( + args[0] === "cancel-scan" && + cancellationFault === "cancel-response" + ) { + completedArtifacts = await Promise.all( + ["report.md", "scan-manifest.json"].map((name) => + readFile(join(scanDir, name)), + ), + ); + throw new Error("Synthetic lost cancellation response"); + } + if ( + args[0] === "complete-budget-exhausted-scan" && + budgetReceiptLost + ) { + budgetReceiptLost = false; + if (budgetCompletionFault === "lost-and-canceled") { + completedArtifacts = await Promise.all( + ["report.md", "scan-manifest.json"].map((name) => + readFile(join(scanDir, name)), + ), + ); + cancellation.abort( + "Synthetic cancellation after budget completion", + ); + } + throw Object.assign( + new Error("Synthetic lost budget completion response"), + { code: "ETIMEDOUT" }, + ); + } + if (args[0] === "complete-scan" && completionReceiptLost) { + completionReceiptLost = false; + throw new Error("Synthetic lost completion response"); + } + if ( + !budgetTriggered && + ((args[0] === "write-scan-draft" && + (outcome === "budget-during-publication" || + outcome === "budget-during-resumed-publication")) || + (args[0] === "finish-deep-scan" && + outcome === "budget-after-deep-finish")) + ) { + budgetTriggered = true; + await recordBudgetUsage(); + await new Promise((resolve) => { + if (options.signal?.aborted) resolve(); + else + options.signal!.addEventListener("abort", () => resolve(), { + once: true, + }); + }); + } + if (args[0] === "write-scan-draft" && closed) { + closePromise = client.close(); + } + if ( + args[0] === "write-scan-draft" && + (outcome === "canceled-during-publication" || + outcome === "canceled-during-resumed-publication") + ) { + const published = await runWorkbench(options, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + threadId, + ]); + const bound = (published["deepScan"] as Record)[ + "finalizationInput" + ]; + expect(bound).toMatchObject( + originalFinalizationInput as Record, + ); + expect(bound).toMatchObject({ + publicationSha256: expect.stringMatching(/^[a-f0-9]{64}$/u), + }); + originalFinalizationInput = bound; + if (initialResumeUsage) { + expect(originalResumeSignal?.reason).toBeInstanceOf( + ScanCostLimitExceededError, + ); + expect(options.signal?.aborted).toBe(false); + } + cancellation.abort("Synthetic user cancellation"); + } + if ( + args[0] === "complete-scan" && + outcome === "published-before-cancellation" + ) { + cancellation.abort( + "Synthetic user cancellation after completion", + ); + } + if (args[0] === "register-cli-scan") + scanId = result["scanId"] as string; + return result; + }, + createCodex: () => { + const thread = { + id: threadId, + async runStreamed(input: string) { + modelInputs.push(input); + if (input === followUp) { + if (canceledFollowUp) { + const reportPath = join(scanDir, "report.md"); + acceptedReport = await readFile(reportPath, "utf8"); + expect(acceptedReport).toContain( + "Validate the resolved destination", + ); + await writeFile( + reportPath, + "Incomplete follow-up report.\n", + ); + cancellation.abort( + "Synthetic cancellation during follow-up", + ); + } + return { events: completedEvents(threadId) }; + } + expect(modelInputs.length).toBe(1); + async function* events(): AsyncGenerator { + yield { type: "thread.started", thread_id: threadId }; + await runWorkbench(workbenchOptions, [ + "begin-deep-scan", + "--scan-id", + scanId, + "--thread-id", + threadId, + ]); + const draft = { + scanId, + complete: true, + findings: [ + { + ruleId: "path-traversal.archive", + title: "Unsafe archive extraction", + summary: + "An untrusted entry reaches a filesystem write.", + severity: { level: "high" }, + confidence: { + level: "high", + rationale: "Source evidence.", + }, + taxonomy: { + category: "path-traversal", + cwe: ["CWE-22"], + }, + locations: [{ path: "extract.py", startLine: 1 }], + remediation: + "Validate the resolved destination before writing.", + provenance: { + source: "local_plugin", + candidateId: "archive-entry", + }, + }, + ], + coverage: { + completeness: "partial", + surfaces: [], + explicitExclusions: [], + deferred: [ + { + id: "dependency", + reason: "A dependency remains unreviewed.", + }, + ], + }, + }; + const seeded = JSON.parse( + execFileSync( + "python3", + [ + fileURLToPath( + new URL( + "./fixtures/selected-deep-scan.py", + import.meta.url, + ), + ), + ], + { + input: JSON.stringify({ + scanId, + scanDir, + database: join(stateDir, "workbench.sqlite3"), + draft, + }), + encoding: "utf8", + env: environment, + }, + ), + ); + // Exercise the dedicated function bridge, without extending CLI arguments. + const selectionOutput = execFileSync( + "python3", + [ + "-c", + "import runpy, sys; script = sys.argv.pop(1); runpy.run_path(script)['main'](select_finalization=True)", + join(PLUGIN_ROOT, "scripts/workbench_db.py"), + "finish-deep-scan", + "--scan-id", + scanId, + "--coordinator-generation", + "2", + "--terminal-reason", + loseCompletionResponse ? "capped" : "saturated", + "--manifest-path", + join(scanDir, "scan-manifest.json"), + ], + { + input: JSON.stringify({ + resultPath: seeded.resultPath, + }), + encoding: "utf8", + env: environment, + }, + ); + originalFinalizationInput = + JSON.parse(selectionOutput).deepScan.finalizationInput; + selectedPath = join( + scanDir, + (originalFinalizationInput as { resultPath: string }) + .resultPath, + ); + selectedBytes = await readFile(selectedPath); + const sessions = join( + codexHome, + "sessions", + "2026", + "01", + "01", + ); + await mkdir(sessions, { recursive: true }); + await writeFile( + join(sessions, `rollout-${threadId}.jsonl`), + JSON.stringify({ + timestamp: new Date().toISOString(), + type: "session_meta", + payload: { id: threadId, cwd: scanDir }, + }) + "\n", + ); + if (ownerSuffix) { + await recordBudgetUsage(); + await resumeSelectedDeepScan({ + scanId, + threadId, + pluginRoot: PLUGIN_ROOT, + runWorkbench: (args) => + runWorkbench(workbenchOptions, args), + signal: cancellation.signal, + }); + await rejoinSelectedScanThroughMcp( + environment, + scanId, + outcome === "completed-owner-explicit", + ); + const saved = await runWorkbench(workbenchOptions, [ + "get-scan", + "--scan-id", + scanId, + ]); + const scan = saved["scan"] as { + progress: { status: string }; + executionAttribution: { completedAt: string | null }; + }; + nativeOwnerStatus = scan.progress.status; + const completedAt = scan.executionAttribution.completedAt; + await recordBudgetUsage( + 2_500, + 60, + new Date( + Math.max( + Date.now(), + completedAt ? Date.parse(completedAt) + 1 : 0, + ), + ).toISOString(), + ); + } + if (outcome === "canceled-before-publication") + cancellation.abort("Synthetic user cancellation"); + if (outcome === "completed" || ownerSuffix) { + yield { + type: "turn.completed", + usage: { + input_tokens: ownerSuffix ? 2_500 : 0, + cached_input_tokens: ownerSuffix ? 200 : 0, + cache_write_input_tokens: 0, + reasoning_output_tokens: 0, + output_tokens: ownerSuffix ? 60 : 0, + }, + }; + } else { + throw parentError; + } + } + return { events: events() }; + }, + }; + return { + startThread: () => thread, + resumeThread: (id: string) => { + expect(id).toBe(threadId); + return thread; + }, + }; + }, + }, + ); + let client = makeClient(); + // Native usage polling is unref'ed; the transport double has no child process. + const keepAlive = setTimeout(() => {}, 30_000); + try { + if (restart) { + await expect( + client.run(repository, { + mode: "deep", + postScanPrompt: resumedStop ? undefined : followUp, + ...(budgeted ? { maxCostUsd: 0.004 } : {}), + }), + ).rejects.toThrow("Synthetic publication write failure"); + const pending = await runWorkbench(workbenchOptions!, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + threadId, + ]); + expect(pending["deepScan"]).toMatchObject({ + status: "running", + terminalReason: "saturated", + }); + expect(commands).not.toContain("fail-scan"); + await client.close(); + client = makeClient(); + if (initialResumeUsage) await recordBudgetUsage(); + } + if (budgeted || closed) { + const running = client.run(repository, { + mode: "deep", + signal: cancellation.signal, + ...(budgeted ? { maxCostUsd: 0.004 } : {}), + ...(resumedStop ? { resumeScanId: scanId, outputDir: scanDir } : {}), + ...(unpricedUsage + ? { onCost: (cost) => reportedCosts.push(cost.estimatedUsd) } + : {}), + postScanPrompt: followUp, + }); + if (budgeted) { + if ( + budgetCompletionFault === "before-commit" || + budgetCompletionFault === "lost-and-canceled" + ) { + await expect(running).rejects.toBeInstanceOf( + ScanCostLimitExceededError, + ); + expect( + commands.filter( + (command) => command === "complete-budget-exhausted-scan", + ), + ).toHaveLength(1); + expect(commands).not.toContain("complete-scan"); + expect(modelInputs).toHaveLength(1); + const saved = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + expect(saved["scan"]).toMatchObject({ + progress: { + status: + budgetCompletionFault === "before-commit" + ? "failed" + : "complete", + }, + findingCount: 1, + reportAvailable: true, + }); + if (budgetCompletionFault === "lost-and-canceled") { + expect( + await Promise.all( + ["report.md", "scan-manifest.json"].map((name) => + readFile(join(scanDir, name)), + ), + ), + ).toEqual(completedArtifacts); + } + return; + } + const result = await running; + expect(result.coverage.completeness).toBe("partial"); + expect(JSON.stringify(result.coverage)).toContain("cost limit"); + if (unpricedUsage) { + expect(result.cost).toBeNull(); + expect(reportedCosts).toEqual([]); + const saved = await runWorkbench(workbenchOptions!, [ + "get-scan", + "--scan-id", + scanId, + ]); + expect( + (saved["scan"] as { cost?: unknown }).cost ?? null, + ).toBeNull(); + } else expect(result.cost?.estimatedUsd).toBeGreaterThan(0.004); + expect(result.threadId).toBe(threadId); + expect(modelInputs).toHaveLength(1); + expect(commands).toContain("complete-budget-exhausted-scan"); + expect(commands).not.toContain("fail-scan"); + expect( + commands.filter( + (command) => command === "complete-budget-exhausted-scan", + ), + ).toHaveLength(1); + if (budgetCompletionFault === "lost") { + expect( + commands.filter((command) => command === "complete-scan"), + ).toHaveLength(1); + expect(result.findings.findings[0]?.remediation).toBe( + "Validate the resolved destination before writing.", + ); + const saved = await runWorkbench(workbenchOptions!, [ + "get-scan", + "--scan-id", + scanId, + ]); + expect(saved["scan"]).toMatchObject({ + progress: { status: "complete" }, + findingCount: 1, + reportAvailable: true, + }); + } + const completed = await runWorkbench(workbenchOptions!, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + threadId, + ]); + expect(completed["deepScan"]).toMatchObject({ + status: "succeeded", + finalizationInput: { terminalReason: "saturated" }, + }); + return; + } + await expect(running).rejects.toThrow(/closed/); + await closePromise; + const stopped = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + const deep = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-deep-scan", "--scan-id", scanId, "--thread-id", threadId], + ); + expect(stopped["scan"]).toMatchObject({ + progress: { status: "failed" }, + findingCount: 1, + reportAvailable: true, + }); + expect(deep["deepScan"]).toMatchObject({ + status: "failed", + finalizationInput: { terminalReason: "saturated" }, + }); + expect(await readFile(join(scanDir, "report.md"), "utf8")).toContain( + "Validate the resolved destination", + ); + expect( + JSON.parse(await readFile(join(scanDir, "coverage.json"), "utf8")) + .completeness, + ).toBe("partial"); + expect(commands).toContain("fail-scan"); + expect(modelInputs).toHaveLength(1); + await client.close(); + client = makeClient(); + await expect( + client.run(repository, { + mode: "deep", + resumeScanId: scanId, + outputDir: scanDir, + }), + ).rejects.toThrow(); + expect(modelInputs).toHaveLength(1); + return; + } + if (outcome.startsWith("canceled-")) { + const error = await client + .run(repository, { + mode: "deep", + signal: cancellation.signal, + ...(resumedStop + ? { resumeScanId: scanId, outputDir: scanDir } + : {}), + postScanPrompt: followUp, + ...(initialResumeUsage ? { maxCostUsd: 0.004 } : {}), + }) + .catch((error: unknown) => error); + if (initialResumeUsage) { + expect(error).toBeInstanceOf(ScanCostLimitExceededError); + expect(error).toBe(originalResumeSignal?.reason); + } else { + expect(error).toBeInstanceOf(ScanInterruptedError); + expect((error as ScanInterruptedError).cause).toBe( + outcome === "canceled-before-publication" + ? parentError + : cancellation.signal.reason, + ); + } + expect(await readFile(selectedPath)).toEqual(selectedBytes!); + if (cancellationReadLost) { + expect(lostCancellationDeepState).toMatchObject({ + status: "running", + finalizationInput: { terminalReason: "saturated" }, + }); + } + expect(originalFinalizationInput).toMatchObject({ + terminalReason: "saturated", + }); + const stopped = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + const deep = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-deep-scan", "--scan-id", scanId, "--thread-id", threadId], + ); + expect(stopped["scan"]).toMatchObject({ + progress: { status: "canceled" }, + findingCount: 1, + reportAvailable: true, + }); + expect( + JSON.parse(await readFile(join(scanDir, "coverage.json"), "utf8")) + .completeness, + ).toBe("partial"); + expect(await readFile(join(scanDir, "report.md"), "utf8")).toContain( + "Validate the resolved destination", + ); + expect(deep["deepScan"]).toMatchObject({ + status: "canceled", + }); + expect( + (deep["deepScan"] as Record)["finalizationInput"], + ).toEqual(originalFinalizationInput); + expect( + commands.filter((command) => command === "cancel-scan"), + ).toHaveLength(1); + expect(commands).not.toContain("fail-scan"); + expect(modelInputs.length).toBe(1); + if (cancellationFault === "cancel-response") { + expect( + await Promise.all( + ["report.md", "scan-manifest.json"].map((name) => + readFile(join(scanDir, name)), + ), + ), + ).toEqual(completedArtifacts); + } + await expect( + client.run(repository, { + mode: "deep", + resumeScanId: scanId, + outputDir: scanDir, + }), + ).rejects.toThrow(); + expect(modelInputs.length).toBe(1); + return; + } + if (canceledFollowUp) { + await expect( + client.run(repository, { + mode: "deep", + signal: cancellation.signal, + postScanPrompt: followUp, + }), + ).rejects.toThrow(/interrupted/); + const completed = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + expect(completed["scan"]).toMatchObject({ + progress: { status: "complete" }, + findingCount: 1, + reportAvailable: true, + }); + expect(modelInputs.length).toBe(2); + expect(modelInputs[1]).toBe(followUp); + expect( + commands.filter((command) => command === "complete-scan"), + ).toHaveLength(loseCompletionResponse ? 2 : 1); + expect(commands).not.toContain("cancel-scan"); + expect(commands).not.toContain("fail-scan"); + expect(await readFile(join(scanDir, "report.md"), "utf8")).toBe( + acceptedReport, + ); + return; + } + if (outcome === "completion-before-commit-fails") { + await expect(client.run(repository, { mode: "deep" })).rejects.toThrow( + "Synthetic completion failure before commit", + ); + expect( + commands.filter((command) => command === "complete-scan"), + ).toHaveLength(1); + expect(commands).not.toContain("fail-scan"); + expect(modelInputs).toHaveLength(1); + const saved = await runWorkbench(workbenchOptions!, [ + "get-scan", + "--scan-id", + scanId, + ]); + expect(saved["scan"]).toMatchObject({ + progress: { status: "running" }, + }); + return; + } + const result = await client.run(repository, { + mode: "deep", + signal: cancellation.signal, + postScanPrompt: + outcome === "published-before-cancellation" ? undefined : followUp, + ...(restart ? { resumeScanId: scanId, outputDir: scanDir } : {}), + }); + expect(result.threadId).toBe(threadId); + if (outcome === "lost-completion-response") { + expect( + commands.filter((command) => command === "complete-scan"), + ).toHaveLength(2); + } + if (ownerSuffix) { + expect(nativeOwnerStatus).toBe("running"); + expect(result.cost?.inputTokens).toBe(2_500); + expect(result.cost?.outputTokens).toBe(60); + } else { + // The synthetic accepted workers have no native usage receipts. + expect(result.cost).toBeNull(); + } + expect(result.coverage.completeness).toBe("partial"); + expect(result.findings.findings[0]?.remediation).toBe( + "Validate the resolved destination before writing.", + ); + // postScanPrompt retains its existing behavior on each caller invocation. + expect(modelInputs.filter((input) => input !== followUp).length).toBe(1); + expect(modelInputs.filter((input) => input === followUp).length).toBe( + outcome === "published-before-cancellation" ? 0 : restart ? 2 : 1, + ); + const completed = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + expect(completed["scan"]).toMatchObject({ + progress: { status: "complete" }, + }); + expect(await readFile(join(scanDir, "report.md"), "utf8")).toContain( + "Validate the resolved destination", + ); + expect(commands).not.toContain("fail-scan"); + } finally { + clearTimeout(keepAlive); + await client.close(); + } + }; + test(name, runCase, 30_000); +} + +async function rejoinSelectedScanThroughMcp( + environment: NodeJS.ProcessEnv, + scanId: string, + explicitCompletion = false, +): Promise { + const child = spawn( + process.execPath, + [join(PLUGIN_ROOT, "mcp/server.mjs"), "--stdio"], + { + env: environment, + stdio: ["pipe", "pipe", "pipe"], + }, + ); + let stderr = ""; + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + stderr += chunk; + }); + const exited = new Promise((resolve) => + child.once("exit", () => resolve()), + ); + const result = new Promise((resolve, reject) => { + let buffer = ""; + child.once("error", reject); + child.once("exit", () => + reject(new Error(`MCP closed before selected rejoin: ${stderr}`)), + ); + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + buffer += chunk; + for (;;) { + const newline = buffer.indexOf("\n"); + if (newline < 0) break; + const line = buffer.slice(0, newline); + buffer = buffer.slice(newline + 1); + try { + const response = JSON.parse(line) as { + id?: number; + error?: unknown; + result?: { + isError?: boolean; + structuredContent?: { manifestPath?: string }; + }; + }; + if (response.id === 1) { + child.stdin.write( + JSON.stringify({ + jsonrpc: "2.0", + id: 2, + method: "tools/call", + params: { + name: explicitCompletion + ? "complete_codex_security_scan" + : "start_codex_security_deep_scan", + arguments: { scanId }, + _meta: { + "openai/threadId": threadId, + "codex/sandbox-state-meta": { + permissionProfile: { + type: "managed", + file_system: { + type: "restricted", + entries: [ + { + path: { + type: "special", + value: { kind: "root" }, + }, + access: "read", + }, + ], + }, + network: "restricted", + }, + sandboxCwd: pathToFileURL(PLUGIN_ROOT).href, + }, + }, + }, + }) + "\n", + ); + } + if (response.id === 2) { + expect(response.error).toBeUndefined(); + expect(response.result?.isError).toBeUndefined(); + if (!explicitCompletion) + expect( + response.result?.structuredContent?.manifestPath, + ).toBeDefined(); + resolve(); + } + } catch (error) { + reject(error); + } + } + }); + }); + child.stdin.write( + JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "initialize", + params: { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "sdk-owner-completion", version: "0.1.0" }, + }, + }) + "\n", + ); + try { + await result; + } finally { + child.stdin.end(); + child.kill(); + await exited; + } +} diff --git a/sdk/typescript/tests-ts/deep-scan-coverage.test.ts b/sdk/typescript/tests-ts/deep-scan-coverage.test.ts new file mode 100644 index 0000000000..4d4996bad1 --- /dev/null +++ b/sdk/typescript/tests-ts/deep-scan-coverage.test.ts @@ -0,0 +1,114 @@ +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { expect, test } from "bun:test"; +import { main } from "../src/cli.js"; +import { loadContract } from "../src/contract.js"; +import { ScanResult } from "../src/result.js"; +import { capture, dependencies } from "./cli-fixtures.js"; + +const fixtureUrl = new URL( + "../../../plugins/codex-security/mcp-app/tests/deep_scan_coverage_fixture.mjs", + import.meta.url, +); + +test.each([ + ["partial", false, false], + ["unknown", false, false], + ["complete", false, false], + ["partial", true, false], + ["partial", true, true], +] as const)( + "publishes %s source coverage through CLI results (resume: %p, continued: %p)", + async (completeness, resume, continueAfterResume) => { + const root = await mkdtemp(join(tmpdir(), "deep-coverage-publication-")); + try { + await mkdir(join(root, "fixture"), { mode: 0o700 }); + // Keep the real workbench outside other suites' persistent module mocks. + const child = Bun.spawn( + [ + Bun.which("node")!, + fileURLToPath(fixtureUrl), + join(root, "fixture"), + completeness, + String(resume), + String(continueAfterResume), + ], + { stdout: "pipe", stderr: "pipe" }, + ); + const [output, errors, exitCode] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + expect(exitCode, errors).toBe(0); + const { scanDir, threadId, terminal } = JSON.parse(output); + const contract = await loadContract(scanDir, { + pluginRoot: fileURLToPath( + new URL("../../../plugins/codex-security/", import.meta.url), + ), + }); + const result = new ScanResult({ + ...contract, + scanDir, + threadId, + turnResult: { status: "completed" }, + }); + expect(result.coverage.completeness).toBe(completeness); + const coverage = JSON.parse( + await readFile(join(scanDir, "coverage.json"), "utf8"), + ); + expect(coverage.reviews[0].attempt).toBe(2); + const report = await readFile(join(scanDir, "report.md"), "utf8"); + expect(report).toContain(`| Coverage | ${completeness} |`); + expect(coverage.explicitExclusions).toHaveLength(coverage.reviews.length); + for (const review of coverage.reviews) + expect(report).toContain(review.workerId); + if (completeness === "partial") { + expect( + coverage.deferred.map((item: { reason: string }) => item.reason), + ).toEqual(["Verify entry boundaries.", "Verify symbolic links."]); + expect( + new Set(coverage.deferred.map((item: { id: string }) => item.id)) + .size, + ).toBe(2); + expect( + coverage.reviews.map( + (review: { completeness: string }) => review.completeness, + ), + ).toEqual(["partial", "complete", "unknown"]); + for (const item of coverage.deferred) { + expect(item.provenance.candidateId).toBe("candidate-1"); + expect(report).toContain(item.reason); + expect( + coverage.surfaces.some((surface: { id: string }) => + item.surfaceIds.includes(surface.id), + ), + ).toBe(true); + } + } + for (const surface of coverage.surfaces) { + expect( + await readFile(join(scanDir, surface.receiptRefs[0]), "utf8"), + ).toContain("Synthetic review evidence."); + } + const stdout = capture(); + const stderr = capture(); + expect( + await main( + ["scan", "--mode", "deep", "--json"], + stdout.stream, + stderr.stream, + dependencies({ result, onWorkbench: () => ({ deepScan: terminal }) }), + ), + ).toBe(completeness === "complete" ? 0 : 2); + expect(JSON.parse(stdout.text()).coverage).toEqual(coverage); + if (completeness !== "complete") + expect(stderr.text()).toContain("STOPPED"); + } finally { + await rm(root, { recursive: true, force: true }); + } + }, + 60_000, +); diff --git a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts index 3dd0343837..7d044043bf 100644 --- a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts +++ b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts @@ -22,12 +22,12 @@ const deepScanOwnershipProbe = [ "connection.executescript('''", "CREATE TABLE workspaces (id TEXT PRIMARY KEY, thread_id TEXT, updated_at TEXT);", "CREATE TABLE scans (id TEXT PRIMARY KEY, workspace_id TEXT, mode TEXT, status TEXT, recipe_json TEXT, handoff_status TEXT, handoff_claim_token TEXT, deep_scan_owner_thread_id TEXT, updated_at TEXT);", - "CREATE TABLE deep_scan_runs (scan_id TEXT PRIMARY KEY);", + "CREATE TABLE deep_scan_runs (scan_id TEXT PRIMARY KEY, schema_version INTEGER NOT NULL DEFAULT 1, workflow_version TEXT NOT NULL DEFAULT 'deep-scan-mcp/v1');", "''')", "scan_id = '11111111-1111-4111-8111-111111111111'", "connection.execute(\"INSERT INTO workspaces VALUES ('workspace', NULL, 'before')\")", "connection.execute(\"INSERT INTO scans VALUES (?, 'workspace', 'deep', 'running', '{}', 'delivered', ?, NULL, 'before')\", (scan_id, case['storedToken']))", - "connection.execute('INSERT INTO deep_scan_runs VALUES (?)', (scan_id,))", + "connection.execute('INSERT INTO deep_scan_runs (scan_id) VALUES (?)', (scan_id,))", "connection.commit()", "if case.get('mutation') == 'rotate':", " connection.executescript(\"CREATE TRIGGER rotate_claim BEFORE UPDATE OF thread_id ON workspaces BEGIN UPDATE scans SET handoff_claim_token = '33333333-3333-4333-8333-333333333333' WHERE workspace_id = NEW.id; END\")", @@ -809,9 +809,7 @@ describe("deep scan workbench ownership", () => { ]), ); const report = await readFile(join(scanDir, "report.md"), "utf8"); - expect(report).toContain( - "No findings were validated before the scan reached its cost limit.", - ); + expect(report).toContain("partial report"); if (existingDeferred) { expect(report).toContain( "Existing candidate validation dependency was unavailable.", @@ -866,7 +864,7 @@ describe("deep scan workbench ownership", () => { false, ], ] as const)( - "only describes an exhausted scan cost limit for %s", + "uses the partial-report summary for an exhausted scan cost limit: %s", (_description, reason, exhausted) => { const python = Bun.which("python3") ?? Bun.which("python"); expect(python).not.toBeNull(); @@ -885,11 +883,7 @@ describe("deep scan workbench ownership", () => { const result = runPython(python!, ["-c", script, PLUGIN_ROOT, reason]); expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0); const report = new TextDecoder().decode(result.stdout); - expect( - report.includes( - "No findings were validated before the scan reached its cost limit.", - ), - ).toBe(exhausted); + expect(report.includes("partial report")).toBe(exhausted); expect( report.includes( "No reportable findings survived the canonical discovery, validation, and reportability gates.", diff --git a/sdk/typescript/tests-ts/deep-scan-worker-shutdown.test.ts b/sdk/typescript/tests-ts/deep-scan-worker-shutdown.test.ts index 9ec71e796d..8ea887be8b 100644 --- a/sdk/typescript/tests-ts/deep-scan-worker-shutdown.test.ts +++ b/sdk/typescript/tests-ts/deep-scan-worker-shutdown.test.ts @@ -31,6 +31,18 @@ async function bundledWorkerExecutor( if (source === undefined) { throw new Error("Bundled Deep Scan worker executor was not found."); } + const sessionSource = + /\n\/\/ [^\n]*\/codex-session\.ts\n([\s\S]*?)(?=\n\/\/)/u.exec( + runtime, + )?.[1]; + expect(sessionSource).toBeDefined(); + const recordFunction = /\b(isRecord\d*)\(/u.exec(source)?.[1]; + expect(recordFunction).toBeDefined(); + const recordSource = new RegExp( + `function ${recordFunction}\\([^\\n]*\\) \\{[\\s\\S]*?\\n\\}`, + "u", + ).exec(runtime)?.[0]; + expect(recordSource).toBeDefined(); const fileSystemImport = /\b(import_node_fs\d*)\.promises\.readFile\(/u.exec( source, )?.[1]; @@ -54,6 +66,8 @@ async function bundledWorkerExecutor( "workerPermissionProfile", "workerPermissionProfileConfigOverrides", "snapshotWorkerEnvironment", + "workerModelConfig", + "workerModelSelection", "workerRuntimeSettings", "environmentVariable", "preflightDeepScanWorkerPermissionProfile", @@ -63,7 +77,7 @@ async function bundledWorkerExecutor( "workerSubagentConfig", "appendSafeItemDiagnostic", "classifyCodexWorkerError", - `${source}\nreturn CodexSdkWorkerExecutor;`, + `${sessionSource}\n${recordSource}\n${source}\nreturn CodexSdkWorkerExecutor;`, )( FakeCodex, { promises: { readFile: async () => "fixture worker prompt" } }, @@ -71,6 +85,8 @@ async function bundledWorkerExecutor( () => [], async () => ({}), async () => ({}), + () => ({}), + async () => ({}), () => undefined, preflight, () => undefined, diff --git a/sdk/typescript/tests-ts/fixtures/selected-deep-scan.py b/sdk/typescript/tests-ts/fixtures/selected-deep-scan.py new file mode 100644 index 0000000000..5b741021ca --- /dev/null +++ b/sdk/typescript/tests-ts/fixtures/selected-deep-scan.py @@ -0,0 +1,67 @@ +"""Synthetic accepted workers for the installed finalization/SDK contract test.""" +import hashlib +import json +import sqlite3 +import sys +import uuid +from pathlib import Path + +payload = json.load(sys.stdin) +scan_id = payload["scanId"] +scan_dir = Path(payload["scanDir"]) +with sqlite3.connect(payload["database"]) as connection: + connection.execute("PRAGMA foreign_keys = ON") + timestamp = connection.execute( + "SELECT created_at FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) + ).fetchone()[0] + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "coordinator_generation = 2, phase = 'reducing', " + "discovery_runs_dispatched = 2, completion_sequence = 2, " + "consecutive_no_new = 2, stop_after_no_new = 2, max_discovery_runs = 2 " + "WHERE scan_id = ?", (scan_id,), + ) + discoveries = [] + for kind, label in [("discovery", "review-1"), ("discovery", "review-2"), ("dedup", "merge-1")]: + draft = dict(payload["draft"]) + if kind == "dedup": + draft["sourceCoverage"] = draft.pop("coverage") + encoded = json.dumps(draft).encode() + digest = hashlib.sha256(encoded).hexdigest() + worker_id = str(uuid.uuid4()) + output = scan_dir / "artifacts" / "deep_discovery" / label / "output" + output.mkdir(parents=True) + prompt = output.parent / "prompt.md" + prompt.write_text("Synthetic accepted audit\n") + accepted = output / "checkpoints" / f"{digest}.json" + accepted.parent.mkdir() + accepted.write_bytes(encoded) + result = output / "result.json" + result.write_bytes(encoded) + sequence = len(discoveries) + 1 if kind == "discovery" else None + connection.execute( + "INSERT INTO deep_scan_workers " + "(id, scan_id, kind, status, merge_state, prompt_path, artifact_dir, " + "result_manifest_path, attempt, completion_sequence, created_at, updated_at, completed_at) " + "VALUES (?, ?, ?, 'succeeded', ?, ?, ?, ?, 1, ?, ?, ?, ?)", + (worker_id, scan_id, kind, "merged" if kind == "discovery" else "none", + str(prompt), str(output), str(result), sequence, timestamp, timestamp, timestamp), + ) + connection.execute( + "INSERT INTO deep_scan_attempts " + "(scan_id, worker_id, attempt, status, started_at, completed_at, accepted_result_path, accepted_result_sha256) " + "VALUES (?, ?, 1, 'succeeded', ?, ?, ?, ?)", + (scan_id, worker_id, timestamp, timestamp, str(accepted), digest), + ) + if kind == "discovery": + discoveries.append(worker_id) + else: + for order, discovery in enumerate(discoveries): + connection.execute( + "INSERT INTO deep_scan_dedup_inputs " + "(scan_id, dedup_worker_id, discovery_worker_id, input_order) VALUES (?, ?, ?, ?)", + (scan_id, worker_id, discovery, order), + ) + # The committed immutable reference survives loss of the replaceable output. + result.unlink() +print(json.dumps({"resultPath": str(result), "acceptedPath": str(accepted)})) diff --git a/sdk/typescript/tests-ts/reasoning-summary.test.ts b/sdk/typescript/tests-ts/reasoning-summary.test.ts new file mode 100644 index 0000000000..854ee85ead --- /dev/null +++ b/sdk/typescript/tests-ts/reasoning-summary.test.ts @@ -0,0 +1,82 @@ +import { execFileSync } from "node:child_process"; +import { + mkdir, + mkdtemp, + readFile, + realpath, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { expect, test } from "bun:test"; +import { captureOriginalReasoningSummary } from "../src/reasoning-summary.js"; +import type { JsonObject } from "../src/config.js"; + +test.each([ + { model_reasoning_summary: "none" }, + { model_reasoning_summary: "concise" }, + { model_reasoning_summary: null }, + { model_reasoning_summary: "" }, + { + profile: "selected", + profiles: { selected: { model_reasoning_summary: "auto" } }, + }, +] as JsonObject[])( + "preserves explicit summary without a native lookup: %j", + async (config) => { + expect( + await captureOriginalReasoningSummary({ + config, + command: { command: "unused-native-executable" }, + cwd: tmpdir(), + environment: {}, + signal: new AbortController().signal, + }), + ).toBeUndefined(); + }, +); + +test.each(["unknown-model", "unsupported-command", "missing-metadata"])( + "keeps an unavailable original model default unknown: %s", + async (scenario) => { + const root = await mkdtemp(join(tmpdir(), "summary-selection-")); + try { + const cwd = join(root, "original output"); + const script = join(root, "native.mjs"); + const receipt = join(root, "receipt.json"); + await mkdir(cwd); + await writeFile( + script, + [ + 'import { writeFileSync } from "node:fs";', + `writeFileSync(${JSON.stringify(receipt)}, JSON.stringify({cwd:process.cwd(),args:process.argv,home:process.env.CODEX_HOME}));`, + `console.log(JSON.stringify({models:[{slug:${JSON.stringify(scenario === "unknown-model" ? "another-model" : "selected-model")}${scenario === "missing-metadata" ? "" : ',default_reasoning_summary:"none"'}}]}));`, + `process.exit(${scenario === "unsupported-command" ? 1 : 0});`, + ].join("\n"), + ); + const command = execFileSync("node", ["-p", "process.execPath"], { + encoding: "utf8", + }).trim(); + const value = await captureOriginalReasoningSummary({ + config: { model: "selected-model", model_reasoning_effort: "high" }, + command: { command }, + cwd, + environment: { + CODEX_HOME: root, + NODE_OPTIONS: `--import=${pathToFileURL(script).href}`, + }, + signal: new AbortController().signal, + }); + expect(value).toBeUndefined(); + const recorded = JSON.parse(await readFile(receipt, "utf8")); + expect(await realpath(recorded.cwd)).toBe(await realpath(cwd)); + expect(recorded.home).toBe(root); + expect(recorded.args).toContain('model="selected-model"'); + expect(recorded.args).toContain('model_reasoning_effort="high"'); + } finally { + await rm(root, { recursive: true, force: true }); + } + }, +); diff --git a/sdk/typescript/tests-ts/scan-logs.test.ts b/sdk/typescript/tests-ts/scan-logs.test.ts index e95366f433..2111a43767 100644 --- a/sdk/typescript/tests-ts/scan-logs.test.ts +++ b/sdk/typescript/tests-ts/scan-logs.test.ts @@ -58,6 +58,414 @@ function commandEvent(command: string, id: string, timestamp?: string) { } describe("saved scan logs", () => { + test.each(["standard", "deep"])( + "ignores model-owned log homes without host ownership for %s scans", + async (mode) => { + const configured = await temporaryHome(); + const foreign = await temporaryHome(); + const scanDir = await temporaryHome(); + const settings = join(scanDir, "artifacts", "deep_discovery"); + await mkdir(settings, { recursive: true }); + await writeFile( + join(settings, "execution-settings.json"), + JSON.stringify({ version: 1, settings: { codexHome: foreign } }), + ); + const configuredEvent = commandEvent( + "configured owner", + "configured-call", + ); + await writeSession(configured, "owner", [configuredEvent]); + await writeSession(foreign, "owner", [ + configuredEvent, + commandEvent("foreign suffix", "foreign-call"), + ]); + const result = await readSavedScanLogs( + { scanId: "scan-1", mode, scanDir, continuationThreadId: "owner" }, + configured, + ); + expect(result.events.map(({ event }) => event)).toEqual([ + { type: "session_meta", payload: { id: "owner" } }, + configuredEvent, + ]); + expect(result.sessions.map(({ path }) => path)).toEqual([ + join(configured, "sessions", "2026", "08", "11", "rollout-owner.jsonl"), + ]); + }, + ); + + test.each( + ["sessions", "archived_sessions"].flatMap((directory) => + ["prefix only", "unrelated owner turn"].map((tail) => [directory, tail]), + ), + )( + "reads the complete same-thread recorded copy from %s after %s", + async (directory, tail) => { + const current = await temporaryHome(); + const original = await temporaryHome(); + const scanDir = await temporaryHome(); + const settings = join(scanDir, "artifacts", "deep_discovery"); + await mkdir(settings, { recursive: true }); + await writeFile( + join(settings, "execution-settings.json"), + JSON.stringify({ version: 1, settings: { codexHome: original } }), + ); + const timestamp = "2026-08-11T12:01:00.000Z"; + const prefix = [ + { type: "turn_context", timestamp, payload: { turn_id: "scan-turn" } }, + commandEvent("prefix", "prefix-call", timestamp), + ]; + const suffix = commandEvent("saved suffix", "suffix-call", timestamp); + await writeSession(current, "owner", [ + ...prefix, + ...(tail === "unrelated owner turn" + ? [ + { + type: "turn_context", + timestamp, + payload: { turn_id: "other-turn" }, + }, + commandEvent( + "unrelated first copy", + "other-first-call", + timestamp, + ), + ] + : []), + ]); + const currentPath = join( + current, + "sessions", + "2026", + "08", + "11", + "rollout-owner.jsonl", + ); + await writeFile( + currentPath, + (await readFile(currentPath, "utf8")) + '\n\n42\n{"type":', + ); + await writeSession(original, "owner", [ + ...prefix, + suffix, + suffix, + { type: "turn_context", timestamp, payload: { turn_id: "other-turn" } }, + commandEvent("unrelated turn", "other-call", timestamp), + ]); + if (directory === "archived_sessions") { + await rename(join(original, "sessions"), join(original, directory)); + } + const result = await readSavedScanLogs( + { + scanId: "scan-1", + mode: "deep", + scanDir, + continuationThreadId: "owner", + executionAttribution: { + formatVersion: 1, + workerCodexHome: original, + executionThreadIds: [], + owner: { + threadId: "owner", + turnId: "scan-turn", + startedAt: timestamp, + }, + startedAt: timestamp, + completedAt: timestamp, + }, + }, + current, + ); + expect(result.sessions.map(({ threadId }) => threadId)).toEqual([ + "owner", + ]); + expect(result.events.map(({ event }) => event)).toEqual([ + { type: "session_meta", payload: { id: "owner" } }, + ...prefix, + suffix, + suffix, + ]); + expect(result.sessions[0]?.path).toStartWith(join(original, directory!)); + }, + ); + + test.each(["equal", "shorter", "divergent"])( + "keeps the first rollout when attributed occurrences are %s", + async (copy) => { + const first = await temporaryHome(); + const second = await temporaryHome(); + const timestamp = "2026-08-11T12:01:00.000Z"; + const turn = { + type: "turn_context", + timestamp, + payload: { turn_id: "scan-turn" }, + }; + const prefix = commandEvent("first", "first-call", timestamp); + const suffix = commandEvent("complete", "complete-call", timestamp); + const unrelated = { + type: "turn_context", + timestamp, + payload: { turn_id: "other-turn" }, + }; + await writeSession(first, "owner", [turn, prefix, suffix, unrelated]); + await writeSession(second, "owner", [ + turn, + ...(copy === "equal" + ? [prefix, suffix] + : copy === "shorter" + ? [prefix] + : [ + prefix, + commandEvent("different", "different-call", timestamp), + suffix, + ]), + unrelated, + commandEvent("more unrelated work", "other-call", timestamp), + ]); + const result = await readScanLogs({ + scanId: "scan-1", + threadId: "owner", + codexHome: [first, second], + executionAttribution: { + formatVersion: 1, + executionThreadIds: [], + owner: { + threadId: "owner", + turnId: "scan-turn", + startedAt: timestamp, + }, + startedAt: timestamp, + completedAt: timestamp, + }, + }); + expect(result.sessions[0]?.path).toStartWith(first); + expect(result.events.map(({ event }) => event)).toEqual([ + { type: "session_meta", payload: { id: "owner" } }, + turn, + prefix, + suffix, + ]); + }, + ); + + test.each(["equal", "shorter", "divergent"])( + "keeps the first rollout when a later copy is %s", + async (copy) => { + const first = await temporaryHome(); + const second = await temporaryHome(); + const prefix = commandEvent("first", "first-call"); + const suffix = commandEvent("complete", "complete-call"); + await writeSession(first, "owner", [prefix, suffix]); + await writeSession( + second, + "owner", + copy === "equal" + ? [prefix, suffix] + : copy === "shorter" + ? [prefix] + : [prefix, commandEvent("different", "different-call"), suffix], + ); + const result = await readScanLogs({ + scanId: "scan-1", + threadId: "owner", + codexHome: [first, second], + }); + expect(result.sessions[0]?.path).toStartWith(first); + expect(result.events.map(({ event }) => event)).toEqual([ + { type: "session_meta", payload: { id: "owner" } }, + prefix, + suffix, + ]); + }, + ); + + test.each([ + ["current", "original"], + ["current", "substituted"], + ["current", "deleted"], + ["original", "original"], + ["original", "substituted"], + ["original", "deleted"], + ] as const)( + "reads recovered Deep workers with owner in %s home and %s artifact", + async (ownerHome, artifact) => { + const current = await temporaryHome(); + const original = await temporaryHome(); + const scanDir = await temporaryHome(); + const settingsDirectory = join(scanDir, "artifacts", "deep_discovery"); + await mkdir(settingsDirectory, { recursive: true }); + await writeFile( + join(settingsDirectory, "execution-settings.json"), + JSON.stringify({ + version: 1, + settings: { codexHome: original, codexPath: join(original, "codex") }, + }), + ); + const startedAt = "2026-08-11T12:00:00.000Z"; + const timestamp = "2026-08-11T12:01:00.000Z"; + await writeSession( + ownerHome === "current" ? current : original, + "owner", + [ + { + type: "turn_context", + timestamp, + payload: { turn_id: "scan-turn" }, + }, + commandEvent("scan owner", "owner-call", timestamp), + { + type: "turn_context", + timestamp, + payload: { turn_id: "later-turn" }, + }, + commandEvent("unrelated owner turn", "other-turn", timestamp), + ], + ); + for (const id of ["discovery", "resumed-discovery", "reducer"]) { + await writeSession(original, id, [commandEvent(id, id, timestamp)]); + } + await writeSession( + original, + "worker-child", + [commandEvent("worker child", "child-call", timestamp)], + "discovery", + ); + // Keep the existing first-home preference for duplicate active rollouts. + await writeSession(current, "reducer", [ + commandEvent("current reducer", "current-reducer-call", timestamp), + ]); + for (const home of [current, original]) { + await writeSession(home, "unrelated", [ + commandEvent("unrelated scan", "unrelated-call", timestamp), + ]); + } + const settingsPath = join(settingsDirectory, "execution-settings.json"); + if (artifact === "deleted") unlinkSync(settingsPath); + if (artifact === "substituted") + await writeFile( + settingsPath, + JSON.stringify({ version: 1, settings: { codexHome: current } }), + ); + const result = await readSavedScanLogs( + { + scanId: "scan-1", + mode: "deep", + scanDir, + continuationThreadId: "owner", + executionAttribution: { + formatVersion: 1, + workerCodexHome: original, + executionThreadIds: ["discovery", "resumed-discovery", "reducer"], + owner: { threadId: "owner", turnId: "scan-turn", startedAt }, + startedAt, + completedAt: "2026-08-11T12:02:00.000Z", + }, + }, + current, + ); + expect(result.sessions.map(({ threadId }) => threadId).sort()).toEqual([ + "discovery", + "owner", + "reducer", + "resumed-discovery", + "worker-child", + ]); + expect( + result.events.filter(({ threadId }) => threadId === "reducer"), + ).toHaveLength(2); + expect(JSON.stringify(result)).toContain("current reducer"); + expect(JSON.stringify(result)).not.toContain("unrelated"); + }, + ); + + test("reads the complete attributed worker copy despite differing inherited history", async () => { + const home = await temporaryHome(); + const current = await temporaryHome(); + await writeSession(home, "parent", []); + const startedAt = "2026-08-11T12:02:00.900Z"; + await writeSession( + home, + "worker", + [ + { + type: "session_meta", + payload: { id: "parent", timestamp: "2026-08-11T12:00:00.000Z" }, + }, + { + type: "event_msg", + payload: { + type: "task_started", + started_at: Date.parse("2026-08-11T12:00:00.000Z") / 1_000, + }, + }, + { + type: "event_msg", + payload: { + type: "agent_message", + message: "PRIVATE PRE-SCAN CONVERSATION", + }, + }, + { + type: "event_msg", + timestamp: startedAt, + payload: { + type: "task_started", + started_at: Math.floor(Date.parse(startedAt) / 1_000), + }, + }, + { + type: "event_msg", + timestamp: startedAt, + payload: { + type: "agent_message", + message: "Reviewing authorization", + }, + }, + ], + "parent", + startedAt, + ); + + const path = join( + home, + "sessions", + "2026", + "08", + "11", + "rollout-worker.jsonl", + ); + const currentPath = join(current, "sessions", "rollout-worker.jsonl"); + await mkdir(join(current, "sessions"), { recursive: true }); + await writeFile( + currentPath, + (await readFile(path, "utf8")) + .split("\n") + .slice(0, 4) + .join("\n") + .replace( + "PRIVATE PRE-SCAN CONVERSATION", + "OTHER PRE-SCAN CONVERSATION", + ), + ); + const result = await readScanLogs({ + scanId: "scan-1", + threadId: "parent", + codexHome: [current, home], + executionAttribution: { + formatVersion: 1, + executionThreadIds: ["worker"], + owner: { threadId: "parent", turnId: "scan-turn", startedAt }, + startedAt, + completedAt: null, + }, + }); + expect(JSON.stringify(result)).toContain("Reviewing authorization"); + expect(JSON.stringify(result)).not.toContain("PRIVATE PRE-SCAN"); + expect(JSON.stringify(result)).not.toContain("OTHER PRE-SCAN"); + expect( + result.sessions.find(({ threadId }) => threadId === "worker")?.path, + ).toBe(path); + }); + test.each([ ["prefix first", [0], [0, 1, 1, 2], 1, false], ["complete first", [0, 1, 1, 2], [0], 0, false], diff --git a/sdk/typescript/tests-ts/scan-resume.test.ts b/sdk/typescript/tests-ts/scan-resume.test.ts index c3a71693c3..9c4ca358ca 100644 --- a/sdk/typescript/tests-ts/scan-resume.test.ts +++ b/sdk/typescript/tests-ts/scan-resume.test.ts @@ -166,10 +166,16 @@ async function interruptedScan( const sessionPath = join(codexHome, "sessions", `rollout-${threadId}.jsonl`); await writeFile( sessionPath, - JSON.stringify({ - type: "session_meta", - payload: { id: threadId, cwd: scanDir }, - }) + "\n", + [ + { type: "session_meta", payload: { id: threadId, cwd: scanDir } }, + { + type: "turn_context", + timestamp: new Date().toISOString(), + payload: { turn_id: "synthetic-scan-turn", model: "gpt-5.6-sol" }, + }, + ] + .map((event) => JSON.stringify(event)) + .join("\n") + "\n", ); if (mode === "deep") { await command([ @@ -521,6 +527,7 @@ test.each([ f.sessionPath, JSON.stringify({ type: "event_msg", + timestamp: new Date().toISOString(), payload: { type: "token_count", info: { diff --git a/sdk/typescript/tests-ts/scan-usage-reconciliation.test.ts b/sdk/typescript/tests-ts/scan-usage-reconciliation.test.ts new file mode 100644 index 0000000000..13bc7e95c9 --- /dev/null +++ b/sdk/typescript/tests-ts/scan-usage-reconciliation.test.ts @@ -0,0 +1,601 @@ +import { describe, expect, test } from "bun:test"; +import { appendFile, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { estimateScanCost } from "../src/cost-model.js"; +import { ScanCostTracker } from "../src/cost.js"; +import { readScanLogs } from "../src/scan-logs.js"; +import type { ScanExecutionAttribution } from "../src/scan-sessions.js"; + +describe("scan usage reconciliation", () => { + test("SDK usage and logs share attempt membership and the original owner turn", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-attribution-")); + const observed: unknown[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + onSessionEvent: (event) => observed.push(event), + }); + const at = (second: number) => + `2026-09-01T00:00:${String(second).padStart(2, "0")}Z`; + const attribution: ScanExecutionAttribution = { + formatVersion: 1, + executionThreadIds: ["old-worker", "replacement-worker"], + owner: { threadId: "parent", turnId: "scan-turn", startedAt: at(1) }, + startedAt: at(1), + completedAt: at(10), + }; + const token = (second: number, count: number) => ({ + timestamp: at(second), + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: { input_tokens: count, output_tokens: 0 } }, + }, + }); + const context = (second: number, turn: string) => ({ + timestamp: at(second), + type: "turn_context", + payload: { turn_id: turn, model: "gpt-5.6-sol" }, + }); + try { + await mkdir(join(home, "sessions")); + for (const [id, parent, events] of [ + [ + "parent", + null, + [ + context(0, "prior-turn"), + token(0, 100), + context(1, "scan-turn"), + token(2, 110), + context(3, "unrelated-turn"), + token(4, 1010), + ], + ], + [ + "old-worker", + null, + [context(1, "worker-turn"), token(2, 20), token(11, 120)], + ], + ["replacement-worker", null, [context(3, "worker-turn"), token(4, 30)]], + ["worker-child", "old-worker", [context(3, "child-turn"), token(4, 5)]], + ["unrelated-child", "parent", [context(3, "side-turn"), token(4, 900)]], + ] as const) { + const records = [ + { + type: "session_meta", + payload: { id, ...(parent ? { parent_thread_id: parent } : {}) }, + }, + ...events, + ]; + await writeFile( + join(home, "sessions", `${id}.jsonl`), + records.map((value) => JSON.stringify(value)).join("\n") + "\n", + ); + } + tracker.setAttributionReader(async () => attribution); + tracker.start("parent"); + const snapshot = await tracker.stop(); + expect(snapshot.cost?.inputTokens).toBe(65); + expect(JSON.stringify(observed)).not.toContain("unrelated-turn"); + expect(JSON.stringify(observed)).not.toContain(at(11)); + const logs = await readScanLogs({ + scanId: "scan", + threadId: "parent", + codexHome: home, + executionAttribution: attribution, + }); + expect(logs.sessions.map((session) => session.threadId).sort()).toEqual([ + "old-worker", + "parent", + "replacement-worker", + "worker-child", + ]); + expect( + logs.events.some(({ event }) => + JSON.stringify(event).includes("unrelated-turn"), + ), + ).toBe(false); + expect( + logs.events.some(({ event }) => JSON.stringify(event).includes(at(11))), + ).toBe(false); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); + + test("retains known usage before attribution and uncertainty until missing attempt usage arrives", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-delayed-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + let attribution: ScanExecutionAttribution | null = null; + tracker.setAttributionReader(async () => attribution); + const at = "2026-09-01T00:00:02Z"; + const records = (id: string, count: number) => + [ + { type: "session_meta", payload: { id } }, + { + timestamp: at, + type: "turn_context", + payload: { model: "gpt-5.6-sol", turn_id: "own" }, + }, + { + timestamp: at, + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { input_tokens: count, output_tokens: 0 }, + }, + }, + }, + ] + .map((event) => JSON.stringify(event)) + .join("\n") + "\n"; + try { + await mkdir(join(home, "sessions")); + await writeFile( + join(home, "sessions", "worker.jsonl"), + records("worker", 20), + ); + tracker.start("worker"); + tracker.recordUsage({ input_tokens: 20, output_tokens: 0 }); + expect((await tracker.refresh()).cost).toMatchObject({ inputTokens: 20 }); + attribution = { + formatVersion: 1, + executionThreadIds: ["worker", "failed-attempt"], + owner: { threadId: "worker", turnId: "own", startedAt: at }, + startedAt: "2026-09-01T00:00:01Z", + completedAt: "2026-09-01T00:00:10Z", + }; + expect((await tracker.refresh()).cost).toMatchObject({ + inputTokens: 20, + coverage: "partial", + }); + await writeFile( + join(home, "sessions", "failed.jsonl"), + records("failed-attempt", 5), + ); + const final = await tracker.stop(); + expect(final.cost?.inputTokens).toBe(25); + expect(final.cost?.coverage).toBeUndefined(); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); + + test("preserves receipt accounting for a resumed legacy Deep scan", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-legacy-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + legacy: true, + executionThreadIds: ["legacy-parent"], + owner: { + threadId: "legacy-parent", + turnId: null, + startedAt: "2026-09-01T00:00:00Z", + }, + startedAt: "2026-09-01T00:00:00Z", + completedAt: null, + })); + try { + tracker.start("legacy-parent"); + const snapshot = await tracker.stop({ + input_tokens: 10000, + output_tokens: 100, + }); + expect(snapshot.cost?.inputTokens).toBe(10000); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); + + test("prices each observed model instead of repricing the sum with the parent", () => { + const usage = { + input_tokens: 200, + output_tokens: 20, + modelUsage: [ + { model: "gpt-5.6-sol", input_tokens: 100, output_tokens: 10 }, + { model: "gpt-6-astra", input_tokens: 100, output_tokens: 10 }, + ], + }; + const expected = + estimateScanCost("gpt-5.6-sol", usage.modelUsage[0])!.estimatedUsd + + estimateScanCost("gpt-6-astra", usage.modelUsage[1])!.estimatedUsd; + expect(estimateScanCost("gpt-5.6-sol", usage)?.estimatedUsd).toBe(expected); + }); + + test("keeps incomplete model attribution unpriced", () => { + expect( + estimateScanCost("gpt-5.6-sol", { + input_tokens: 200, + output_tokens: 20, + modelUsage: [ + { model: "gpt-5.6-sol", input_tokens: 100, output_tokens: 10 }, + { model: null, input_tokens: 100, output_tokens: 10 }, + ], + }), + ).toBeNull(); + }); + + test("reconciles stale and missing cumulative receipts without reducing usage", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-receipts-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + try { + tracker.start("worker"); + tracker.recordUsage({ input_tokens: 160, output_tokens: 0 }); + tracker.recordUsage({ input_tokens: 100, output_tokens: 0 }); + tracker.recordUsage(null); + expect((await tracker.stop()).cost?.inputTokens).toBe(160); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); + + test("tracks per-model deltas within one resumed session", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-models-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + try { + await mkdir(join(home, "sessions")); + const records = [ + { type: "session_meta", payload: { id: "worker" } }, + { + type: "turn_context", + payload: { model: "gpt-5.6-sol", turn_id: "turn-1" }, + }, + { + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { input_tokens: 100, output_tokens: 10 }, + }, + }, + }, + { + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: { input_tokens: 60, output_tokens: 6 } }, + }, + }, + { + type: "turn_context", + payload: { model: "gpt-6-astra", turn_id: "turn-2" }, + }, + { + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { input_tokens: 200, output_tokens: 20 }, + }, + }, + }, + ]; + await writeFile( + join(home, "sessions", "worker.jsonl"), + records.map((record) => JSON.stringify(record)).join("\n") + "\n", + ); + tracker.start("worker"); + tracker.recordUsage({ input_tokens: 200, output_tokens: 20 }); + const snapshot = await tracker.stop(); + expect(snapshot.cost?.inputTokens).toBe(200); + expect(snapshot.cost?.estimatedUsd).toBeCloseTo(0.0021, 12); + expect(snapshot.cost?.modelCosts?.map((cost) => cost.model)).toEqual([ + "gpt-5.6-sol", + "gpt-6-astra", + ]); + tracker.recordUsage({ input_tokens: 250, output_tokens: 25 }); + expect((await tracker.refresh()).cost).toBeNull(); + await appendFile( + join(home, "sessions", "worker.jsonl"), + JSON.stringify({ + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { input_tokens: 250, output_tokens: 25 }, + }, + }, + }) + "\n", + ); + expect((await tracker.refresh()).cost?.estimatedUsd).toBeCloseTo( + 0.00285, + 12, + ); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); +}); + +describe("charged response receipts", () => { + for (const exactOnly of [false, true]) { + test(`counts compaction and deduplicates responses across counter resets (exact only: ${exactOnly})`, async () => { + const home = await mkdtemp(join(tmpdir(), "usage-response-receipts-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + const usage = (input: number, cached: number, output: number) => ({ + input_tokens: input, + cached_input_tokens: cached, + cache_write_input_tokens: 0, + output_tokens: output, + reasoning_output_tokens: 0, + total_tokens: input + output, + }); + const record = ( + id: string, + count: unknown, + cumulative: unknown, + model = "gpt-5.6-sol", + ) => ({ + type: "token_usage_record", + payload: { + thread_id: "worker", + turn_id: "turn", + response_id: id, + model, + usage: count, + thread_token_usage: cumulative, + }, + }); + const first = record("normal-1", usage(100, 80, 10), usage(100, 80, 10)); + const compact = record( + "compaction", + usage(50, 40, 5), + usage(150, 120, 15), + "gpt-6-astra", + ); + const second = record("normal-2", usage(120, 90, 12), usage(120, 90, 12)); + const counter = (count: unknown) => ({ + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: count }, + }, + }); + const events = [ + { type: "session_meta", payload: { id: "worker" } }, + first, + ...(!exactOnly ? [counter(usage(100, 80, 10))] : []), + compact, + { + type: "compacted", + payload: { message: "Synthetic context summary" }, + }, + compact, + second, + ...(!exactOnly ? [counter(usage(220, 170, 22))] : []), + first, + ]; + try { + await mkdir(join(home, "sessions")); + await writeFile( + join(home, "sessions", "worker.jsonl"), + events.map((e) => JSON.stringify(e)).join("\n") + "\n", + ); + tracker.start("worker"); + tracker.recordUsage(usage(220, 170, 22)); + const result = await tracker.stop(); + expect(result.usage).toMatchObject(usage(270, 210, 27)); + expect( + result.cost?.modelCosts?.map((part) => [ + part.model, + part.inputTokens + part.outputTokens, + ]), + ).toEqual([ + ["gpt-5.6-sol", 242], + ["gpt-6-astra", 55], + ]); + expect((await tracker.refresh()).usage).toEqual(result.usage); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); + } + + test("uses receipt turn identity for shared-parent usage and logs", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-response-owner-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + const attribution: ScanExecutionAttribution = { + formatVersion: 1, + executionThreadIds: [], + owner: { + threadId: "parent", + turnId: "scan-turn", + startedAt: "2026-09-01T00:00:01Z", + }, + startedAt: "2026-09-01T00:00:01Z", + completedAt: "2026-09-01T00:00:10Z", + }; + const receipt = ( + id: string, + turn: string, + second: string, + input: number, + ) => ({ + type: "token_usage_record", + timestamp: `2026-09-01T00:00:${second}Z`, + payload: { + response_id: id, + thread_id: "parent", + turn_id: turn, + model: "gpt-5.6-sol", + usage: { input_tokens: input, output_tokens: 0 }, + }, + }); + try { + await mkdir(join(home, "sessions")); + await writeFile( + join(home, "sessions", "parent.jsonl"), + [ + { type: "session_meta", payload: { id: "parent" } }, + receipt("prior", "prior-turn", "00", 900), + receipt("owned", "scan-turn", "02", 20), + receipt("side", "other-turn", "03", 800), + receipt("post", "scan-turn", "11", 700), + ] + .map((e) => JSON.stringify(e)) + .join("\n") + "\n", + ); + tracker.setAttributionReader(async () => attribution); + tracker.start("parent"); + expect((await tracker.stop()).cost?.inputTokens).toBe(20); + const logs = await readScanLogs({ + scanId: "scan", + threadId: "parent", + codexHome: home, + executionAttribution: attribution, + }); + const ids = logs.events + .map( + ({ event }) => + (event as { payload?: Record })["payload"]?.[ + "response_id" + ], + ) + .filter(Boolean); + expect(ids).toEqual(["owned"]); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); +}); + +test("delayed response receipts resolve cumulative gaps without treating smaller counters as stale", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-delayed-response-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + const record = (id: string, tokens: number, cumulative: number) => + JSON.stringify({ + type: "token_usage_record", + payload: { + response_id: id, + thread_id: "worker", + model: "gpt-5.6-sol", + usage: { input_tokens: tokens, output_tokens: 0 }, + thread_token_usage: { input_tokens: cumulative, output_tokens: 0 }, + }, + }) + "\n"; + try { + await mkdir(join(home, "sessions")); + const file = join(home, "sessions", "worker.jsonl"); + await writeFile( + file, + JSON.stringify({ type: "session_meta", payload: { id: "worker" } }) + + "\n" + + record("first", 100, 100) + + record("third", 50, 180), + ); + tracker.start("worker"); + expect((await tracker.refresh()).cost).toMatchObject({ + inputTokens: 150, + coverage: "partial", + }); + await appendFile(file, record("second", 30, 130)); + const result = await tracker.stop(); + expect(result.cost?.inputTokens).toBe(180); + expect(result.cost?.coverage).toBeUndefined(); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } +}); + +test("a reader installed before the native attribution writer preserves legacy receipts", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-reader-first-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + try { + tracker.setAttributionReader(async () => undefined); + tracker.start("parent"); + tracker.recordUsage({ input_tokens: 100, output_tokens: 0 }); + expect((await tracker.stop()).cost?.inputTokens).toBe(100); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } +}); + +test("late exact receipts replace an overlapping legacy counter without adding it twice", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-overlap-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + const usage = (count: number) => ({ input_tokens: count, output_tokens: 0 }); + const receipt = (id: string, count: number, cumulative: number) => ({ + type: "token_usage_record", + payload: { + thread_id: "worker", + response_id: id, + model: "gpt-5.6-sol", + usage: usage(count), + thread_token_usage: usage(cumulative), + }, + }); + try { + await mkdir(join(home, "sessions")); + await writeFile( + join(home, "sessions", "worker.jsonl"), + [ + { type: "session_meta", payload: { id: "worker" } }, + { + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: usage(100) }, + }, + }, + receipt("new", 10, 110), + receipt("old", 100, 100), + { + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: usage(10) }, + }, + }, + ] + .map((e) => JSON.stringify(e)) + .join("\n") + "\n", + ); + tracker.start("worker"); + const result = await tracker.stop(); + expect(result.cost?.inputTokens).toBe(110); + expect(result.cost?.coverage).toBeUndefined(); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } +}); diff --git a/sdk/typescript/tests-ts/support/api-events.ts b/sdk/typescript/tests-ts/support/api-events.ts index f22b012232..d09618c327 100644 --- a/sdk/typescript/tests-ts/support/api-events.ts +++ b/sdk/typescript/tests-ts/support/api-events.ts @@ -1,3 +1,6 @@ +import { chmod, cp, mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import type { CodexOptions, ThreadEvent } from "@openai/codex-sdk"; import { CodexSecurity, runScanEvents } from "../../src/api.js"; import type { ScanOptions } from "../../src/index.js"; @@ -48,6 +51,38 @@ type ScanEventOptions = Omit< "thread" | "events" | "signal" | "scanDir" | "pluginRoot" | "expectation" > & { abortController?: AbortController }; +export function createApiTestFixtures() { + const temporaryDirectories: string[] = []; + + return { + async cleanup(): Promise { + await Promise.all( + temporaryDirectories + .splice(0) + .map((path) => rm(path, { recursive: true, force: true })), + ); + }, + + async copyCompletedScan(root: string): Promise { + const scanDir = join(root, "scan"); + await cp(join(PLUGIN_ROOT, "examples", "completed-scan"), scanDir, { + recursive: true, + }); + await chmod(scanDir, 0o700); + await writeFile(join(scanDir, "report.md"), "# Scan report\n"); + return scanDir; + }, + + async temporaryDirectory(): Promise { + const path = await realpath( + await mkdtemp(join(tmpdir(), "codex-security-api-")), + ); + temporaryDirectories.push(path); + return path; + }, + }; +} + export async function* completedEvents( threadId = "thread-1", ): AsyncGenerator {