diff --git a/plugins/codex-security/mcp-app/helpers-main.ts b/plugins/codex-security/mcp-app/helpers-main.ts index 18c8611be..fc542cdf6 100644 --- a/plugins/codex-security/mcp-app/helpers-main.ts +++ b/plugins/codex-security/mcp-app/helpers-main.ts @@ -1,34 +1,48 @@ +import { existsSync, realpathSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +export { parseCanonicalScanDraft } from "./src/artifact-scan-draft.js"; +export { resumeSelectedDeepScan } from "./src/deep-scan/finalization.js"; import { resolveSecurityMdCommand } from "./src/helpers/resolve-security-md"; import { decodePosixBytes } from "./src/helpers/posix-path"; import { windowsBinding } from "./src/native"; -let commandLine = process.argv.slice(2); -if (process.platform === "win32") { - const original = windowsBinding().windowsArguments(); - commandLine = original - .slice(original.length - commandLine.length) - .map((argument) => argument.toString("utf16le")); -} -let posixHome = process.env.HOME; -if (commandLine[0] === "--helper") { +// Importing the bundled helper from the SDK does not invoke its CLI adapter. +const entryPath = import.meta.url.startsWith("file:") ? fileURLToPath(import.meta.url) : import.meta.url; +const invokedPath = process.argv[1]; +if ( + invokedPath && existsSync(invokedPath) + && realpathSync(invokedPath) === realpathSync(entryPath) +) runHelper(); + +function runHelper(): void { + let commandLine = process.argv.slice(2); if (process.platform === "win32") { - commandLine = commandLine.slice(1); + const original = windowsBinding().windowsArguments(); + commandLine = original + .slice(original.length - commandLine.length) + .map((argument) => argument.toString("utf16le")); + } + let posixHome = process.env.HOME; + if (commandLine[0] === "--helper") { + if (process.platform === "win32") { + commandLine = commandLine.slice(1); + } else { + const [homeSet, home, ...args] = decodePosixBytes( + Buffer.from(commandLine[1] ?? "", "hex"), + ) + .split("\0") + .slice(0, -1); + posixHome = homeSet ? home : undefined; + commandLine = args; + } + } + const [command, ...args] = commandLine; + if (command === "resolve-security-md") { + process.exitCode = resolveSecurityMdCommand(args, posixHome); } else { - const [homeSet, home, ...args] = decodePosixBytes( - Buffer.from(commandLine[1] ?? "", "hex"), - ) - .split("\0") - .slice(0, -1); - posixHome = homeSet ? home : undefined; - commandLine = args; + console.error( + "Usage: launch_codex_security_mcp[.cmd] --helper resolve-security-md [options]", + ); + process.exitCode = 2; } } -const [command, ...args] = commandLine; -if (command === "resolve-security-md") { - process.exitCode = resolveSecurityMdCommand(args, posixHome); -} else { - console.error( - "Usage: launch_codex_security_mcp[.cmd] --helper resolve-security-md [options]", - ); - process.exitCode = 2; -} diff --git a/plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs b/plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs index 94022da4f..2b2e67842 100644 --- a/plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs +++ b/plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs @@ -1,7 +1,8 @@ #!/usr/bin/env node +import { existsSync, realpathSync } from "node:fs"; import { copyFile, mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; -import { pathToFileURL } from "node:url"; +import { fileURLToPath } from "node:url"; import { brotliCompressSync, constants as zlibConstants } from "node:zlib"; import { execFileSync } from "node:child_process"; import { build } from "esbuild"; @@ -41,6 +42,7 @@ export async function buildMcpApp({ output }) { loader: { ".md": "text" }, logLevel: "info", logOverride: { "empty-import-meta": "silent" }, + nodePaths: [join(root, "node_modules")], outfile: bundle, platform: "node", target: "node20" @@ -68,8 +70,8 @@ export async function buildMcpApp({ output }) { const invokedPath = process.argv[1]; if ( - invokedPath !== undefined - && pathToFileURL(resolve(invokedPath)).href === import.meta.url + invokedPath !== undefined && existsSync(invokedPath) + && realpathSync(invokedPath) === realpathSync(fileURLToPath(import.meta.url)) ) { const args = process.argv.slice(2); if (args.length !== 2 || args[0] !== "--output") { @@ -108,5 +110,6 @@ const runtimeModule = new Module(loaderPath); runtimeModule.filename = loaderPath; runtimeModule.paths = Module._nodeModulePaths(dirname(loaderPath)); runtimeModule._compile(runtimeSource, loaderPath); +export default runtimeModule.exports; `; } diff --git a/plugins/codex-security/mcp-app/server.ts b/plugins/codex-security/mcp-app/server.ts index a21fbb627..f1a034eb6 100644 --- a/plugins/codex-security/mcp-app/server.ts +++ b/plugins/codex-security/mcp-app/server.ts @@ -22,6 +22,7 @@ import { DeepScanStartLock, startOrJoinDeepScanCoordinator } from "./src/deep-scan/registry.js"; +import { captureDeepScanExecutionSettings, loadDeepScanExecutionSettings, restoredDeepScanWorkerSettings, type DeepScanLegacySettingsContext } from "./src/deep-scan/recovery-settings.js"; import { CodexSdkWorkerExecutor } from "./src/deep-scan/executor.js"; import { CODEX_SANDBOX_STATE_META_CAPABILITY, @@ -750,6 +751,27 @@ export function createCodexSecurityServer(): McpServer { registry: deepScanCoordinators, options: { store: deepScanStore, + prepareExecutor: async (run) => new CodexSdkWorkerExecutor({ + ...restoredDeepScanWorkerSettings( + begun.shouldStart + ? await captureDeepScanExecutionSettings(run, parentSandbox, process.env, + { threadId, startedAt: run.createdAt }) + : await loadDeepScanExecutionSettings(run.scanDir, run, async () => { + const context = await runWorkbench(["get-scan", "--scan-id", run.scanId]); + const recipe = context.recipe as Pick | undefined; + const scan = context.scan as { executionAttribution?: { owner: DeepScanRunState["usageOwner"] } }; + return { config: recipe?.config, usageOwner: scan.executionAttribution?.owner }; + }), + parentSandbox + ), + artifactContext: { + pluginRoot: PLUGIN_ROOT, + scanRoot: run.scanDir, + repoRoot: run.targetPath, + scanId: run.scanId, + scope: run.scope + } + }), executor: new CodexSdkWorkerExecutor({ ...modelSettings, parentSandbox, @@ -765,7 +787,7 @@ export function createCodexSecurityServer(): McpServer { log: logDeepScanEvent, handoffClaimToken, threadId, - onComplete: async (draft, signal) => { + onComplete: async (draft, signal, publication) => { const context = await createScanArtifactContext( begun.run.scanId, runWorkbench, @@ -779,7 +801,7 @@ export function createCodexSecurityServer(): McpServer { await recordCodexSecurityScanDraftViaWorkbench(context, { ...draft, ...(handoffClaimToken === undefined ? {} : { handoffClaimToken }) - }, runWorkbench, signal); + }, runWorkbench, signal, publication); }, onStopped: async (run) => { await runWorkbench([ @@ -796,12 +818,26 @@ export function createCodexSecurityServer(): McpServer { invocationFailure: toolErrorResult(deepScanInvocationFailureMessage(error)) })); if ("invocationFailure" in preparation) return preparation.invocationFailure; - if (preparation.immediate) return preparation.immediate; + const completeSelectedParent = async (run: DeepScanRunState) => { + if (run.finalizationInput && run.status === "succeeded") { + await runWorkbench([ + "complete-scan", "--scan-id", run.scanId, "--thread-id", threadId, + ...optionalArg("--claim-token", handoffClaimToken), + ]); + } + }; + if (preparation.immediate) { + try { await completeSelectedParent(preparation.begun.run); } + catch (error) { return toolErrorResult(deepScanInvocationFailureMessage(error)); } + return preparation.immediate; + } const { begun, coordinator, joined } = preparation; if (joined) { logDeepScanEvent({ event: "coordinator_joined", scanId: begun.run.scanId }); } const terminal = await coordinator.wait(abortSignalFromExtra(extra)); + try { await completeSelectedParent(terminal); } + catch (error) { return toolErrorResult(deepScanInvocationFailureMessage(error)); } const result = deepScanTerminalResult(terminal); if (!result) { return toolErrorResult(deepScanInvocationFailureMessage( @@ -1632,12 +1668,14 @@ function logDeepScanEvent(event: { async function runWorkbench( args: string[], - input?: string | Buffer + input?: string | Buffer, + selectFinalization = false, + withExecutionSettings = false, ): Promise { let pythonCommand: string | undefined; try { pythonCommand = await resolvePythonCommand(); - return await executeWorkbenchWithStateSelection(pythonCommand, args, input); + return await executeWorkbenchWithStateSelection(pythonCommand, args, input, selectFinalization, withExecutionSettings); } catch (error) { const launchError = pythonCommand ? missingPythonHelperMessage(error, pythonCommand) @@ -1655,36 +1693,38 @@ async function runWorkbench( async function executeWorkbenchWithStateSelection( pythonCommand: string, args: string[], - input?: string | Buffer + input?: string | Buffer, + selectFinalization = false, + withExecutionSettings = false, ): Promise { if (WORKBENCH_COMMANDS_WITHOUT_DATABASE.has(args[0] ?? "")) { - return await executeWorkbench(pythonCommand, args, undefined, input); + return await executeWorkbench(pythonCommand, args, undefined, input, selectFinalization, withExecutionSettings); } if (CONFIGURED_WORKBENCH_STATE_DIR) { - return await executeWorkbench(pythonCommand, args, undefined, input); + return await executeWorkbench(pythonCommand, args, undefined, input, selectFinalization, withExecutionSettings); } if (fallbackWorkbenchStateDir) { - return await executeWorkbench(pythonCommand, args, await fallbackWorkbenchStateDir, input); + return await executeWorkbench(pythonCommand, args, await fallbackWorkbenchStateDir, input, selectFinalization, withExecutionSettings); } if (persistentWorkbenchStateSucceeded) { - return await executeWorkbench(pythonCommand, args, undefined, input); + return await executeWorkbench(pythonCommand, args, undefined, input, selectFinalization, withExecutionSettings); } return await withWorkbenchStateSelectionLock(async () => { if (fallbackWorkbenchStateDir) { - return await executeWorkbench(pythonCommand, args, await fallbackWorkbenchStateDir, input); + return await executeWorkbench(pythonCommand, args, await fallbackWorkbenchStateDir, input, selectFinalization, withExecutionSettings); } if (persistentWorkbenchStateSucceeded) { - return await executeWorkbench(pythonCommand, args, undefined, input); + return await executeWorkbench(pythonCommand, args, undefined, input, selectFinalization, withExecutionSettings); } try { - const result = await executeWorkbench(pythonCommand, args, undefined, input); + const result = await executeWorkbench(pythonCommand, args, undefined, input, selectFinalization, withExecutionSettings); persistentWorkbenchStateSucceeded = true; return result; } catch (error) { if (!isUnwritableSqliteOpenError(error)) throw error; const fallbackStateDir = await pinFallbackWorkbenchStateDir(); logWorkbenchStateFallback(); - return await executeWorkbench(pythonCommand, args, fallbackStateDir, input); + return await executeWorkbench(pythonCommand, args, fallbackStateDir, input, selectFinalization, withExecutionSettings); } }); } @@ -1707,7 +1747,9 @@ async function executeWorkbench( pythonCommand: string, args: string[], stateDir?: string, - input?: string | Buffer + input?: string | Buffer, + selectFinalization = false, + withExecutionSettings = false, ): Promise { const userContextIndex = args.indexOf("--user-context"); const userContext = userContextIndex === -1 ? undefined : args[userContextIndex + 1]; @@ -1716,7 +1758,12 @@ async function executeWorkbench( workbenchArgs.splice(userContextIndex, 2, "--user-context-stdin"); } const workbenchInput = input ?? userContext; - const execution = execFileAsync(pythonCommand, [workbenchScriptPath(), ...workbenchArgs], { + const internalInvocation = selectFinalization ? "select_finalization=True" + : withExecutionSettings ? "with_execution_settings=True" : undefined; + const pythonArgs = internalInvocation + ? ["-c", `import runpy, sys; script = sys.argv.pop(1); runpy.run_path(script)['main'](${internalInvocation})`, workbenchScriptPath(), ...workbenchArgs] + : [workbenchScriptPath(), ...workbenchArgs]; + const execution = execFileAsync(pythonCommand, pythonArgs, { cwd: PLUGIN_ROOT, env: stateDir ? { ...process.env, CODEX_SECURITY_STATE_DIR: stateDir } diff --git a/plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts b/plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts index b8e147ed9..7749d0ceb 100644 --- a/plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts +++ b/plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts @@ -1,4 +1,4 @@ -import { join } from "node:path"; +import { dirname, join, relative, sep } from "node:path"; import type { ZodType } from "zod/v4"; import commonSchema from "../../schemas/definitions/artifact-common.schema.json"; import reducerSchema from "../../schemas/tools/deep-reducer.schema.json"; @@ -20,7 +20,9 @@ import { type DeepScanArtifacts } from "./deep-scan/artifacts.js"; import { + deepReductionForPersistence, parseDeepReduction, + projectDiscoveryCoverage, reconcileDeepReduction, type DeepReductionInput, type DeepReductionSources, @@ -54,6 +56,18 @@ interface BoundReducer { /** Read the findings and scan context assigned to this reducer. */ export async function getCodexSecurityDeepReducerInputs( context: ArtifactContext +): Promise { + const inputs = await readDeepReductionSources(context); + const { sourceCoverage: _coverage, ...previous } = inputs.previous ?? {}; + return { + discoveries: inputs.discoveries.map(({ workerId, result }) => ({ workerId, result })), + previous: inputs.previous === null ? null : previous as DeepReductionInput, + }; +} + +/** Capture host coverage alongside the reducer's immutable finding inputs. */ +export async function readDeepReductionSources( + context: ArtifactContext ): Promise { return withLogicalReducerErrors(context, async () => { const bound = bindDeepReducer(context); @@ -73,8 +87,13 @@ export async function getCodexSecurityDeepReducerInputs( sourceFindingIds: [`${worker.id}:${index}`], }, })); - const { coverage: _coverage, ...reduction } = result; - return { workerId: worker.id, result: reduction }; + const { coverage, ...reduction } = result; + return { + workerId: worker.id, + ...(worker.attempt === undefined ? {} : { attempt: worker.attempt }), + coverage: projectDiscoveryCoverage(coverage, worker, relative(bound.artifacts.scanDir, worker.artifactDir ?? dirname(worker.resultPath)).split(sep).join("/")), + result: reduction, + }; })); const previous = await readPreviousReduction(bound); const scanId = bound.scanId ?? previous?.scanId ?? discoveries[0]?.result.scanId; @@ -107,7 +126,7 @@ export async function recordCodexSecurityDeepReduction( const submitted = deepReductionInputSchema.parse(input); let reduction = parseDeepReduction(submitted); if (reduction.complete === false) throw new Error("Deep reduction is only a checkpoint, not a complete result."); - const inputs = await getCodexSecurityDeepReducerInputs(context); + const inputs = await readDeepReductionSources(context); const expectedScanId = bound.scanId ?? inputs.previous?.scanId ?? inputs.discoveries[0]?.result.scanId; @@ -116,8 +135,9 @@ export async function recordCodexSecurityDeepReduction( } reduction = reconcileDeepReduction(reduction, inputs.discoveries, inputs.previous); - await saveScanDraftCheckpoint(context, reduction); - await writeJsonAtomic(bound.resultPath, reduction); + const persisted = deepReductionForPersistence(reduction, bound.state.persistSourceCoverage); + await saveScanDraftCheckpoint(context, persisted); + await writeJsonAtomic(bound.resultPath, persisted); return { findingCount: reduction.findings.length, consumedWorkerIds: bound.state.claimedWorkers.map((worker) => worker.id) diff --git a/plugins/codex-security/mcp-app/src/artifact-io.ts b/plugins/codex-security/mcp-app/src/artifact-io.ts index 3208f6d8a..66a7e8f38 100644 --- a/plugins/codex-security/mcp-app/src/artifact-io.ts +++ b/plugins/codex-security/mcp-app/src/artifact-io.ts @@ -5,12 +5,16 @@ import { dirname, isAbsolute, join, resolve, sep } from "node:path"; export interface DeepReducerWorkerContext { id: string; resultPath: string; + /** Original output owner for relative evidence, including accepted checkpoints. */ + artifactDir?: string; + attempt?: number; } export interface DeepReducerContext { scanRoot: string; claimedWorkers: DeepReducerWorkerContext[]; previousReducerResultPath?: string; + persistSourceCoverage?: boolean; } /** diff --git a/plugins/codex-security/mcp-app/src/artifact-scan-draft.ts b/plugins/codex-security/mcp-app/src/artifact-scan-draft.ts index 4c560822c..651ff1f10 100644 --- a/plugins/codex-security/mcp-app/src/artifact-scan-draft.ts +++ b/plugins/codex-security/mcp-app/src/artifact-scan-draft.ts @@ -4,6 +4,7 @@ import { dirname, join, sep } from "node:path"; import type * as z from "zod/v4"; import commonSchema from "../../schemas/definitions/artifact-common.schema.json"; import scanDraftDocument from "../../schemas/tools/scan-draft.schema.json"; +import scanManifestDocument from "../../schemas/scan-manifest.schema.json"; import type { ArtifactContext } from "./artifact-context.js"; import type { RunArtifactWorkbench } from "./artifact-context.js"; import { @@ -17,17 +18,10 @@ import { type SchemaDocument, } from "./artifact-schema-loader.js"; -type JsonObject = Record; +import type { ScanDraftInput } from "../../../../sdk/typescript/src/accepted-audit.js"; +export type { ScanDraftInput } from "../../../../sdk/typescript/src/accepted-audit.js"; -export interface ScanDraftInput { - scanId: string; - complete?: boolean; - handoffClaimToken?: string; - scope?: JsonObject; - threatModel?: JsonObject; - findings: JsonObject[]; - coverage: JsonObject; -} +type JsonObject = Record; export interface CompletedScanInput { scanId: string; @@ -55,6 +49,12 @@ interface PreparedScanDraft { coverage: JsonObject; } +/** Host-selected Deep aggregate, separate from model-authored draft fields. */ +export interface DeepScanPublication { + coordinatorGeneration?: number; + resultPath: string | null; +} + type PublishScanDraft = ( draft: PreparedScanDraft, expectedDigest: string | undefined, @@ -69,6 +69,19 @@ export const scanDraftInputSchema = loadArtifactZodSchema( "scanDraftInput", ) as z.ZodType; +// Sealed documents retain the existing public ID contract; live drafts require UUIDs. +const canonicalScanDraftInputSchema = loadArtifactZodSchema( + [commonSchema, { + ...scanDraftDocument, + $defs: { + ...scanDraftDocument.$defs, + scanId: scanManifestDocument.properties.scan.properties.id, + }, + }] as SchemaDocument[], + scanDraftDocument.$id, + "scanDraftInput", +) as z.ZodType; + export const completedScanInputSchema = loadArtifactZodSchema( schemaDocuments, scanDraftDocument.$id, @@ -165,6 +178,7 @@ export async function recordCodexSecurityScanDraftViaWorkbench( input: ScanDraftInput, runWorkbench: RunArtifactWorkbench, signal?: AbortSignal, + publication?: DeepScanPublication, ): Promise { return recordCodexSecurityScanDraft( context, @@ -184,7 +198,10 @@ export async function recordCodexSecurityScanDraftViaWorkbench( const { handoffClaimToken: _claim, ...snapshot } = checkpoint; await Promise.all([ replaceArtifactJson(checkpointPath, snapshot), - replaceArtifactJson(draftPath, draft), + replaceArtifactJson(draftPath, { + ...draft, + ...(publication === undefined ? {} : { deepScanPublication: publication }), + }), ]); const arguments_ = [ "write-scan-draft", @@ -548,32 +565,9 @@ async function readPreviousScanDraft( const manifest = parseJsonObject(contents[0]!, "previous scan draft manifest"); const findings = parseJsonObject(contents[1]!, "previous scan draft findings"); const coverage = parseJsonObject(contents[2]!, "previous scan draft coverage"); - const scan = requireObject(manifest.scan, "previous scan draft.scan"); - const semanticScope = isObject(scan.scope) ? { ...scan.scope } : undefined; - if (semanticScope) { - delete semanticScope.includePaths; - delete semanticScope.excludePaths; - } - const semanticCoverage = { ...coverage }; - for (const field of ["documentType", "schemaVersion", "scanId", "mode", "includePaths", "excludePaths", "receiptRefs", "inventoryStrategy"]) delete semanticCoverage[field]; return { digest, - input: parsePersistedScanDraft({ - scanId: context.scanId, - ...(scan.complete === false ? { complete: false } : {}), - ...(semanticScope && Object.keys(semanticScope).length > 0 - ? { scope: semanticScope } - : {}), - ...(isObject(scan.threatModel) - ? { threatModel: structuredClone(scan.threatModel) } - : {}), - findings: (findings.findings as JsonObject[]).map((finding) => { - const semantic = { ...finding }; - for (const field of ["findingId", "occurrenceId", "fingerprints"]) delete semantic[field]; - return semantic; - }), - coverage: semanticCoverage, - }), + input: parseCanonicalScanDraft({ scanId: context.scanId, manifest, findings, coverage }), }; } @@ -973,8 +967,35 @@ export async function getCodexSecurityCompletedScan( return { scanId: parsed.scanId, manifest, findings, coverage }; } +/** Project canonical documents through the same semantic parser as worker drafts. */ +export function parseCanonicalScanDraft(input: { + scanId?: string; + manifest: JsonObject; + findings: JsonObject; + coverage: JsonObject; +}): ScanDraftInput { + const scan = requireObject(input.manifest.scan, "scan draft manifest.scan"); + for (const scanId of [scan.id, input.findings.scanId, input.coverage.scanId]) { + if (scanId !== undefined && scanId !== input.scanId) { + throw new Error("scan draft: canonical documents belong to a different scan."); + } + } + return parsePersistedCheckpoint({ + scanId: input.scanId, + ...(scan.complete === undefined ? {} : { complete: scan.complete }), + ...(scan.scope === undefined ? {} : { scope: scan.scope }), + ...(scan.threatModel === undefined ? {} : { threatModel: scan.threatModel }), + findings: input.findings.findings, + coverage: input.coverage, + }, canonicalScanDraftInputSchema); +} + export function parseScanDraft(input: ScanDraftInput): ScanDraftInput { - const parsed = scanDraftInputSchema.parse(input); + return parseSemanticScanDraft(input, scanDraftInputSchema); +} + +function parseSemanticScanDraft(input: unknown, schema: z.ZodType): ScanDraftInput { + const parsed = schema.parse(input); validateFindingSemantics(parsed.findings); validateCoverageSemantics(parsed.coverage); return parsed; @@ -984,18 +1005,23 @@ export function parseScanDraft(input: ScanDraftInput): ScanDraftInput { export function parsePersistedScanDraft( input: Record ): ScanDraftInput { + return parsePersistedDraft(input, scanDraftInputSchema); +} + +function parsePersistedDraft(input: Record, schema: z.ZodType): ScanDraftInput { const compatible = structuredClone(input); - if (!Array.isArray(compatible.findings)) { - return parseScanDraft(compatible as unknown as ScanDraftInput); - } - for (const finding of compatible.findings) { - if (!isObject(finding)) continue; - normalizePersistedFindingDetails(finding); + if (Array.isArray(compatible.findings)) { + for (const finding of compatible.findings) { + if (isObject(finding)) normalizePersistedFindingDetails(finding); + } } - return parseScanDraft(compatible as unknown as ScanDraftInput); + return parseSemanticScanDraft(compatible, schema); } -function parsePersistedCheckpoint(input: Record): ScanDraftInput { +function parsePersistedCheckpoint( + input: Record, + schema = scanDraftInputSchema, +): ScanDraftInput { const compatible = structuredClone(input); if (isObject(compatible.scope)) { delete compatible.scope.includePaths; @@ -1022,7 +1048,7 @@ function parsePersistedCheckpoint(input: Record): ScanDraftInpu delete finding.fingerprints; } } - return parsePersistedScanDraft(compatible); + return parsePersistedDraft(compatible, schema); } function normalizePersistedFindingDetails(finding: JsonObject): void { diff --git a/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts b/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts index 291557a9e..099add31b 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts @@ -10,10 +10,13 @@ import { import { readJsonObject, requireRegularFile, writeJsonAtomic } from "./artifacts.js"; import type { DeepScanArtifacts } from "./artifacts.js"; -export type DeepReductionInput = Omit; +export type DeepReductionInput = Omit & { + /** Host projection of accepted source coverage; never supplied by the reducer. */ + sourceCoverage?: ScanDraftInput["coverage"]; +}; export interface DeepReductionSources { - discoveries: { workerId: string; result: DeepReductionInput }[]; + discoveries: { workerId: string; attempt?: number; coverage?: ScanDraftInput["coverage"]; result: DeepReductionInput }[]; previous: DeepReductionInput | null; } @@ -22,6 +25,21 @@ export interface ReducerArtifactValidation { result: DeepReductionInput; } +export function deepReductionToScanDraft(result: DeepReductionInput): ScanDraftInput { + const { sourceCoverage, ...draft } = structuredClone(result); + return { ...draft, coverage: sourceCoverage ?? unknownSourceCoverage() }; +} + +/** Older workflow readers reject the host field; retain their persisted shape. */ +export function deepReductionForPersistence( + result: DeepReductionInput, + persistSourceCoverage = false, +): DeepReductionInput { + if (persistSourceCoverage) return result; + const { sourceCoverage: _coverage, ...legacy } = result; + return legacy; +} + /** * Check reducer findings with the Standard scan validator. * It requires coverage, so add an empty value and remove it after validation. @@ -30,8 +48,9 @@ export function parseDeepReduction( input: Record, persisted = false, ): DeepReductionInput { + const { sourceCoverage, ...submitted } = input; const standard = { - ...input, + ...submitted, coverage: { completeness: "complete", surfaces: [], @@ -42,6 +61,9 @@ export function parseDeepReduction( const { coverage: _coverage, ...parsed } = persisted ? parsePersistedScanDraft(standard) : parseScanDraft(standard as unknown as ScanDraftInput); + if (persisted && sourceCoverage !== undefined) { + return { ...parsed, sourceCoverage: parsePersistedScanDraft({ ...standard, coverage: sourceCoverage }).coverage }; + } return parsed; } @@ -50,6 +72,16 @@ export async function validateDiscoveryArtifacts( artifacts: DeepScanArtifacts, resultPath: string, expectedScanId: string +): Promise { + const result = await readDiscoveryAuditDraft(artifacts, resultPath, expectedScanId); + if (result.complete === false) throw new Error("Standard scan worker wrote only a checkpoint; its audit is not complete."); + return result; +} + +export async function readDiscoveryAuditDraft( + artifacts: DeepScanArtifacts, + resultPath: string, + expectedScanId: string, ): Promise { await requireRegularFile(resultPath, artifacts.workersRoot); const result = parseStoredScanDraft( @@ -58,7 +90,6 @@ export async function validateDiscoveryArtifacts( expectedScanId, parsePersistedScanDraft ); - if (result.complete === false) throw new Error("Standard scan worker wrote only a checkpoint; its audit is not complete."); return result; } @@ -70,6 +101,7 @@ export async function validateReducerArtifacts(input: { reducerId: string; previousReducerResultPath?: string; sources?: DeepReductionSources; + persistSourceCoverage?: boolean; }, expectedScanId?: string): Promise { const { artifacts, @@ -101,8 +133,9 @@ export async function validateReducerArtifacts(input: { if (input.sources) { result = reconcileDeepReduction(result, input.sources.discoveries, input.sources.previous); - await saveScanDraftCheckpoint({ root: artifactDir, repoRoot: artifacts.scanDir, layout: "reducer" }, result); - await writeJsonAtomic(resultPath, result); + const persisted = deepReductionForPersistence(result, input.persistSourceCoverage); + await saveScanDraftCheckpoint({ root: artifactDir, repoRoot: artifacts.scanDir, layout: "reducer" }, persisted); + await writeJsonAtomic(resultPath, persisted); } else { validateRetainedFindings(result, [], previous); } @@ -122,6 +155,7 @@ export function reconcileDeepReduction( previous: DeepReductionInput | null, ): DeepReductionInput { const result = structuredClone(input); + result.sourceCoverage = aggregateSourceCoverage(discoveries, previous); if (result.complete === false) throw new Error("Deep reduction is only a checkpoint, not a complete result."); for (const source of [...discoveries.map((discovery) => discovery.result), ...(previous ? [previous] : [])]) { if (source.scanId !== result.scanId) throw new Error("Deep reduction source belongs to a different scan."); @@ -180,6 +214,79 @@ export function reconcileDeepReduction( return result; } +/** Keep independent reviews separate: matching labels do not resolve another pass's proof gap. */ +export function aggregateSourceCoverage( + discoveries: DeepReductionSources["discoveries"], + previous: DeepReductionInput | null, +): ScanDraftInput["coverage"] { + const sources = [ + ...(previous ? [previous.sourceCoverage ?? unknownSourceCoverage()] : []), + ...discoveries.map((source) => source.coverage ?? unknownSourceCoverage()), + ]; + const result: ScanDraftInput["coverage"] = { + completeness: "complete", surfaces: [], explicitExclusions: [], deferred: [], reviews: [], + }; + for (const field of ["surfaces", "explicitExclusions", "deferred", "openQuestions", "reviews"]) { + const entries = sources.flatMap((source) => (source[field] as unknown[] | undefined) ?? []); + if (entries.length || field !== "openQuestions") result[field] = structuredClone(entries); + } + if (sources.some((source) => source.completeness === "partial") + || (result.deferred as unknown[]).length > 0 + || (result.surfaces as Record[]).some((surface) => surface.disposition === "needs_follow_up")) { + result.completeness = "partial"; + } else if (sources.some((source) => source.completeness === "unknown")) { + result.completeness = "unknown"; + } + return result; +} + +function unknownSourceCoverage(): ScanDraftInput["coverage"] { + return { completeness: "unknown", surfaces: [], explicitExclusions: [], deferred: [] }; +} + +/** Qualify worker-local IDs and receipt paths before combining accepted coverage. */ +export function projectDiscoveryCoverage( + coverage: ScanDraftInput["coverage"], + worker: { id: string; attempt?: number }, + artifactPrefix: string, +): ScanDraftInput["coverage"] { + const provenance = { workerId: worker.id, ...(worker.attempt === undefined ? {} : { attempt: worker.attempt }) }; + const prefix = `${worker.id}-attempt-${worker.attempt ?? "unknown"}`; + const surfaces = coverage.surfaces as Record[]; + const surfaceIds = new Map(surfaces.map((surface, index) => [surface.id, `${prefix}-surface-${index + 1}`])); + const project = (item: Record) => ({ + ...structuredClone(item), + provenance: { + ...provenance, + ...(item.id === undefined ? {} : { sourceId: item.id }), + ...(item.candidateId === undefined ? {} : { candidateId: item.candidateId }), + }, + }); + return { + completeness: coverage.completeness, + reviews: [{ ...provenance, completeness: coverage.completeness }], + surfaces: surfaces.map((surface, index) => ({ + ...project(surface), + id: `${prefix}-surface-${index + 1}`, + receiptRefs: ((surface.receiptRefs as string[] | undefined) ?? []).map((ref) => `${artifactPrefix}/${ref}`), + })), + explicitExclusions: (coverage.explicitExclusions as Record[]).map(project), + deferred: (coverage.deferred as Record[]).map((item, index) => ({ + ...project(item), + id: `${prefix}-deferred-${index + 1}`, + ...(item.candidateId === undefined ? {} : { candidateId: `${prefix}-candidate-${index + 1}` }), + ...(item.surfaceIds === undefined ? {} : { + surfaceIds: (item.surfaceIds as string[]).map((id) => surfaceIds.get(id) ?? id), + }), + })), + ...(coverage.openQuestions === undefined ? {} : { + openQuestions: (coverage.openQuestions as (string | Record)[]).map((question) => ( + project(typeof question === "string" ? { question } : question) + )), + }), + }; +} + function findingSourceIds(finding: Record): string[] { const provenance = finding.provenance as Record; const ids = provenance.sourceFindingIds; diff --git a/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts b/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts index a76f737f5..34210a97d 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts @@ -1,3 +1,4 @@ +import { publishSelectedDeepScan } from "./finalization.js"; import { randomUUID } from "node:crypto"; import { promises as fs } from "node:fs"; import { basename, dirname, join } from "node:path"; @@ -5,22 +6,20 @@ import { createDeepScanArtifacts, ensureDeepScanDirectories } from "./artifacts.js"; -import { validateDiscoveryArtifacts, validateReducerArtifacts, type DeepReductionInput } from "./artifact-validation.js"; +import { aggregateSourceCoverage, deepReductionToScanDraft, validateDiscoveryArtifacts, validateReducerArtifacts, type DeepReductionInput } from "./artifact-validation.js"; +import { readDeepReductionSources } from "../artifact-deep-reducer.js"; import { scanDraftInputSchema, + type DeepScanPublication, type ScanDraftInput } from "../artifact-scan-draft.js"; import type { DeepScanArtifacts } from "./artifacts.js"; -import { - DeepScanWorkerRunner, - sha256 -} from "./worker-runner.js"; +import { DeepScanWorkerRunner } from "./worker-runner.js"; import type { AcceptedDiscovery, DedupOutcome, DiscoveryOutcome, - SuccessfulDedupOutcome, - WorkerExecutionAudit + SuccessfulDedupOutcome } from "./worker-runner.js"; import { boundedDeepScanErrorPair, @@ -58,20 +57,11 @@ interface SchedulerResult { mergedWorkerIds: string[]; reducers: AcceptedReducer[]; result?: DeepReductionInput; + resultPath?: string; } type CoordinatorPhase = "setup" | "discovery" | "terminal"; -interface SchedulerAudit { - accepted: AcceptedDiscovery[]; - mergedWorkerIds: string[]; - omittedWorkerIds: string[]; - canceledWorkerIds: string[]; - bufferedWorkerIds: string[]; - reducers: AcceptedReducer[]; - executions: WorkerExecutionAudit[]; -} - export interface CoordinatorOptions { run: DeepScanRunState; store: DeepScanStore; @@ -86,7 +76,7 @@ export interface CoordinatorOptions { threadId?: string; heartbeatIntervalMs?: number; observeReplacement?: (run: DeepScanRunState) => Promise; - onComplete?: (draft: ScanDraftInput, signal: AbortSignal) => Promise; + onComplete?: (draft: ScanDraftInput, signal: AbortSignal, publication: DeepScanPublication) => Promise; onStopped?: (run: DeepScanRunState) => Promise; } @@ -124,15 +114,6 @@ export class DeepScanCoordinator { private externallyFailed = false; private phase: CoordinatorPhase = "setup"; private discoveryDeadlineReached = false; - private readonly audit: SchedulerAudit = { - accepted: [], - mergedWorkerIds: [], - omittedWorkerIds: [], - canceledWorkerIds: [], - bufferedWorkerIds: [], - reducers: [], - executions: [] - }; private state: DeepScanRunState; constructor(private readonly options: CoordinatorOptions) { @@ -149,10 +130,7 @@ export class DeepScanCoordinator { clock: this.clock, random: options.random ?? Math.random, log: this.log, - retryDelaysMs: options.retryDelaysMs ?? RETRY_DELAYS_MS, - recordExecution: (execution) => { - this.audit.executions.push(execution); - } + retryDelaysMs: options.retryDelaysMs ?? RETRY_DELAYS_MS } satisfies Omit[0], "signal">; this.workers = new DeepScanWorkerRunner({ ...workerOptions, @@ -175,7 +153,7 @@ export class DeepScanCoordinator { if (this.started) return; this.started = true; this.log({ event: "coordinator_started", scanId: this.state.scanId }); - this.scheduleDiscoveryDeadline(); + if (!this.state.finalizationInput) this.scheduleDiscoveryDeadline(); this.scheduleHeartbeat(); void this.run().catch((error: unknown) => { this.log({ @@ -288,22 +266,17 @@ export class DeepScanCoordinator { await ensureDeepScanDirectories(this.artifacts); if (this.canceled || this.externallyFailed) return; + if (this.state.finalizationInput) { + this.phase = "terminal"; + await this.completeSelectedFinalization(); + return; + } this.phase = "discovery"; const schedulerResult = await this.runScheduler(); if (this.canceled || this.externallyFailed) return; this.phase = "terminal"; const draft = schedulerResult.result - ? { - ...structuredClone(schedulerResult.result), - // Readers require coverage.json. The coordinator has accepted this - // result, so mark it complete and leave review notes empty. - coverage: { - completeness: "complete", - surfaces: [], - explicitExclusions: [], - deferred: [] - } - } + ? deepReductionToScanDraft(schedulerResult.result) : scanDraftInputSchema.parse({ scanId: this.state.scanId, findings: [], @@ -319,9 +292,17 @@ export class DeepScanCoordinator { if (draft.scanId !== this.state.scanId) { throw new Error("Deep Scan aggregate does not match its authoritative scan identity."); } - await this.options.onComplete?.(draft, this.publicationAbortController.signal); + await this.options.onComplete?.(draft, this.publicationAbortController.signal, { + coordinatorGeneration: this.state.coordinatorGeneration, + resultPath: schedulerResult.resultPath ?? null, + }); if (this.canceled || this.externallyFailed) return; - this.state = await this.finishWithReplay(schedulerResult); + this.state = await this.options.store.finish({ + scanId: this.state.scanId, + reason: schedulerResult.reason, + manifestPath: join(this.state.scanDir, "scan-manifest.json"), + omittedWorkerIds: schedulerResult.omittedWorkerIds, + }); if (this.canceled || this.externallyFailed) return; this.log({ event: "coordinator_terminal", @@ -341,6 +322,12 @@ export class DeepScanCoordinator { await this.settleSchedulerWork(); return; } + if (this.state.finalizationInput) { + // Publication can be retried from the committed input without model work. + this.log({ event: "coordinator_publication_pending", scanId: this.state.scanId, reason: errorKind(error) }); + this.failLocally(error); + return; + } const message = errorMessage(error); const persistedMessage = boundedDeepScanErrorMessage(error); if (this.phase === "setup") { @@ -437,6 +424,17 @@ export class DeepScanCoordinator { } } + private async completeSelectedFinalization(): Promise { + this.state = await publishSelectedDeepScan({ + run: this.state, + artifacts: this.artifacts, + signal: this.publicationAbortController.signal, + publish: async (...args) => { await this.options.onComplete?.(...args); }, + finish: (input) => this.options.store.finish(input), + }); + this.finishLocally(this.state); + } + private finishLocally(state: DeepScanRunState): void { if (this.terminal) return; this.terminal = true; @@ -560,7 +558,12 @@ export class DeepScanCoordinator { && this.state.coordinatorGeneration !== undefined && current.coordinatorGeneration > this.state.coordinatorGeneration ); - if (current.status === "running" && !replacementConfirmed) return false; + if (current.status === "running" && !replacementConfirmed) { + // A selection response can be lost after its transaction commits. + if (current.finalizationInput) this.state = { ...this.state, + finalizationInput: current.finalizationInput, terminalReason: current.terminalReason }; + return false; + } this.externallyFailed = true; this.abortController.abort("deep_scan_coordinator_lease_lost"); @@ -610,10 +613,6 @@ export class DeepScanCoordinator { .filter((worker) => worker.kind === "discovery" && worker.status === "canceled") .map((worker) => worker.id); const omittedWorkerIds: string[] = []; - this.audit.accepted = [...accepted]; - this.audit.mergedWorkerIds = mergedDiscoveries.map((worker) => worker.id); - this.audit.canceledWorkerIds = [...canceledWorkerIds]; - this.audit.executions = await this.recoverPersistedExecutions(); const recoveredReducers = await this.recoverCompletedReducers(recovered); const reducerOutcomes = recoveredReducers.reducers; let latestResult = recoveredReducers.result; @@ -636,8 +635,6 @@ export class DeepScanCoordinator { let stopReason: DeepScanTerminalReason | undefined; let lastReplaceableFailure: Extract | undefined; - this.audit.bufferedWorkerIds = buffer.map((worker) => worker.id); - this.audit.reducers = [...reducerOutcomes]; const errorLimit = config.stopAfterConsecutiveErrors ?? config.stopAfterNoNew; let reducerFailures = persistedReducerFailureStreak(this.state.persistedWorkers ?? []); if (this.state.consecutiveErrors >= errorLimit) { @@ -733,10 +730,6 @@ export class DeepScanCoordinator { canceledWorkerIds.push(outcome.workerId); } } - this.audit.accepted = [...accepted]; - this.audit.omittedWorkerIds = unique(omittedWorkerIds); - this.audit.canceledWorkerIds = unique(canceledWorkerIds); - this.audit.bufferedWorkerIds = buffer.map((worker) => worker.id); return firstFailure; }; const reconcileReducerSettlement = async (): Promise => { @@ -750,7 +743,6 @@ export class DeepScanCoordinator { const outcome = result.value; if ("status" in outcome) { buffer = [...outcome.consumed, ...buffer].sort(compareCompletionSequence); - this.audit.bufferedWorkerIds = buffer.map((worker) => worker.id); return outcome.error; } this.state = outcome.run; @@ -759,9 +751,6 @@ export class DeepScanCoordinator { const { result: acceptedResult, ...metadata } = outcome; latestResult = acceptedResult; reducerOutcomes.push(metadata); - this.audit.reducers = [...reducerOutcomes]; - this.audit.mergedWorkerIds = unique(mergedDiscoveries.map((worker) => worker.id)); - this.audit.bufferedWorkerIds = buffer.map((worker) => worker.id); return undefined; }; @@ -809,7 +798,8 @@ export class DeepScanCoordinator { id: randomUUID(), label: `dedup-${String(reducerSequence).padStart(4, "0")}`, consumed, - previousReducerResultPath + previousReducerResultPath, + previousSourceCoverage: latestResult?.sourceCoverage, })); observe(reducer); } @@ -847,7 +837,6 @@ export class DeepScanCoordinator { ?? (this.state.consecutiveErrors ?? 0) + 1; this.state = { ...this.state, consecutiveErrors }; canceledWorkerIds.push(outcome.workerId); - this.audit.canceledWorkerIds = unique(canceledWorkerIds); this.log({ event: "discovery_worker_replaced", scanId: this.state.scanId, @@ -875,7 +864,6 @@ export class DeepScanCoordinator { } if (outcome.status === "canceled") { canceledWorkerIds.push(outcome.workerId); - this.audit.canceledWorkerIds = unique(canceledWorkerIds); if ( !this.abortController.signal.aborted && !this.discoveryAbortController.signal.aborted @@ -887,8 +875,6 @@ export class DeepScanCoordinator { accepted.push(outcome.worker); this.state = { ...this.state, consecutiveErrors: 0 }; buffer.push(outcome.worker); - this.audit.accepted = [...accepted]; - this.audit.bufferedWorkerIds = buffer.map((worker) => worker.id); this.logProgress(accepted.length); continue; } @@ -896,7 +882,6 @@ export class DeepScanCoordinator { reducer = undefined; if ("status" in outcome) { buffer = [...outcome.consumed, ...buffer].sort(compareCompletionSequence); - this.audit.bufferedWorkerIds = buffer.map((worker) => worker.id); reducerFailures += 1; this.log({ event: "dedup_worker_replaced", @@ -923,9 +908,6 @@ export class DeepScanCoordinator { const { result: acceptedResult, ...metadata } = outcome; latestResult = acceptedResult; reducerOutcomes.push(metadata); - this.audit.reducers = [...reducerOutcomes]; - this.audit.mergedWorkerIds = unique(mergedDiscoveries.map((worker) => worker.id)); - this.audit.bufferedWorkerIds = buffer.map((worker) => worker.id); if ( !this.discoveryDeadlineReached && outcome.run.noNewStreak >= config.stopAfterNoNew @@ -933,8 +915,6 @@ export class DeepScanCoordinator { ) { stopReason = "saturated"; canceledWorkerIds.push(...active.keys()); - this.audit.canceledWorkerIds = unique(canceledWorkerIds); - this.audit.bufferedWorkerIds = []; this.abortController.abort("deep_scan_saturated"); } } @@ -943,12 +923,6 @@ export class DeepScanCoordinator { // the manifest records which results completed and which were canceled. const lateFailure = await reconcileRemainingDiscoveries("omitted"); - this.audit.accepted = [...accepted]; - this.audit.mergedWorkerIds = unique(mergedDiscoveries.map((worker) => worker.id)); - this.audit.omittedWorkerIds = unique(omittedWorkerIds); - this.audit.canceledWorkerIds = unique(canceledWorkerIds); - this.audit.bufferedWorkerIds = buffer.map((worker) => worker.id); - // Once Deep reaches saturation, late worker errors cannot fail the scan. if (lateFailure && stopReason !== "saturated") throw lateFailure; @@ -966,6 +940,7 @@ export class DeepScanCoordinator { mergedWorkerIds: unique(mergedDiscoveries.map((worker) => worker.id)), reducers: reducerOutcomes, result: latestResult, + resultPath: previousReducerResultPath, }; } @@ -973,24 +948,29 @@ export class DeepScanCoordinator { const recovered: AcceptedDiscovery[] = []; for (const worker of this.state.persistedWorkers ?? []) { if (worker.kind !== "discovery" || worker.status !== "succeeded") continue; - if (!worker.resultManifestPath || !worker.completionSequence) { + // Migrated workers can have frozen merge inputs without an attempt record. + const claimedInput = this.state.persistedDedupInputs?.find((input) => ( + input.discoveryWorkerId === worker.id + && (input.attempt === undefined || input.attempt === worker.attempt) + && input.resultManifestPath + )); + const resultPath = worker.acceptedResultPath ?? claimedInput?.resultManifestPath ?? worker.resultManifestPath; + if (!resultPath || !worker.completionSequence) { throw new Error(`Accepted discovery ${worker.id} has incomplete persisted evidence.`); } await validateDiscoveryArtifacts( this.artifacts, - worker.resultManifestPath, + resultPath, this.state.scanId ); - const evidence = await persistedWorkerEvidence(worker); recovered.push({ id: worker.id, label: basename(dirname(worker.promptPath)), artifactDir: worker.artifactDir, - resultPath: worker.resultManifestPath, + resultPath, completionSequence: worker.completionSequence, attempt: worker.attempt, - ...(worker.threadId ? { threadId: worker.threadId } : {}), - ...evidence + ...(worker.threadId ? { threadId: worker.threadId } : {}) }); } return recovered.sort(compareCompletionSequence); @@ -1011,76 +991,70 @@ export class DeepScanCoordinator { )); let noNewStreak = 0; for (const worker of completedReducers) { - if (!worker.resultManifestPath) { + // A later merge claim can retain a legacy aggregate's accepted reference. + const resultPath = worker.acceptedResultPath + ?? this.state.persistedMergeClaims?.find((claim) => ( + claim.previousWorkerId === worker.id && claim.previousResultPath + ))?.previousResultPath + ?? worker.resultManifestPath; + if (!resultPath) { throw new Error(`Completed reducer ${worker.id} has no persisted result manifest.`); } const consumed = inputs .filter((input) => input.dedupWorkerId === worker.id) .sort((left, right) => left.inputOrder - right.inputOrder) - .map((input) => discoveriesById.get(input.discoveryWorkerId)); + .map((input) => { + const discovery = discoveriesById.get(input.discoveryWorkerId); + return discovery && { + ...discovery, + resultPath: input.resultManifestPath ?? discovery.resultPath, + attempt: input.attempt ?? discovery.attempt, + }; + }); if (consumed.length === 0 || consumed.some((value) => !value)) { throw new Error(`Completed reducer ${worker.id} has incomplete persisted inputs.`); } const accepted = consumed as AcceptedDiscovery[]; + const claim = this.state.persistedMergeClaims?.find((item) => item.workerId === worker.id); const { newFindings, result } = await validateReducerArtifacts({ artifacts: this.artifacts, artifactDir: worker.artifactDir, - resultPath: worker.resultManifestPath, + resultPath, reducerId: worker.id, - previousReducerResultPath: outcomes.at(-1)?.resultPath + previousReducerResultPath: claim ? claim.previousResultPath : outcomes.at(-1)?.resultPath }, this.state.scanId); + if (result.sourceCoverage === undefined) { + const context = { + root: worker.artifactDir, + repoRoot: this.state.targetPath, + scanId: this.state.scanId, + layout: "reducer" as const, + deepReducer: { + scanRoot: this.artifacts.scanDir, + claimedWorkers: accepted.map((source) => ({ + id: source.id, resultPath: source.resultPath, artifactDir: source.artifactDir, attempt: source.attempt, + })), + }, + }; + const sources = await readDeepReductionSources(context); + result.sourceCoverage = aggregateSourceCoverage(sources.discoveries, latestResult ?? null); + } latestResult = result; noNewStreak = newFindings > 0 ? 0 : noNewStreak + accepted.length; - const evidence = await persistedWorkerEvidence(worker); outcomes.push({ type: "dedup", id: worker.id, consumed: accepted, - resultPath: worker.resultManifestPath, + resultPath, newFindings, attempt: worker.attempt, ...(worker.threadId ? { threadId: worker.threadId } : {}), - ...evidence, run: { ...this.state, noNewStreak } }); } return { reducers: outcomes, result: latestResult }; } - private async recoverPersistedExecutions(): Promise { - const executions: WorkerExecutionAudit[] = []; - for (const worker of this.state.persistedWorkers ?? []) { - if ( - worker.kind === "setup" - || worker.status === "queued" - || worker.status === "running" - || (worker.status === "canceled" && worker.attempt === 0) - ) { - continue; - } - const replaceableFailure = persistedReplaceableFailure(worker); - const status = replaceableFailure || worker.status === "failed" - ? "failed" - : worker.status; - executions.push({ - id: worker.id, - label: basename(dirname(worker.promptPath)), - kind: worker.kind, - status, - attempt: worker.attempt, - ...(worker.threadId ? { threadId: worker.threadId } : {}), - promptPath: worker.promptPath, - artifactDir: worker.artifactDir, - ...await persistedWorkerEvidence(worker), - ...(status === "failed" && worker.error - ? { error: replaceableFailure?.message ?? worker.error } - : {}), - ...(replaceableFailure ? { failureKind: replaceableFailure.kind } : {}) - }); - } - return executions; - } - private reducerReady( buffer: AcceptedDiscovery[], previousReducerResultPath: string | undefined, @@ -1119,36 +1093,7 @@ export class DeepScanCoordinator { }); } - /** - * A workbench process can commit SQLite and still lose its stdout response. - * Replay the exact idempotent finish once before treating the run as failed; - * otherwise we could overwrite a successful terminal state after durable success. - */ - private async finishWithReplay(result: SchedulerResult): Promise { - const input = { - scanId: this.state.scanId, - reason: result.reason, - manifestPath: join(this.state.scanDir, "scan-manifest.json"), - omittedWorkerIds: result.omittedWorkerIds - }; - try { - return await this.options.store.finish(input); - } catch (firstError) { - this.log({ - event: "coordinator_finish_replay", - scanId: this.state.scanId, - reason: errorKind(firstError) - }); - try { - return await this.options.store.finish(input); - } catch (replayError) { - throw new Error( - `Deep Scan terminal persistence replay failed: ${errorMessage(replayError)}`, - { cause: firstError } - ); - } - } - } + } const systemClock: DeepScanClock = { @@ -1238,30 +1183,6 @@ function persistedReplaceableFailure( return undefined; } -async function persistedWorkerEvidence(worker: PersistedDeepScanWorker): Promise<{ - basePromptSha256: string; - attemptPromptPaths: string[]; -}> { - const attemptPromptPaths = [worker.promptPath]; - for (let attempt = 2; attempt <= worker.attempt; attempt += 1) { - const promptPath = join( - dirname(worker.promptPath), - "prompts", - `attempt-${String(attempt).padStart(2, "0")}.md` - ); - try { - await fs.access(promptPath); - attemptPromptPaths.push(promptPath); - } catch { - // Transient execution retries reuse the original prompt. - } - } - return { - basePromptSha256: sha256(await fs.readFile(worker.promptPath, "utf8")), - attemptPromptPaths - }; -} - function discoveryErrorLimitError( count: number, limit: number, diff --git a/plugins/codex-security/mcp-app/src/deep-scan/executor.ts b/plugins/codex-security/mcp-app/src/deep-scan/executor.ts index 788745ae4..51db8dedd 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/executor.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/executor.ts @@ -1,7 +1,11 @@ import { accessSync, constants as fsConstants, existsSync, promises as fs, readdirSync, statSync } from "node:fs"; import { createRequire } from "node:module"; import { delimiter, dirname, isAbsolute, join, resolve, win32 } from "node:path"; -import { Codex } from "@openai/codex-sdk"; +import { + createCodexClient, + readCodexSessionTurn +} from "../../../../../sdk/typescript/src/codex-session.js"; +import type { CodexOptions } from "@openai/codex-sdk"; import { parse as parseToml } from "smol-toml"; import { executablePathForSpawn } from "./executable-path.js"; import { @@ -22,6 +26,8 @@ import type { } from "./types.js"; export interface CodexSdkWorkerModelSettings { + /** Resolved by the execution owner, including when reconstructing a scan. */ + codexOptions?: CodexOptions; model?: string; reasoningEffort?: string; artifactContext?: CodexSdkWorkerArtifactContext; @@ -39,7 +45,7 @@ export interface CodexSdkWorkerArtifactContext { } export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { - private runtimeReasoningSummary?: Promise; + private runtimeModelConfig?: Promise>; constructor(private readonly modelSettings: CodexSdkWorkerModelSettings = {}) {} @@ -52,15 +58,32 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { ); } const workerProfile = workerPermissionProfile(parentSandbox); - const configOverrides = workerPermissionProfileConfigOverrides(workerProfile); + const resolved = this.modelSettings.codexOptions; const originalCwd = process.cwd(); - const childEnv = await snapshotWorkerEnvironment(); - // Snapshot the SDK's per-scan config once for this coordinator, including resumes. - const reasoningSummary = await (this.runtimeReasoningSummary ??= workerReasoningSummary(childEnv)); + const childEnv = await snapshotWorkerEnvironment(resolved?.env); + if (resolved?.apiKey !== undefined) childEnv.CODEX_API_KEY = resolved.apiKey; + // Snapshot per-scan selections once; a reconstructed owner can supply them. + // Native account credentials continue to refresh in the selected home. + const modelConfig: NonNullable = { + ...await (this.runtimeModelConfig ??= resolved?.config + ? Promise.resolve(resolved.config) + : workerModelConfig(childEnv)), + ...(this.modelSettings.model ? { model: this.modelSettings.model } : {}), + // The CLI can add effort levels before the pinned SDK widens ThreadOptions. + ...(this.modelSettings.reasoningEffort + ? { model_reasoning_effort: this.modelSettings.reasoningEffort } + : {}) + }; + const configOverrides = [ + ...(resolved?.configOverrides ?? []), + ...workerPermissionProfileConfigOverrides(workerProfile) + ]; const openAiApiKey = environmentVariable(childEnv, "OPENAI_API_KEY", process.platform)?.trim(); const codexApiKey = environmentVariable(childEnv, "CODEX_API_KEY", process.platform)?.trim(); const codexPath = resolveCodexPath( - childEnv, + resolved?.codexPathOverride === undefined + ? childEnv + : { ...childEnv, CODEX_CLI_PATH: resolved.codexPathOverride }, process.platform, process.arch, originalCwd @@ -69,34 +92,35 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { codexPath, cwd: request.workingDirectory, profileId: DEEP_SCAN_WORKER_PERMISSION_PROFILE_ID, - configOverrides, + configOverrides: [ + ...Object.entries(workerModelSelection(modelConfig)) + .map(([key, value]) => `${key}=${tomlInlineValue(value)}`), + ...configOverrides, + ...(resolved?.baseUrl ? [`openai_base_url=${tomlString(resolved.baseUrl)}`] : []) + ], expectedProfile: workerProfile, env: childEnv, allowOpenAiApiKeyFallback: Boolean(openAiApiKey && !codexApiKey), signal: request.signal }); const prompt = await fs.readFile(request.promptPath, "utf8"); - const codex = new Codex({ + const codex = createCodexClient({ + ...resolved, codexPathOverride: executablePathForSpawn(codexPath), env: childEnv, // Codex exec reads CODEX_API_KEY; the SDK maps apiKey to that variable. // Keep native credentials unless the worker has no configured account. ...(useOpenAiApiKey ? { apiKey: openAiApiKey } : {}), config: { - ...(reasoningSummary === undefined - ? {} - : { model_reasoning_summary: reasoningSummary }), - // The CLI can add effort levels before the pinned SDK widens ThreadOptions. - ...(this.modelSettings.reasoningEffort - ? { model_reasoning_effort: this.modelSettings.reasoningEffort } - : {}), + ...modelConfig, mcp_servers: { + ...(isRecord(modelConfig.mcp_servers) ? modelConfig.mcp_servers : {}), // Discovery workers use the bundled skills and artifacts, not the parent workbench MCP. // A disabled server still needs a valid transport while Codex resolves plugin configuration. "codex-security": { command: "node", enabled: false }, ...this.compactArtifactServer(request) }, - ...workerSubagentConfig(request.subagents) + ...workerSubagentConfig(request.subagents, modelConfig) }, // Structured SDK config cannot preserve literal filesystem keys such as // ":root" or "/repo/.env"; raw overrides keep this inline TOML intact. @@ -110,7 +134,7 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { workingDirectory: request.workingDirectory } as const; const thread = request.resumeThreadId - ? codex.resumeThread(request.resumeThreadId, threadOptions) + ? codex.resumeThread!(request.resumeThreadId, threadOptions) : codex.startThread(threadOptions); const input = request.resumeThreadId ? request.continuationPrompt ?? prompt @@ -125,51 +149,44 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { try { const { events } = await thread.runStreamed(input, { signal: controller.signal }); - let finalResponse = ""; - let threadId: string | undefined; - let turnCompleted = false; - let lastStreamError: string | undefined; const diagnostics: CodexWorkerDiagnostic[] = []; - for await (const event of events) { - if (event.type === "thread.started") { - threadId = event.thread_id; - await request.onThreadStarted?.(threadId); - } else if (event.type === "item.completed") { - const fallbackError = event.item.type === "error" - ? deepScanPermissionProfileFallbackError(event.item.message) - : undefined; - if (fallbackError) { - controller.abort(fallbackError); - throw fallbackError; - } - if (event.item.type === "agent_message") { - finalResponse = event.item.text; - } else { + const turn = await readCodexSessionTurn({ + thread, + events, + stopOnCompletion: true, + onEvent: async (event) => { + if (event.type === "thread.started" && typeof event.thread_id === "string") { + await request.onThreadStarted?.(event.thread_id); + } else if (event.type === "item.completed" && isRecord(event.item)) { + const fallbackError = event.item.type === "error" && typeof event.item.message === "string" + ? deepScanPermissionProfileFallbackError(event.item.message) + : undefined; + if (fallbackError) { + controller.abort(fallbackError); + throw fallbackError; + } appendSafeItemDiagnostic(diagnostics, event.item); + } else if (event.type === "turn.completed") { + request.signal.removeEventListener("abort", forwardAbort); + } else if (event.type === "turn.failed") { + throw new Error((event.error as { message: string }).message); + } else if (event.type === "error" && typeof event.message === "string") { + const fallbackError = deepScanPermissionProfileFallbackError(event.message); + if (fallbackError) { + controller.abort(fallbackError); + throw fallbackError; + } + // Codex exec emits retry-in-progress notifications as error events. } - } else if (event.type === "turn.completed") { - turnCompleted = true; - request.signal.removeEventListener("abort", forwardAbort); - break; - } else if (event.type === "turn.failed") { - throw new Error(event.error.message); - } else if (event.type === "error") { - const fallbackError = deepScanPermissionProfileFallbackError(event.message); - if (fallbackError) { - controller.abort(fallbackError); - throw fallbackError; - } - // Codex exec currently emits retry-in-progress notifications as error events. - lastStreamError = event.message; } - } - if (!turnCompleted) { - const detail = lastStreamError ? `: ${lastStreamError}` : ""; + }); + if (turn.status !== "completed") { + const detail = turn.lastStreamError ? `: ${turn.lastStreamError}` : ""; throw new Error(`Codex worker stream ended before turn.completed${detail}`); } return { - finalResponse, - threadId: threadId ?? thread.id ?? undefined, + finalResponse: turn.finalResponse, + threadId: turn.threadId ?? thread.id ?? undefined, ...(diagnostics.length > 0 ? { diagnostics } : {}) }; } finally { @@ -246,13 +263,16 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { } } -function workerSubagentConfig(subagents: number) { +function workerSubagentConfig(subagents: number, config: NonNullable) { return { // V1 counts children; V2 counts the root plus its children. Keeping its // feature disabled lets the model choose either runtime without rejecting // inherited agents.max_threads configuration. - ...(subagents > 0 ? { agents: { max_threads: subagents } } : {}), + ...(subagents > 0 + ? { agents: { ...(isRecord(config.agents) ? config.agents : {}), max_threads: subagents } } + : {}), features: { + ...(isRecord(config.features) ? config.features : {}), multi_agent_v2: { enabled: false, max_concurrent_threads_per_session: subagents + 1 @@ -268,7 +288,7 @@ function workerSubagentConfig(subagents: number) { }; } -type TomlValue = string | number | boolean | TomlObject; +type TomlValue = string | number | boolean | TomlValue[] | TomlObject; type TomlObject = { [key: string]: TomlValue }; function workerPermissionProfile( @@ -307,6 +327,7 @@ function tomlInlineValue(value: TomlValue): string { if (typeof value === "string") return tomlString(value); if (typeof value === "number") return String(value); if (typeof value === "boolean") return value ? "true" : "false"; + if (Array.isArray(value)) return `[${value.map(tomlInlineValue).join(",")}]`; return `{${Object.entries(value) .map(([key, entry]) => `${tomlKey(key)}=${tomlInlineValue(entry)}`) .join(",")}}`; @@ -383,30 +404,38 @@ function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null; } -async function workerReasoningSummary(environment: Record): Promise { +// These are the existing non-secret selections written by the SDK preflight +// adapter. Reading only summary left provider selection in a shared home. +function workerModelSelection(config: NonNullable): TomlObject { + const result: TomlObject = {}; + for (const key of ["model", "model_provider", "model_reasoning_effort", "model_reasoning_summary", "service_tier", "model_providers"]) { + const value = config[key]; + if (value !== undefined) result[key] = value; + } + return result; +} + +async function workerModelConfig(environment: Record): Promise> { const configPath = environmentVariable(environment, "CODEX_SECURITY_CONFIG_PATH", process.platform); - if (!configPath) return undefined; + if (!configPath) return {}; const config = parseToml(await fs.readFile(configPath, "utf8")); const profiles = config.profiles; const profile = typeof config.profile === "string" && isRecord(profiles) ? profiles[config.profile] : undefined; - const summary = isRecord(profile) && profile.model_reasoning_summary !== undefined - ? profile.model_reasoning_summary - : config.model_reasoning_summary; - return typeof summary === "string" ? summary : undefined; + return workerModelSelection({ ...config, ...(isRecord(profile) ? profile : {}) } as NonNullable); } -async function snapshotWorkerEnvironment(): Promise> { +async function snapshotWorkerEnvironment(source: NodeJS.ProcessEnv = process.env): Promise> { const environment = Object.fromEntries( - Object.entries(process.env) + Object.entries(source) .filter((entry): entry is [string, string] => entry[1] !== undefined) ) as Record; if (process.platform === "win32") { // process.env is case-insensitive on Windows; a plain object is not. // Keep its selected values while giving the child one spelling per key. for (const name of ["CODEX_CLI_PATH", "CODEX_HOME", "CODEX_MANAGED_PACKAGE_ROOT", "LOCALAPPDATA"]) { - const value = process.env[name]; + const value = environmentVariable(source, name, process.platform); for (const key of Object.keys(environment)) { if (key.toUpperCase() === name) delete environment[key]; } diff --git a/plugins/codex-security/mcp-app/src/deep-scan/finalization.ts b/plugins/codex-security/mcp-app/src/deep-scan/finalization.ts new file mode 100644 index 000000000..06ad7896f --- /dev/null +++ b/plugins/codex-security/mcp-app/src/deep-scan/finalization.ts @@ -0,0 +1,193 @@ +import { + deepReductionToScanDraft, + parseDeepReduction, +} from "./artifact-validation.js"; +import { + createScanArtifactContext, + type RunArtifactWorkbench, +} from "../artifact-context.js"; +import { + recordCodexSecurityScanDraftViaWorkbench, + type DeepScanPublication, +} from "../artifact-scan-draft.js"; +import { WorkbenchDeepScanStore } from "./store.js"; +import { createDeepScanArtifacts } from "./artifacts.js"; +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { type ScanDraftInput } from "../artifact-scan-draft.js"; +import { requireRegularFile, type DeepScanArtifacts } from "./artifacts.js"; +import type { + DeepScanFinalizationInput, + DeepScanRunState, + DeepScanStore, +} from "./types.js"; + +export type { DeepScanFinalizationInput } from "./types.js"; + +/** Publish exactly the saved aggregate; the enclosing scan owns public completion. */ +export async function publishSelectedDeepScan(input: { + run: DeepScanRunState; + artifacts: DeepScanArtifacts; + signal: AbortSignal; + publish: ( + draft: ScanDraftInput, + signal: AbortSignal, + publication: DeepScanPublication, + ) => Promise; + finish: DeepScanStore["finish"]; +}): Promise { + const { run, artifacts, signal } = input; + const selection = selectedInput(run); + if (run.status === "succeeded") return run; + if (run.status !== "running") + throw new Error("Stopped Deep Scan finalization cannot become successful."); + signal.throwIfAborted(); + const draft = await readSelectedDeepScanDraft( + artifacts, + run.scanId, + selection, + ); + await input.publish(draft, signal, { + coordinatorGeneration: run.coordinatorGeneration, + resultPath: + selection.resultPath === null + ? null + : join(run.scanDir, selection.resultPath), + }); + signal.throwIfAborted(); + return input.finish({ + scanId: run.scanId, + reason: selection.terminalReason, + manifestPath: join(run.scanDir, "scan-manifest.json"), + omittedWorkerIds: selection.omittedWorkerIds, + }); +} + +/** Recreate the chosen draft without scheduling discovery or reducer work. */ +export async function readSelectedDeepScanDraft( + artifacts: DeepScanArtifacts, + scanId: string, + selection: DeepScanFinalizationInput, +): Promise { + if (selection.version !== 1) + throw new Error("Unsupported Deep Scan finalization input version."); + if (selection.resultPath === null) { + if ( + selection.terminalReason !== "capped" || + selection.resultSha256 !== null + ) { + throw new Error( + "An empty Deep Scan finalization requires the recorded discovery deadline.", + ); + } + return { + scanId, + findings: [], + coverage: { + completeness: "partial", + surfaces: [], + explicitExclusions: [], + deferred: [ + { + reason: + "The configured discovery time limit elapsed before any source review completed.", + }, + ], + }, + }; + } + const resultPath = join(artifacts.scanDir, selection.resultPath); + await requireRegularFile(resultPath, artifacts.scanDir); + const contents = await readFile(resultPath); + if ( + createHash("sha256").update(contents).digest("hex") !== + selection.resultSha256 + ) { + throw new Error( + "The selected Deep Scan finalization input changed after acceptance.", + ); + } + const stored = JSON.parse(contents.toString("utf8")); + const draft = deepReductionToScanDraft(parseDeepReduction(stored, true)); + if (draft.scanId !== scanId || draft.complete === false) { + throw new Error( + "Deep Scan finalization requires the selected complete result for this scan.", + ); + } + return draft; +} + +/** SDK recovery uses the installed plugin's publisher and the original parent scan. */ +export async function resumeSelectedDeepScan(input: { + scanId: string; + threadId: string; + pluginRoot: string; + runWorkbench: RunArtifactWorkbench; + signal: AbortSignal; + handoffClaimToken?: string; +}): Promise { + const store = new WorkbenchDeepScanStore(input.runWorkbench); + const run = await store.get(input.scanId, input.threadId); + selectedInput(run); + const prepare = [ + "prepare-scan-completion", "--scan-id", input.scanId, + ...(input.handoffClaimToken ? ["--claim-token", input.handoffClaimToken] : []), + ]; + if (run.status === "succeeded") { + await input.runWorkbench(prepare); + return; + } + try { + await publishSelectedDeepScan({ + run, + artifacts: createDeepScanArtifacts(run.scanDir), + signal: input.signal, + publish: async (draft, signal, publication) => { + const context = await createScanArtifactContext( + input.scanId, + input.runWorkbench, + { + requireRunning: true, + requireClaim: true, + handoffClaimToken: input.handoffClaimToken, + pluginRoot: input.pluginRoot, + }, + ); + await recordCodexSecurityScanDraftViaWorkbench( + context, + draft, + input.runWorkbench, + signal, + publication, + ); + }, + finish: (selection) => + store.finish({ + ...selection, + coordinatorGeneration: run.coordinatorGeneration, + }), + }); + } catch (error) { + // A succeeded child does not prove that its parent publication is valid. + // Validate the publication and any existing seal; the caller owns completion. + const committed = await store + .get(input.scanId, input.threadId) + .catch(() => null); + if (committed?.status !== "succeeded") throw error; + await input.runWorkbench(prepare); + } +} + +function selectedInput(run: DeepScanRunState): DeepScanFinalizationInput { + const selection = run.finalizationInput; + if (!selection) + throw new Error("Deep Scan has no selected finalization input."); + if ( + run.workflowVersion !== "deep-security-scan/v2" || + selection.version !== 1 + ) { + throw new Error("Unsupported Deep Scan finalization input version."); + } + return selection; +} diff --git a/plugins/codex-security/mcp-app/src/deep-scan/recovery-settings.ts b/plugins/codex-security/mcp-app/src/deep-scan/recovery-settings.ts new file mode 100644 index 000000000..222bee459 --- /dev/null +++ b/plugins/codex-security/mcp-app/src/deep-scan/recovery-settings.ts @@ -0,0 +1,284 @@ +import { promises as fs } from "node:fs"; +import { homedir } from "node:os"; +import { isAbsolute, join, win32 } from "node:path"; +import type { CodexOptions } from "@openai/codex-sdk"; +import { parse as parseToml } from "smol-toml"; +import { scanPreflightCodexConfig } from "../../../../../sdk/typescript/src/preflight-config.js"; +import { resolveCodexProfile, type JsonObject } from "../../../../../sdk/typescript/src/config.js"; +import { readScanLogs } from "../../../../../sdk/typescript/src/scan-logs.js"; +import { resolveCodexPath } from "./executor.js"; +import type { DeepWorkerParentSandbox } from "./parent-sandbox.js"; +import type { DeepScanRunState } from "./types.js"; + +/** Credentials and arbitrary environment/configuration stay with Codex. */ +export interface DeepScanExecutionSettings { + codexPath: string; + codexHome: string; + model?: string; + modelProvider?: string; + reasoningEffort?: string; + reasoningSummary?: string; + serviceTier?: string; + /** The native snapshot recorded no request tier; serviceTier preserves its wire behavior. */ + nativeServiceTierAbsent?: true; + providerConfig?: JsonObject; + parentSandbox?: DeepWorkerParentSandbox; +} + +export interface DeepScanLegacySettingsContext { + config?: JsonObject; + usageOwner?: DeepScanRunState["usageOwner"]; +} + +export interface DeepScanExecutionSettingsSnapshot { + version: number; + settings: DeepScanExecutionSettings; +} + +export async function captureDeepScanExecutionSettings( + original: Pick, + parentSandbox: DeepWorkerParentSandbox, + environment: NodeJS.ProcessEnv = process.env, + parent?: { threadId: string; startedAt?: string } +): Promise { + const codexHome = environment.CODEX_HOME || join(homedir(), ".codex"); + const configPath = environment.CODEX_SECURITY_CONFIG_PATH ?? join(codexHome, "config.toml"); + let config: JsonObject; + try { + config = parseToml(await fs.readFile(configPath, "utf8")) as JsonObject; + } catch (error) { + if (environment.CODEX_SECURITY_CONFIG_PATH || (error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + config = {}; + } + // Reuse the SDK projection: custom provider credentials belong in the native home. + const selected = scanPreflightCodexConfig(resolveCodexProfile(config)); + // A recovered scan can have a different continuation. Only its recorded owner + // establishes original history; null means that historical binding is missing. + const owner = original.usageOwner === undefined ? parent : original.usageOwner; + const native = !owner?.threadId ? {} : await originalParentSettings(codexHome, { + ...owner, threadId: owner.threadId, startedAt: parent?.startedAt ?? owner.startedAt + }); + return executionSettings({ + codexPath: resolveCodexPath(environment, process.platform, process.arch, process.cwd()), + codexHome: !isAbsolute(codexHome) + || (process.platform === "win32" && ["\\", "/"].includes(win32.parse(codexHome).root)) + ? await fs.realpath(codexHome) : codexHome, + model: original.model ?? (selected.model as string | undefined) ?? native.model, + reasoningEffort: original.reasoningEffort ?? (selected.model_reasoning_effort as string | undefined) ?? native.reasoningEffort, + modelProvider: (selected.model_provider as string | undefined) ?? native.modelProvider, + reasoningSummary: (selected.model_reasoning_summary as string | undefined) ?? native.reasoningSummary, + serviceTier: (selected.service_tier as string | undefined) ?? native.serviceTier, + ...(selected.service_tier === undefined && native.nativeServiceTierAbsent + ? { nativeServiceTierAbsent: true as const } : {}), + providerConfig: selected.model_providers as JsonObject | undefined, + parentSandbox + }); +} + +async function originalParentSettings( + codexHome: string, + parent: { threadId: string; turnId?: string | null; startedAt?: string } +): Promise> { + // Native config/read represents omitted selections as null. Recover recorded + // selections from the original parent; some native records omit the summary. + // History can be disabled or unavailable; configured selections still work. + try { + const log = await readScanLogs({ + scanId: parent.threadId, threadId: parent.threadId, executionThreadIds: [], + codexHome, allowMissingRoot: true + }); + const settings: Partial = {}; + let applied: Partial | undefined; + let summaryIsCompatibilityOnly = false; + const cutoff = parent.startedAt === undefined ? Infinity : Date.parse(parent.startedAt); + for (const entry of log.events) { + const event = entry.event as Record; + const timestamp = typeof event.timestamp === "string" ? Date.parse(event.timestamp) : undefined; + if (timestamp !== undefined && timestamp > cutoff) continue; + const payload = event.payload; + if (!payload || typeof payload !== "object" || Array.isArray(payload)) continue; + const context = payload as Record; + if (event.type === "event_msg" && context.type === "thread_settings_applied") { + if (typeof context.thread_id === "string" && context.thread_id !== parent.threadId) continue; + const snapshot = context.thread_settings; + if (!snapshot || typeof snapshot !== "object" || Array.isArray(snapshot)) continue; + const value = snapshot as Record; + applied = { + model: typeof value.model === "string" ? value.model : undefined, + modelProvider: typeof value.model_provider_id === "string" ? value.model_provider_id : undefined, + reasoningEffort: typeof value.reasoning_effort === "string" ? value.reasoning_effort : undefined, + reasoningSummary: typeof value.reasoning_summary === "string" ? value.reasoning_summary : undefined, + // A persisted native absent tier and explicit standard both omit the + // request tier. This does not infer a tier from missing history. + serviceTier: typeof value.service_tier === "string" ? value.service_tier + : value.service_tier === undefined ? "default" : undefined, + ...(value.service_tier === undefined ? { nativeServiceTierAbsent: true as const } : {}) + }; + } + if (event.type === "session_meta") { + if (typeof context.model_provider === "string") settings.modelProvider = context.model_provider; + // Codex 0.133 replaced turn_context.summary with a compatibility default. + // Fresh threads need not have a thread_settings_applied record to replace it. + const version = typeof context.cli_version === "string" + ? /^(\d+)\.(\d+)\./u.exec(context.cli_version) : null; + summaryIsCompatibilityOnly = version !== null + && (Number(version[1]) > 0 || Number(version[2]) >= 133); + } + if (event.type === "turn_context") { + if (parent.turnId && context.turn_id !== parent.turnId) continue; + if (typeof context.model === "string") settings.model = context.model; + if (typeof context.effort === "string") settings.reasoningEffort = context.effort; + if (!summaryIsCompatibilityOnly && typeof context.summary === "string") settings.reasoningSummary = context.summary; + } + } + // Applied snapshots contain native selected values. Newer turn-context + // summaries are only a compatibility field, not the active selection. + return { ...settings, ...applied }; + } catch { + return {}; + } +} + +/** Read recorded execution settings or original legacy facts. */ +export async function loadDeepScanExecutionSettings( + _scanDir: string, + original?: Pick, + readLegacyContext?: () => Promise, + environment: NodeJS.ProcessEnv = process.env +): Promise> { + // Only the workbench creation transaction records the launch selection. Scan + // artifacts are model-writable and cannot select a preflight executable/home. + const saved = original?.executionSettings; + let settings: DeepScanExecutionSettings; + if (saved) { + if (saved.version !== 1) { + throw new Error("This Deep Scan uses an unsupported execution settings version."); + } + settings = executionSettings(saved.settings); + } else { + if (original?.workflowVersion === "deep-security-scan/v1" || original?.workflowVersion === "deep-scan-mcp/v1") { + // Legacy runs predate the binding. Their saved recipe and recorded owner + // can recover selections, but cannot establish an original executable or + // home. Leave those unknown and retain the existing native launch behavior. + const context = await readLegacyContext?.(); + const selected = scanPreflightCodexConfig(resolveCodexProfile(context?.config ?? {})); + const owner = original.usageOwner ?? context?.usageOwner; + const home = environment.CODEX_HOME || join(homedir(), ".codex"); + const native = !owner?.threadId ? {} : await originalParentSettings(home, { + ...owner, threadId: owner.threadId, startedAt: original.createdAt ?? owner.startedAt + }); + return { + model: original.model ?? (selected.model as string | undefined) ?? native.model, + reasoningEffort: original.reasoningEffort ?? (selected.model_reasoning_effort as string | undefined) ?? native.reasoningEffort, + modelProvider: (selected.model_provider as string | undefined) ?? native.modelProvider, + reasoningSummary: (selected.model_reasoning_summary as string | undefined) ?? native.reasoningSummary, + serviceTier: (selected.service_tier as string | undefined) ?? native.serviceTier, + ...(selected.service_tier === undefined && native.nativeServiceTierAbsent + ? { nativeServiceTierAbsent: true as const } : {}), + providerConfig: selected.model_provider === "amazon-bedrock" + ? selected.model_providers as JsonObject | undefined : undefined + }; + } + throw new Error("This Deep Scan has no recorded original execution settings; its executable and Codex home cannot be recovered."); + } + if (!original || (settings.model !== undefined && settings.reasoningEffort !== undefined + && settings.modelProvider !== undefined && settings.reasoningSummary !== undefined + && settings.serviceTier !== undefined)) return settings; + // Earlier snapshots can omit native selections. Recover only from the saved + // home and recorded owner; the continuation's current config is not history. + const owner = original.usageOwner; + const native = !owner?.threadId ? {} : await originalParentSettings(settings.codexHome, { + ...owner, threadId: owner.threadId, startedAt: original.createdAt + }); + // History reads can outlive this coordinator. Project missing selections for + // its workers without overwriting a snapshot owned by a newer coordinator. + return executionSettings({ + ...settings, + model: settings.model ?? original.model ?? native.model, + reasoningEffort: settings.reasoningEffort ?? original.reasoningEffort ?? native.reasoningEffort, + modelProvider: settings.modelProvider ?? native.modelProvider, + reasoningSummary: settings.reasoningSummary ?? native.reasoningSummary, + serviceTier: settings.serviceTier ?? native.serviceTier, + ...(settings.serviceTier === undefined && native.nativeServiceTierAbsent + ? { nativeServiceTierAbsent: true as const } : {}) + }); +} + +export function restoredDeepScanWorkerSettings( + settings: Partial, + currentParentSandbox: DeepWorkerParentSandbox, + environment: () => NodeJS.ProcessEnv = () => process.env +): { + codexOptions: CodexOptions; + model?: string; + reasoningEffort?: string; + parentSandbox: DeepWorkerParentSandbox; +} { + const originalSandbox = settings.parentSandbox; + const depths = [originalSandbox?.globScanMaxDepth, currentParentSandbox.globScanMaxDepth] + .filter((depth): depth is number => depth !== undefined); + // Native depth caps limit deny-glob expansion, not allowed traversal. Keep + // the larger finite cap, or no cap when either known policy has uncapped globs. + const uncapped = [originalSandbox, currentParentSandbox].some((sandbox) => + sandbox?.globScanMaxDepth === undefined && sandbox?.filesystemDenies.some((path) => + ["*", "?", "[", "]"].some((character) => path.includes(character)))); + return { + model: settings.model, + reasoningEffort: settings.reasoningEffort, + parentSandbox: { + filesystemDenies: [...new Set([ + ...(originalSandbox?.filesystemDenies ?? []), ...currentParentSandbox.filesystemDenies + ])], + ...(uncapped || depths.length === 0 ? {} : { globScanMaxDepth: Math.max(...depths) }) + }, + codexOptions: { + codexPathOverride: settings.codexPath, + // The executor reads this property for each launch. API keys can refresh; + // only the original account home and non-secret selections are bound. + get env() { + return Object.fromEntries(Object.entries({ ...environment(), + ...(settings.codexPath === undefined ? {} : { CODEX_CLI_PATH: settings.codexPath }), + ...(settings.codexHome === undefined ? {} : { CODEX_HOME: settings.codexHome }) }) + .filter((entry): entry is [string, string] => entry[1] !== undefined)); + }, + config: scanPreflightCodexConfig({ + ...(settings.model === undefined ? {} : { model: settings.model }), + ...(settings.reasoningEffort === undefined ? {} : { model_reasoning_effort: settings.reasoningEffort }), + ...(settings.modelProvider === undefined ? {} : { model_provider: settings.modelProvider }), + ...(settings.reasoningSummary === undefined ? {} : { model_reasoning_summary: settings.reasoningSummary }), + ...(settings.serviceTier === undefined ? {} : { service_tier: settings.serviceTier }), + ...(settings.providerConfig === undefined ? {} : { model_providers: settings.providerConfig }) + }) as NonNullable + } + }; +} + +function executionSettings(value: DeepScanExecutionSettings): DeepScanExecutionSettings { + // Catalog provider definitions are reconstructed by the existing launch + // projection. Only Bedrock's per-scan AWS selectors need persistence. + const provider = value.modelProvider === "amazon-bedrock" ? scanPreflightCodexConfig({ + ...(value.modelProvider === undefined ? {} : { model_provider: value.modelProvider }), + ...(value.providerConfig === undefined ? {} : { model_providers: value.providerConfig }) + }).model_providers as JsonObject | undefined : undefined; + const settings: DeepScanExecutionSettings = { + codexPath: value.codexPath, + codexHome: value.codexHome, + model: value.model, + modelProvider: value.modelProvider, + reasoningEffort: value.reasoningEffort, + reasoningSummary: value.reasoningSummary, + serviceTier: value.serviceTier, + ...(value.nativeServiceTierAbsent === true ? { nativeServiceTierAbsent: true } : {}), + ...(provider === undefined ? {} : { providerConfig: provider }), + ...(value.parentSandbox === undefined ? {} : { parentSandbox: { + filesystemDenies: [...value.parentSandbox.filesystemDenies], + ...(value.parentSandbox.globScanMaxDepth === undefined ? {} : { + globScanMaxDepth: value.parentSandbox.globScanMaxDepth + }) + } }) + }; + if (typeof settings.codexPath !== "string" || typeof settings.codexHome !== "string") { + throw new Error("Deep Scan execution settings are missing the recorded executable or Codex home."); + } + return settings; +} diff --git a/plugins/codex-security/mcp-app/src/deep-scan/registry.ts b/plugins/codex-security/mcp-app/src/deep-scan/registry.ts index 574ff2c53..3bbac0a2a 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/registry.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/registry.ts @@ -2,13 +2,18 @@ import { setTimeout as delay } from "node:timers/promises"; import { DeepScanCoordinator } from "./coordinator.js"; import type { CoordinatorOptions } from "./coordinator.js"; import { isTransientPersistenceError } from "./store.js"; -import type { BeginDeepScanResult, DeepScanCoordinatorClaim, DeepScanRunState } from "./types.js"; +import type { BeginDeepScanResult, CodexWorkerExecutor, DeepScanCoordinatorClaim, DeepScanRunState } from "./types.js"; const COORDINATOR_LEASE_MS = 30_000; const COORDINATOR_POLL_MS = 1_000; export { DeepScanCoordinator, DeepScanNonRetryableError } from "./coordinator.js"; +export interface DeepScanCoordinatorStartOptions extends CoordinatorOptions { + /** Resolve persisted execution settings only after acquiring this run. */ + prepareExecutor?: (run: DeepScanRunState) => Promise; +} + /** Owns the live coordinators in this MCP server process. */ export class DeepScanCoordinatorRegistry { private readonly coordinators = new Map(); @@ -17,7 +22,8 @@ export class DeepScanCoordinatorRegistry { return this.coordinators.get(scanId); } - start(options: CoordinatorOptions): DeepScanCoordinator { + start(options: DeepScanCoordinatorStartOptions): DeepScanCoordinator { + requireSupportedDeepScan(options.run); const existing = this.coordinators.get(options.run.scanId); if (existing) return existing; const { observeReplacement: _unused, ...remoteOptions } = options; @@ -104,7 +110,7 @@ export class DeepScanRemoteCoordinator { private readonly input: { run: DeepScanRunState; registry: Pick; - options: Omit; + options: Omit; } ) {} @@ -136,6 +142,7 @@ export class DeepScanRemoteCoordinator { continue; } if (run.status !== "running") return run; + requireSupportedDeepScan(run); const heartbeat = run.updatedAt ? Date.parse(run.updatedAt) : Number.NaN; if ( @@ -167,7 +174,7 @@ export class DeepScanRemoteCoordinator { } } if (claim?.acquired) { - const coordinator = registry.start({ ...options, run: claim.run }); + const coordinator = await startClaimedCoordinator(registry, options, claim.run); return deadline === undefined ? await coordinator.wait(signal) : await coordinator.wait(signal, Math.max(0, deadline - Date.now())); @@ -185,11 +192,12 @@ export class DeepScanRemoteCoordinator { export async function startOrJoinDeepScanCoordinator(input: { begin: BeginDeepScanResult; registry: Pick; - options: Omit; + options: Omit; }): Promise<{ coordinator: DeepScanCoordinator | DeepScanRemoteCoordinator; joined: boolean; }> { + requireSupportedDeepScan(input.begin.run); const existing = input.registry.get(input.begin.run.scanId); if (existing) return { coordinator: existing, joined: true }; const threadId = input.options.threadId; @@ -212,11 +220,44 @@ export async function startOrJoinDeepScanCoordinator(input: { }; } return { - coordinator: input.registry.start({ ...input.options, run: claim.run }), + coordinator: await startClaimedCoordinator(input.registry, input.options, claim.run), joined: false }; } +async function startClaimedCoordinator( + registry: Pick, + options: Omit, + run: DeepScanRunState +): Promise { + requireSupportedDeepScan(run); + const executor = options.prepareExecutor && !run.finalizationInput + ? await options.prepareExecutor(run) + : options.executor; + return registry.start({ ...options, executor, run }); +} + +function requireSupportedDeepScan(run: DeepScanRunState): void { + if (run.finalizationInput !== undefined && ( + run.workflowVersion !== "deep-security-scan/v2" || run.finalizationInput.version !== 1 + )) { + throw new Error("This executor does not support this Deep Scan finalization input version."); + } + // Missing versions are supported for older adapters that did not project them. + if ( + (run.schemaVersion !== undefined && run.schemaVersion !== 1) + || (run.workflowVersion !== undefined + && run.workflowVersion !== "deep-security-scan/v1" + && run.workflowVersion !== "deep-scan-mcp/v1" + && run.workflowVersion !== "deep-security-scan/v2") + ) { + throw new Error( + "This Deep Scan uses an unsupported workflow or schema version. " + + "Resume it with a compatible Codex Security release." + ); + } +} + function remoteAbortError(reason: unknown): Error { const error = new Error("Deep Scan observation was aborted.", { cause: reason }); error.name = "AbortError"; diff --git a/plugins/codex-security/mcp-app/src/deep-scan/store.ts b/plugins/codex-security/mcp-app/src/deep-scan/store.ts index 8bf6b0bf7..9acc99a0c 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/store.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/store.ts @@ -28,10 +28,12 @@ import type { type JsonObject = Record; export type WorkbenchRunner = ( args: string[], - input?: string + input?: string, + selectFinalization?: boolean, + withExecutionSettings?: boolean, ) => Promise; -const WORKFLOW_VERSION = "deep-scan-mcp/v1"; +const WORKFLOW_VERSION = "deep-security-scan/v1"; const MAX_IDEMPOTENT_PERSISTENCE_ATTEMPTS = 3; const PERSISTENCE_RETRY_BASE_DELAY_MS = 100; @@ -176,7 +178,7 @@ export class WorkbenchDeepScanStore implements DeepScanStore { input.threadId, ...this.coordinatorLeaseArgs(input.scanId), ...(input.handoffClaimToken ? ["--claim-token", input.handoffClaimToken] : []) - ]); + ], false, undefined, false, true); const run = parseDeepScan(result); const disposition = result.coordinatorDisposition; if (disposition !== "claimed" && disposition !== "adopted" && disposition !== "observing") { @@ -250,8 +252,10 @@ export class WorkbenchDeepScanStore implements DeepScanStore { ? ["--replaceable-failure-kind", update.replaceableFailureKind] : []) ], true); - const worker = parseWorker(result, update.id); const state = objectValue(result.deepScan, "deepScan"); + const worker = parseWorker(state.workerReceipt + ? { deepScan: { ...state, workers: [state.workerReceipt] } } + : result, update.id); return state.consecutiveErrors === undefined ? worker : { @@ -269,8 +273,8 @@ export class WorkbenchDeepScanStore implements DeepScanStore { workerIds: string[]; promptPath: string; artifactDir: string; - }): Promise { - await this.enqueueWrite([ + }): Promise { + return parseDeepScan(await this.enqueueWrite([ "claim-deep-scan-dedup", "--scan-id", input.scanId, @@ -282,7 +286,7 @@ export class WorkbenchDeepScanStore implements DeepScanStore { input.artifactDir, ...this.coordinatorLeaseArgs(input.scanId), ...input.workerIds.flatMap((workerId) => ["--input-worker-id", workerId]) - ], true); + ], true)); } async commitDedup(commit: DedupCommit): Promise { @@ -303,8 +307,32 @@ export class WorkbenchDeepScanStore implements DeepScanStore { ], true)); } + async selectFinalization(input: { + scanId: string; + coordinatorGeneration?: number; + reason: DeepScanTerminalReason; + manifestPath: string; + resultPath?: string; + omittedWorkerIds: string[]; + }): Promise { + return parseDeepScan(await this.enqueueWrite([ + "finish-deep-scan", + "--scan-id", + input.scanId, + ...(input.coordinatorGeneration === undefined + ? this.coordinatorLeaseArgs(input.scanId) + : ["--coordinator-generation", String(input.coordinatorGeneration)]), + "--terminal-reason", + input.reason, + "--manifest-path", + input.manifestPath, + ...input.omittedWorkerIds.flatMap((workerId) => ["--omitted-worker-id", workerId]) + ], true, JSON.stringify({ resultPath: input.resultPath ?? null }), true)); + } + async finish(input: { scanId: string; + coordinatorGeneration?: number; reason: DeepScanTerminalReason; manifestPath: string; stagedManifestPath?: string; @@ -314,7 +342,9 @@ export class WorkbenchDeepScanStore implements DeepScanStore { "finish-deep-scan", "--scan-id", input.scanId, - ...this.coordinatorLeaseArgs(input.scanId), + ...(input.coordinatorGeneration === undefined + ? this.coordinatorLeaseArgs(input.scanId) + : ["--coordinator-generation", String(input.coordinatorGeneration)]), "--terminal-reason", input.reason, "--manifest-path", @@ -407,13 +437,15 @@ export class WorkbenchDeepScanStore implements DeepScanStore { private enqueueWrite( args: string[], retryTransientFailure = false, - input?: string + input?: string, + selectFinalization = false, + withExecutionSettings = false, ): Promise { const operation = this.writeTail.then(async () => { try { return retryTransientFailure - ? await this.runIdempotentPersistence(args) - : await this.runWorkbench(args, input); + ? await this.runIdempotentPersistence(args, input, selectFinalization) + : await this.runWorkbench(args, input, selectFinalization, withExecutionSettings); } catch (error) { const scanId = argumentValue(args, "--scan-id"); if (scanId && isStaleCoordinatorGenerationError(error)) { @@ -430,11 +462,11 @@ export class WorkbenchDeepScanStore implements DeepScanStore { } /** Replay only existing, same-identity workbench mutations after transient failures. */ - private async runIdempotentPersistence(args: string[]): Promise { + private async runIdempotentPersistence(args: string[], input?: string, selectFinalization = false): Promise { const startedAt = Date.now(); for (let attempt = 1; attempt <= MAX_IDEMPOTENT_PERSISTENCE_ATTEMPTS; attempt += 1) { try { - return await this.runWorkbench(args); + return await this.runWorkbench(args, input, selectFinalization); } catch (error) { if (!isTransientPersistenceError(error)) { throw error; @@ -586,6 +618,12 @@ export function parseDeepScan(result: JsonObject): DeepScanRunState { }; return { scanId: requiredString(value.scanId, "deepScan.scanId"), + schemaVersion: optionalPositiveInteger(value.schemaVersion), + workflowVersion: optionalString(value.workflowVersion), + finalizationInput: parseFinalizationInput(value.finalizationInput), + usageOwner: parseUsageOwner(value.usageOwner), + executionSettings: value.executionSettings == null ? undefined + : objectValue(value.executionSettings, "deepScan.executionSettings") as unknown as DeepScanRunState["executionSettings"], status, phase: deepScanPhase(value.phase), coordinatorGeneration: optionalPositiveInteger(value.coordinatorGeneration), @@ -594,6 +632,8 @@ export function parseDeepScan(result: JsonObject): DeepScanRunState { targetPath: requiredString(value.targetPath, "deepScan.targetPath"), scope: requiredString(value.scope, "deepScan.scope"), userContext: optionalString(value.userContext), + model: optionalString(value.model), + reasoningEffort: optionalString(value.reasoningEffort), scanDir: requiredString(value.scanDir, "deepScan.scanDir"), config, dispatchedCount: nonNegativeInteger(value.dispatchedCount, "deepScan.dispatchedCount"), @@ -609,7 +649,48 @@ export function parseDeepScan(result: JsonObject): DeepScanRunState { : undefined, error: optionalString(value.error), persistedWorkers: parsePersistedWorkers(value.workers), - persistedDedupInputs: parsePersistedDedupInputs(value.dedupInputs) + persistedDedupInputs: parsePersistedDedupInputs(value.dedupInputs), + persistedMergeClaims: Array.isArray(value.mergeClaims) ? value.mergeClaims.map((candidate) => { + const claim = objectValue(candidate, "deepScan.mergeClaim"); + return { + workerId: requiredString(claim.workerId, "deepScan.mergeClaim.workerId"), + previousWorkerId: optionalString(claim.previousWorkerId), + previousResultPath: optionalString(claim.previousResultPath), + previousResultSha256: optionalString(claim.previousResultSha256) + }; + }) : [], + ...(value.committedMerge ? { committedMerge: parseCommittedMerge(value.committedMerge) } : {}) + }; +} + +function parseUsageOwner(value: unknown): DeepScanRunState["usageOwner"] { + if (value === undefined || value === null) return null; + const owner = objectValue(value, "deepScan.usageOwner"); + return { + threadId: optionalString(owner.threadId) ?? null, + turnId: optionalString(owner.turnId) ?? null, + startedAt: requiredString(owner.startedAt, "deepScan.usageOwner.startedAt") + }; +} + +function parseFinalizationInput(value: unknown): DeepScanRunState["finalizationInput"] { + if (value === undefined || value === null) return undefined; + const input = objectValue(value, "deepScan.finalizationInput"); + if (input.terminalReason !== "saturated" && input.terminalReason !== "capped") { + throw new Error("Codex Security workbench returned invalid finalization terminal reason."); + } + if (!Array.isArray(input.omittedWorkerIds)) { + throw new Error("Codex Security workbench returned invalid finalization omissions."); + } + return { + version: positiveInteger(input.version, "deepScan.finalizationInput.version"), + resultPath: input.resultPath === null + ? null : requiredString(input.resultPath, "deepScan.finalizationInput.resultPath"), + resultSha256: input.resultSha256 === null + ? null : requiredString(input.resultSha256, "deepScan.finalizationInput.resultSha256"), + terminalReason: input.terminalReason, + omittedWorkerIds: input.omittedWorkerIds.map((id) => requiredString(id, "omittedWorkerId")), + selectedAt: requiredString(input.selectedAt, "deepScan.finalizationInput.selectedAt") }; } @@ -632,11 +713,24 @@ function parsePersistedDedupInputs(value: unknown): PersistedDeepScanDedupInput[ inputOrder: nonNegativeInteger( input.inputOrder, "deepScan.dedupInput.inputOrder" - ) + ), + resultManifestPath: optionalString(input.resultManifestPath), + resultManifestSha256: optionalString(input.resultManifestSha256), + attempt: optionalPositiveInteger(input.attempt) }; }); } +function parseCommittedMerge(value: unknown): NonNullable { + const commit = objectValue(value, "deepScan.committedMerge"); + return { + workerId: requiredString(commit.workerId, "deepScan.committedMerge.workerId"), + resultManifestPath: requiredString(commit.resultManifestPath, "deepScan.committedMerge.resultManifestPath"), + resultManifestSha256: requiredString(commit.resultManifestSha256, "deepScan.committedMerge.resultManifestSha256"), + newFindings: nonNegativeInteger(commit.newFindings, "deepScan.committedMerge.newFindings") + }; +} + function deepScanPhase(value: unknown): DeepScanRunState["phase"] { if (value === undefined || value === null) return undefined; if (value === "setup" || value === "discovery" || value === "reducing" || value === "terminal") { @@ -698,6 +792,7 @@ function parsePersistedWorker(value: JsonObject): PersistedDeepScanWorker { attempt: nonNegativeInteger(value.attempt, "deepScan.worker.attempt"), threadId: optionalString(value.sdkThreadId), resultManifestPath: optionalString(value.resultManifestPath), + acceptedResultPath: optionalString(value.acceptedResultPath), completionSequence: optionalPositiveInteger(value.completionSequence), error: optionalString(value.error) }; diff --git a/plugins/codex-security/mcp-app/src/deep-scan/types.ts b/plugins/codex-security/mcp-app/src/deep-scan/types.ts index dea2d9431..2d1cc8f85 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/types.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/types.ts @@ -1,4 +1,6 @@ import type { DeepReducerContext } from "../artifact-io.js"; +import type { ScanExecutionAttribution } from "../../../../../sdk/typescript/src/scan-sessions.js"; +import type { DeepScanExecutionSettingsSnapshot } from "./recovery-settings.js"; export type DeepScanTerminalReason = "saturated" | "capped"; @@ -37,8 +39,22 @@ export interface DeepScanCanonicalArtifacts { export type DeepScanReducerArtifacts = DeepScanCanonicalArtifacts; +export interface DeepScanFinalizationInput { + version: number; + resultPath: string | null; + resultSha256: string | null; + terminalReason: DeepScanTerminalReason; + omittedWorkerIds: string[]; + selectedAt: string; +} + export interface DeepScanRunState { scanId: string; + schemaVersion?: number; + workflowVersion?: string; + finalizationInput?: DeepScanFinalizationInput; + usageOwner?: ScanExecutionAttribution["owner"] | null; + executionSettings?: DeepScanExecutionSettingsSnapshot | null; status: DeepScanRunStatus; phase?: "setup" | "discovery" | "reducing" | "terminal"; coordinatorGeneration?: number; @@ -47,6 +63,8 @@ export interface DeepScanRunState { targetPath: string; scope: string; userContext?: string; + model?: string; + reasoningEffort?: string; scanDir: string; config: DeepScanConfig; dispatchedCount: number; @@ -58,12 +76,29 @@ export interface DeepScanRunState { error?: string; persistedWorkers?: PersistedDeepScanWorker[]; persistedDedupInputs?: PersistedDeepScanDedupInput[]; + persistedMergeClaims?: PersistedDeepScanMergeClaim[]; + committedMerge?: { + workerId: string; + resultManifestPath: string; + resultManifestSha256: string; + newFindings: number; + }; +} + +export interface PersistedDeepScanMergeClaim { + workerId: string; + previousWorkerId?: string; + previousResultPath?: string; + previousResultSha256?: string; } export interface PersistedDeepScanDedupInput { dedupWorkerId: string; discoveryWorkerId: string; inputOrder: number; + resultManifestPath?: string; + resultManifestSha256?: string; + attempt?: number; } export interface BeginDeepScanResult { @@ -112,6 +147,7 @@ export interface PersistedDeepScanWorker { attempt: number; threadId?: string; resultManifestPath?: string; + acceptedResultPath?: string; completionSequence?: number; consecutiveErrors?: number; mergeState: DeepScanMergeState; @@ -151,10 +187,19 @@ export interface DeepScanStore { workerIds: string[]; promptPath: string; artifactDir: string; - }): Promise; + }): Promise; commitDedup(commit: DedupCommit): Promise; + selectFinalization?(input: { + scanId: string; + coordinatorGeneration?: number; + reason: DeepScanTerminalReason; + manifestPath: string; + resultPath?: string; + omittedWorkerIds: string[]; + }): Promise; finish(input: { scanId: string; + coordinatorGeneration?: number; reason: DeepScanTerminalReason; manifestPath: string; stagedManifestPath?: string; diff --git a/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts b/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts index 82f119979..b118f2efe 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts @@ -1,9 +1,10 @@ -import { createHash } from "node:crypto"; +import type { ScanDraftInput } from "../artifact-scan-draft.js"; +import { auditEvidence, runAcceptedAudit } from "../../../../../sdk/typescript/src/accepted-audit.js"; import { promises as fs } from "node:fs"; import { dirname, join } from "node:path"; -import { getCodexSecurityDeepReducerInputs } from "../artifact-deep-reducer.js"; +import { readDeepReductionSources } from "../artifact-deep-reducer.js"; import { - validateDiscoveryArtifacts, + readDiscoveryAuditDraft, validateReducerArtifacts } from "./artifact-validation.js"; import type { DeepReductionInput, ReducerArtifactValidation } from "./artifact-validation.js"; @@ -41,8 +42,6 @@ export interface AcceptedDiscovery { completionSequence: number; attempt: number; threadId?: string; - basePromptSha256: string; - attemptPromptPaths: string[]; } export type DiscoveryOutcome = @@ -66,8 +65,6 @@ export interface SuccessfulDedupOutcome { newFindings: number; attempt: number; threadId?: string; - basePromptSha256: string; - attemptPromptPaths: string[]; run: DeepScanRunState; } @@ -81,27 +78,12 @@ export interface FailedDedupOutcome { export type DedupOutcome = SuccessfulDedupOutcome | FailedDedupOutcome; -/** Audit evidence for every logical SDK execution, including failures and cancellation. */ -export interface WorkerExecutionAudit { - id: string; - label: string; - kind: DeepScanWorkerKind; - status: "succeeded" | "failed" | "canceled"; - attempt: number; - threadId?: string; - promptPath: string; - artifactDir: string; - basePromptSha256: string; - attemptPromptPaths: string[]; - error?: string; - failureKind?: DeepScanReplaceableFailureKind; -} - export interface ReducerRequest { id: string; label: string; consumed: AcceptedDiscovery[]; previousReducerResultPath?: string; + previousSourceCoverage?: DeepReductionInput["sourceCoverage"]; } export interface DeepScanWorkerRunnerOptions { @@ -115,13 +97,11 @@ export interface DeepScanWorkerRunnerOptions { log: DeepScanLogger; retryDelaysMs: readonly number[]; signal: AbortSignal; - recordExecution?: (execution: WorkerExecutionAudit) => void; } interface WorkerAttemptEvidence { attempt: number; threadId?: string; - attemptPromptPaths: string[]; } type WorkerAttemptOutcome = @@ -185,8 +165,9 @@ export class DeepScanWorkerRunner { artifactContext: { root: artifactDir, layout: "worker" }, subagents: run.config.subagents, validate: async () => { - await validateDiscoveryArtifacts(artifacts, files.resultPath, run.scanId); - discoveryValidated = true; + const draft = await readDiscoveryAuditDraft(artifacts, files.resultPath, run.scanId); + discoveryValidated = draft.complete !== false; + return draft; }, beforeRetry: async (attempt) => { await archiveDirectory( @@ -207,15 +188,6 @@ export class DeepScanWorkerRunner { if (!discoveryValidated) { await fs.rm(files.resultPath, { force: true }); } - const basePromptSha256 = sha256(basePrompt); - this.recordExecution({ - id: workerId, - label: workerLabel, - kind: "discovery", - promptPath, - artifactDir, - basePromptSha256 - }, outcome); if (outcome.status === "failed") { return { type: "discovery", @@ -257,11 +229,7 @@ export class DeepScanWorkerRunner { }; let persisted: PersistedDeepScanWorker; try { - persisted = await this.replayStoreMutation( - "discovery_acceptance_replay", - workerId, - async () => await this.options.store.updateWorker(acceptance) - ); + persisted = await this.options.store.updateWorker(acceptance); } catch (error) { if (!this.options.signal.aborted) throw error; return { type: "discovery", status: "canceled", workerId }; @@ -280,12 +248,10 @@ export class DeepScanWorkerRunner { id: workerId, label: workerLabel, artifactDir, - resultPath: files.resultPath, + resultPath: persisted.acceptedResultPath ?? files.resultPath, completionSequence: persisted.completionSequence, attempt: outcome.attempt, - threadId: outcome.threadId, - basePromptSha256, - attemptPromptPaths: outcome.attemptPromptPaths + threadId: outcome.threadId } }; } @@ -294,9 +260,9 @@ export class DeepScanWorkerRunner { const { id: reducerId, label: reducerLabel, - consumed, - previousReducerResultPath + previousSourceCoverage } = request; + let { consumed, previousReducerResultPath } = request; const { artifacts, run } = this.options; const reducerRoot = join(artifacts.dedupRoot, reducerLabel); const artifactDir = join(reducerRoot, "output"); @@ -312,13 +278,23 @@ export class DeepScanWorkerRunner { })) }); await writePrivateFile(promptPath, basePrompt); - await this.options.store.claimDedup({ + const claimed = await this.options.store.claimDedup({ id: reducerId, scanId: run.scanId, workerIds: consumed.map((worker) => worker.id), promptPath, artifactDir }); + const claim = claimed?.persistedMergeClaims?.find((item) => item.workerId === reducerId); + if (claim) { + previousReducerResultPath = claim.previousResultPath; + const inputs = claimed?.persistedDedupInputs?.filter((item) => item.dedupWorkerId === reducerId) ?? []; + consumed = inputs.sort((a, b) => a.inputOrder - b.inputOrder).map((item) => { + const discovery = consumed.find((worker) => worker.id === item.discoveryWorkerId); + if (!discovery || !item.resultManifestPath) throw new Error("The reducer claim is missing an accepted input."); + return { ...discovery, resultPath: item.resultManifestPath, attempt: item.attempt ?? discovery.attempt }; + }); + } this.options.log({ event: "dedup_claimed", scanId: run.scanId, @@ -326,6 +302,7 @@ export class DeepScanWorkerRunner { count: consumed.length }); + const persistSourceCoverage = false; const artifactContext = { root: artifactDir, repoRoot: run.targetPath, @@ -333,13 +310,17 @@ export class DeepScanWorkerRunner { layout: "reducer" as const, deepReducer: { scanRoot: artifacts.scanDir, - claimedWorkers: consumed.map((worker) => ({ id: worker.id, resultPath: worker.resultPath })), + persistSourceCoverage, + claimedWorkers: consumed.map((worker) => ({ id: worker.id, resultPath: worker.resultPath, artifactDir: worker.artifactDir, attempt: worker.attempt })), previousReducerResultPath } }; // Snapshot inputs before execution: direct file output has the same // conservation checks as the MCP writer without rereading consumed sources. - const sources = await getCodexSecurityDeepReducerInputs(artifactContext); + const sources = await readDeepReductionSources(artifactContext); + if (sources.previous && previousSourceCoverage !== undefined) { + sources.previous.sourceCoverage = structuredClone(previousSourceCoverage); + } let reducerValidation: ReducerArtifactValidation | undefined; let outcome = await this.runWorkerWithRetries({ workerId: reducerId, @@ -356,7 +337,8 @@ export class DeepScanWorkerRunner { resultPath, reducerId, previousReducerResultPath, - sources + sources, + persistSourceCoverage }, run.scanId); }, beforeRetry: async (attempt) => { @@ -377,15 +359,6 @@ export class DeepScanWorkerRunner { }, outcome.attempt, outcome.threadId); outcome = { ...outcome, status: "canceled" }; } - const basePromptSha256 = sha256(basePrompt); - this.recordExecution({ - id: reducerId, - label: reducerLabel, - kind: "dedup", - promptPath, - artifactDir, - basePromptSha256 - }, outcome); if (outcome.status === "failed") { if (outcome.error instanceof DeepScanNonRetryableError) throw outcome.error; return { @@ -426,29 +399,28 @@ export class DeepScanWorkerRunner { newFindings: reducerValidation.newFindings, resultManifestPath: resultPath }; - const committed = await this.replayStoreMutation( - "dedup_commit_replay", - reducerId, - async () => await this.options.store.commitDedup(commit) - ); + const committed = await this.options.store.commitDedup(commit); + const accepted = committed.committedMerge; + const acceptedPath = accepted?.resultManifestPath ?? resultPath; + // V1 checkpoints omit host-only coverage; retain the validated projection. + const acceptedResult = reducerValidation.result; + const newFindings = accepted?.newFindings ?? reducerValidation.newFindings; this.options.log({ event: "dedup_committed", scanId: run.scanId, workerId: reducerId, count: consumed.length, - newFindings: reducerValidation.newFindings + newFindings }); return { type: "dedup", id: reducerId, consumed, - resultPath, - result: reducerValidation.result, - newFindings: reducerValidation.newFindings, + resultPath: acceptedPath, + result: acceptedResult, + newFindings, attempt: outcome.attempt, threadId: outcome.threadId, - basePromptSha256, - attemptPromptPaths: outcome.attemptPromptPaths, run: committed }; } @@ -461,7 +433,7 @@ export class DeepScanWorkerRunner { artifactDir: string; artifactContext?: CodexWorkerArtifactContext; subagents: number; - validate: () => Promise; + validate: () => Promise; beforeRetry: (attempt: number) => Promise; }): Promise { const { run, signal } = this.options; @@ -470,10 +442,9 @@ export class DeepScanWorkerRunner { let continuationPrompt: string | undefined; let lastThreadId: string | undefined; let executionPromptPath = input.promptPath; - const attemptPromptPaths = [input.promptPath]; for (let attempt = 1; attempt <= maximumAttempts; attempt += 1) { if (signal.aborted) { - return await this.cancelAttempt(input, attempt, lastThreadId, attemptPromptPaths); + return await this.cancelAttempt(input, attempt, lastThreadId); } let validationStarted = false; let validationCompleted = false; @@ -497,7 +468,7 @@ export class DeepScanWorkerRunner { attempt }); try { - const result = await this.options.executor.run({ + const execute = () => this.options.executor.run({ kind: input.kind, promptPath: executionPromptPath, // Discovery workers write only to their isolated directory. Setup and @@ -526,19 +497,36 @@ export class DeepScanWorkerRunner { }); } }); - if (signal.aborted) { - return await this.cancelAttempt(input, attempt, activeThreadId, attemptPromptPaths); - } - validationStarted = true; - try { - await input.validate(); - } catch (validationError) { - throw withWorkerDiagnostics(validationError, result.diagnostics); - } - validationCompleted = true; - if (signal.aborted) { - return await this.cancelAttempt(input, attempt, activeThreadId, attemptPromptPaths); + const accept = async (result: Awaited>) => { + validationStarted = true; + let accepted: ScanDraftInput | void; + try { + accepted = await input.validate(); + } catch (validationError) { + throw withWorkerDiagnostics(validationError, result.diagnostics); + } + validationCompleted = accepted?.complete !== false; + return accepted === undefined ? {} : auditEvidence(accepted); + }; + // Reducers keep their aggregate contract; discovery uses the shared audit. + const audit = input.kind === "discovery" + ? await runAcceptedAudit({ signal, execute, accept }) + : undefined; + let result: Awaited>; + if (audit) { + if (audit.status === "checkpoint") { + throw withWorkerDiagnostics( + new Error("Standard scan worker wrote only a checkpoint; its audit is not complete."), + audit.execution.diagnostics, + ); + } + result = audit.execution; + } else { + result = await execute(); + if (signal.aborted) return await this.cancelAttempt(input, attempt, activeThreadId); + await accept(result); } + if (signal.aborted) return await this.cancelAttempt(input, attempt, activeThreadId); this.options.log({ event: "worker_succeeded", scanId: run.scanId, @@ -549,12 +537,11 @@ export class DeepScanWorkerRunner { return { status: "succeeded", attempt, - threadId: result.threadId ?? activeThreadId, - attemptPromptPaths: [...attemptPromptPaths] + threadId: result.threadId ?? activeThreadId }; } catch (error) { if (signal.aborted) { - return await this.cancelAttempt(input, attempt, activeThreadId, attemptPromptPaths); + return await this.cancelAttempt(input, attempt, activeThreadId); } const normalized = asError(error); const policyRefusal = input.kind === "discovery" @@ -588,8 +575,7 @@ export class DeepScanWorkerRunner { ? {} : { consecutiveErrors: persistedFailure.consecutiveErrors }), attempt, - threadId: activeThreadId, - attemptPromptPaths: [...attemptPromptPaths] + threadId: activeThreadId }; } await this.options.store.updateWorker({ @@ -627,7 +613,6 @@ export class DeepScanWorkerRunner { failedAttempt: attempt, error: normalized }); - attemptPromptPaths.push(executionPromptPath); } } const delayMs = Math.ceil( @@ -645,7 +630,7 @@ export class DeepScanWorkerRunner { await this.options.clock.sleep(delayMs, signal); } catch (sleepError) { if (signal.aborted) { - return await this.cancelAttempt(input, attempt, activeThreadId, attemptPromptPaths); + return await this.cancelAttempt(input, attempt, activeThreadId); } throw sleepError; } @@ -680,32 +665,6 @@ export class DeepScanWorkerRunner { }); } - /** Replay idempotent SQLite commits when their process response is ambiguous. */ - private async replayStoreMutation( - event: string, - workerId: string, - operation: () => Promise - ): Promise { - try { - return await operation(); - } catch (firstError) { - this.options.log({ - event, - scanId: this.options.run.scanId, - workerId, - reason: errorKind(firstError) - }); - try { - return await operation(); - } catch (replayError) { - throw new Error( - `Deep Scan persistence replay failed: ${asError(replayError).message}`, - { cause: firstError } - ); - } - } - } - private async cancelAttempt( input: { workerId: string; @@ -714,36 +673,15 @@ export class DeepScanWorkerRunner { artifactDir: string; }, attempt: number, - threadId: string | undefined, - attemptPromptPaths: string[] + threadId: string | undefined ): Promise { await this.persistWorkerCancellation(input, attempt, threadId); return { status: "canceled", attempt, - threadId, - attemptPromptPaths: [...attemptPromptPaths] + threadId }; } - - private recordExecution( - input: Omit, - outcome: WorkerAttemptOutcome - ): void { - this.options.recordExecution?.({ - ...input, - status: outcome.status, - attempt: outcome.attempt, - ...(outcome.threadId ? { threadId: outcome.threadId } : {}), - attemptPromptPaths: [...outcome.attemptPromptPaths], - ...(outcome.status === "failed" ? { - error: outcome.error.message, - ...(outcome.replaceableFailureKind - ? { failureKind: outcome.replaceableFailureKind } - : {}) - } : {}) - }); - } } /** @@ -880,22 +818,10 @@ function validationErrorData(error: Error, message: string): Record { + const script = path.join(pluginRoot, "scripts", "workbench_db.py"); + const pythonArgs = selectFinalization + ? ["-c", "import runpy, sys; script = sys.argv.pop(1); runpy.run_path(script)['main'](select_finalization=True)", script, ...args] + : [script, ...args]; + const execution = exec(process.env.PYTHON || "python3", pythonArgs, { + env: { ...process.env, CODEX_HOME: codexHome, CODEX_SECURITY_STATE_DIR: path.join(root, "state") }, + }); + if (input !== undefined) execution.child.stdin.end(input); + const { stdout } = await execution; + return JSON.parse(stdout); + }; + const store = new WorkbenchDeepScanStore(runWorkbench); + let { run } = await store.begin({ targetPath, scope: ".", threadId, scanRoot }); + assert.equal(run.workflowVersion, "deep-security-scan/v1", "the prior reader starts the legacy workflow"); + if (selectedRecovery) { + ({ run } = await store.claimCoordinator({ scanId: run.scanId, threadId })); + } else { + // Seed an existing v1 run for legacy direct publication and in-memory coverage recovery. + await exec(process.env.PYTHON || "python3", ["-c", [ + "import sqlite3, sys", + "with sqlite3.connect(sys.argv[1]) as db:", + " db.execute(\"UPDATE deep_scan_runs SET workflow_version = 'deep-scan-mcp/v1' WHERE scan_id = ?\", (sys.argv[2],))", + ].join("\n"), path.join(root, "state", "workbench.sqlite3"), run.scanId]); + run = await store.get(run.scanId, threadId); + assert.equal(run.workflowVersion, "deep-scan-mcp/v1"); + } + const context = await createScanArtifactContext(run.scanId, runWorkbench, { requireRunning: true }); + const rawSources = new Map(); + const writeReduction = async (context) => { + const inputs = await getCodexSecurityDeepReducerInputs(context); + const sources = [...(inputs.previous?.findings ?? []), ...inputs.discoveries.flatMap((source) => source.result.findings)]; + const findings = []; + if (sources.length) { + const finding = structuredClone(sources[0]); + finding.provenance.sourceFindingIds = [...new Set(sources.flatMap((source) => source.provenance.sourceFindingIds))]; + delete finding.provenance.sourceFindings; + findings.push(finding); + } + await recordCodexSecurityDeepReduction(context, { scanId: run.scanId, findings }); + }; + const writeDiscovery = async (artifactDir, index) => { + const status = statuses[index]; + const pending = completeness === "partial" && status !== "complete"; + const coverage = { + completeness: status, + surfaces: [{ id: "shared-surface", label: "Archive route", disposition: pending ? "needs_follow_up" : "no_issue_found", receiptRefs: ["artifacts/review.md"] }], + explicitExclusions: [{ pattern: "vendor/", reason: "External dependency." }], + deferred: pending ? [{ id: "same-id", candidateId: "candidate-1", reason: index === 0 ? "Verify entry boundaries." : "Verify symbolic links.", paths: ["source.py"], surfaceIds: ["shared-surface"] }] : [], + openQuestions: pending ? [{ question: `Deployment question ${index + 1}.` }] : [], + }; + await mkdir(path.join(artifactDir, "artifacts"), { recursive: true }); + await writeFile(path.join(artifactDir, "artifacts", "review.md"), "Synthetic review evidence.\n"); + const resultPath = path.join(artifactDir, "result.json"); + const findings = materialFindings && index < 2 ? [{ + ruleId: "archive-extraction", identity: { anchor: "archive-destination" }, + title: "Archive entries can escape the destination", + summary: "Archive extraction requires both entry containment and symbolic-link handling.", + severity: { level: "high" }, + confidence: { level: "high", rationale: "Synthetic accepted source evidence." }, + taxonomy: { category: "path-traversal", cwe: ["CWE-22"] }, + locations: [{ path: "source.py", startLine: 1, endLine: 1 }], + remediation: materialRemediations[index], + remediationTests: [materialRemediationTests[index]], + provenance: { source: "local_plugin" }, + }] : []; + const bytes = JSON.stringify({ scanId: run.scanId, complete: true, findings, coverage }); + await writeFile(resultPath, bytes); + rawSources.set(resultPath, bytes); + if (immutableInputs) { + await saveScanDraftCheckpoint({ root: artifactDir, repoRoot: targetPath, layout: "worker" }, JSON.parse(bytes)); + const head = JSON.parse(await readFile(path.join(artifactDir, "checkpoint-head.json"), "utf8")); + const acceptedPath = path.join(artifactDir, "checkpoints", head.checkpoint); + rawSources.set(acceptedPath, await readFile(acceptedPath, "utf8")); + return acceptedPath; + } + return resultPath; + }; + if (resume) { + const workers = []; + const seeded = continueAfterResume ? statuses.slice(0, -1) : statuses; + for (const index of seeded.keys()) { + const workerRoot = path.join(run.scanDir, "artifacts", "deep_discovery", "workers", `discovery-${String(index + 1).padStart(4, "0")}`); + const artifactDir = path.join(workerRoot, "output"); + const worker = { id: randomUUID(), scanId: run.scanId, kind: "discovery", promptPath: path.join(workerRoot, "prompt.md"), artifactDir, attempt: index === 0 ? 2 : 1 }; + const writtenPath = await writeDiscovery(artifactDir, index); + const resultManifestPath = discardMutableResults ? path.join(artifactDir, "result.json") : writtenPath; + await writeFile(worker.promptPath, "Synthetic discovery prompt.\n"); + for (const status of ["queued", "running", "succeeded"]) { + await store.updateWorker({ ...worker, status, ...(status === "succeeded" ? { resultManifestPath } : {}) }); + } + workers.push({ ...worker, resultPath: resultManifestPath }); + } + const batches = splitSeededReducers ? [workers.slice(0, 2), workers.slice(2)] : [workers]; + let lastReducerId; + let lastReducerReference; + for (const [index, batch] of batches.entries()) { + const label = `dedup-${String(index + 1).padStart(4, "0")}`; + const artifactDir = path.join(run.scanDir, "artifacts", "deep_discovery", "dedup", label, "output"); + const promptPath = path.join(path.dirname(artifactDir), "prompt.md"); + await mkdir(artifactDir, { recursive: true }); + await writeFile(promptPath, "Synthetic reducer prompt.\n"); + const id = randomUUID(); + const claimed = await store.claimDedup({ id, scanId: run.scanId, workerIds: batch.map((worker) => worker.id), artifactDir, promptPath }); + if (discardMutableResults) { + await store.updateWorker({ id, scanId: run.scanId, kind: "dedup", status: "running", artifactDir, promptPath, attempt: 1 }); + } + const resultManifestPath = path.join(artifactDir, "result.json"); + // Legacy accepted reducers omitted coverage entirely. + if (materialFindings) { + await writeReduction({ + root: artifactDir, repoRoot: targetPath, scanId: run.scanId, layout: "reducer", + deepReducer: { + scanRoot: run.scanDir, + claimedWorkers: batch.map((worker) => { + const input = claimed.persistedDedupInputs.find((input) => input.dedupWorkerId === id && input.discoveryWorkerId === worker.id); + return { ...worker, resultPath: input.resultManifestPath ?? worker.resultPath, attempt: input.attempt ?? worker.attempt }; + }), + persistSourceCoverage: selectedRecovery, + previousReducerResultPath: claimed.persistedMergeClaims?.find((claim) => claim.workerId === id)?.previousResultPath, + }, + }); + } else { + await writeFile(resultManifestPath, JSON.stringify({ scanId: run.scanId, findings: [] })); + } + rawSources.set(resultManifestPath, await readFile(resultManifestPath, "utf8")); + const committed = await store.commitDedup({ id, scanId: run.scanId, newFindings: materialFindings && index === 0 ? 1 : 0, resultManifestPath }); + lastReducerReference = committed.persistedWorkers.find((worker) => worker.id === id).resultManifestPath; + lastReducerId = id; + } + if (legacyAttempts) { + // Migrated discoveries and prior reducers can have frozen claims without attempt rows. + await exec(process.env.PYTHON || "python3", ["-c", [ + "import sqlite3, sys", + "with sqlite3.connect(sys.argv[1]) as db:", + " db.execute(\"DELETE FROM deep_scan_attempts WHERE worker_id != ?\", (sys.argv[2],))", + ].join("\n"), path.join(root, "state", "workbench.sqlite3"), lastReducerId]); + } + run = await store.get(run.scanId, threadId); + if (discardMutableResults) { + for (const worker of run.persistedWorkers) { + const acceptedPath = worker.acceptedResultPath ?? run.persistedDedupInputs + .find((input) => input.discoveryWorkerId === worker.id)?.resultManifestPath + ?? run.persistedMergeClaims.find((claim) => claim.previousWorkerId === worker.id)?.previousResultPath; + assert.ok(acceptedPath, "the real store retains an accepted reference"); + assert.notEqual(acceptedPath, worker.resultManifestPath); + rawSources.set(acceptedPath, await readFile(acceptedPath, "utf8")); + rawSources.delete(worker.resultManifestPath); + await rm(worker.resultManifestPath); + } + } + if (selectedRecovery) { + run = await store.selectFinalization({ + scanId: run.scanId, reason: "capped", manifestPath: path.join(run.scanDir, "scan-manifest.json"), + resultPath: lastReducerReference, omittedWorkerIds: [], + }); + } + } + let discoveryCalls = 0; + const executor = { + async run(request) { + assert.equal(resume && !continueAfterResume, false, "accepted legacy sources should resume without new model work"); + const thread = request.resumeThreadId ?? randomUUID(); + await request.onThreadStarted?.(thread); + if (request.kind === "discovery") { + discoveryCalls++; + const index = Number(path.basename(path.dirname(request.promptPath)).split("-").at(-1)) - 1; + if (index === 0 && !request.resumeThreadId) return { threadId: thread, finalResponse: "Continue the unfinished audit." }; + await writeDiscovery(request.artifactContext.root, index); + } else { + if (immutableInputs) { + const current = await store.get(run.scanId, threadId); + for (const claimed of request.artifactContext.deepReducer.claimedWorkers) { + const accepted = current.persistedWorkers.find((worker) => worker.id === claimed.id); + assert.equal(claimed.resultPath, accepted.acceptedResultPath ?? accepted.resultManifestPath, "the reducer uses the exact accepted input"); + assert.equal(claimed.artifactDir, accepted.artifactDir, "receipts retain their original output owner"); + } + } + await writeReduction({ ...request.artifactContext, repoRoot: targetPath, scanId: run.scanId }); + } + return { threadId: thread, finalResponse: "Audit finished." }; + }, + }; + let publicationCalls = 0; + const options = { + run, store, executor, pluginRoot, retryDelaysMs: [1], + onComplete: async (draft, signal, publication) => { + publicationCalls++; + if (selectedRecovery && publicationCalls === 1) throw new Error("Synthetic selected publication failure"); + await recordCodexSecurityScanDraftViaWorkbench(context, draft, runWorkbench, signal, selectedRecovery ? publication : undefined); + }, + }; + const coordinator = new DeepScanCoordinator(options); + coordinator.start(); + let terminal; + if (selectedRecovery) { + await assert.rejects(coordinator.wait(undefined, 30_000), /Synthetic selected publication failure/); + const pending = await store.get(run.scanId, threadId); + assert.equal(pending.status, "running"); + assert.deepEqual(pending.finalizationInput, run.finalizationInput); + const worker = pending.persistedWorkers.find((worker) => worker.kind === "discovery"); + const rejected = { scanId: run.scanId, complete: false, findings: [], coverage: { + completeness: "complete", surfaces: [], explicitExclusions: [], deferred: [], + } }; + await saveScanDraftCheckpoint({ root: worker.artifactDir, repoRoot: targetPath, layout: "worker" }, rejected); + const replacement = path.join(worker.artifactDir, "result.json"); + await writeFile(replacement, JSON.stringify(rejected)); + await assert.rejects(validateDiscoveryArtifacts(createDeepScanArtifacts(run.scanDir), replacement, run.scanId), /only a checkpoint/); + const headPath = path.join(worker.artifactDir, "checkpoint-head.json"); + const head = JSON.parse(await readFile(headPath, "utf8")); + for (const file of [replacement, headPath, path.join(worker.artifactDir, "checkpoints", head.checkpoint)]) { + rawSources.set(file, await readFile(file, "utf8")); + } + const restarted = new DeepScanCoordinator({ ...options, run: pending }); + restarted.start(); + terminal = await restarted.wait(undefined, 30_000); + assert.deepEqual(terminal.finalizationInput, run.finalizationInput); + assert.equal(publicationCalls, 2); + } else { + terminal = await coordinator.wait(undefined, 30_000); + } + assert.equal(terminal?.status, "succeeded", terminal?.error); + assert.equal(terminal.noNewStreak, materialFindings ? (resume && !continueAfterResume && !splitSeededReducers ? 0 : 1) : statuses.length, + "source coverage must not change stopping policy"); + assert.equal(discoveryCalls, resume ? (continueAfterResume ? 1 : 0) : statuses.length + 1); + const accepted = await store.get(run.scanId, threadId); + for (const worker of accepted.persistedWorkers.filter((worker) => worker.kind === "dedup")) { + const resultPath = worker.acceptedResultPath + ?? accepted.persistedMergeClaims.find((claim) => claim.previousWorkerId === worker.id)?.previousResultPath + ?? worker.resultManifestPath; + const result = JSON.parse(await readFile(resultPath, "utf8")); + assert.equal(Object.hasOwn(result, "sourceCoverage"), selectedRecovery, "coverage persistence follows the accepted workflow version"); + if (!rawSources.has(worker.resultManifestPath)) { + for (const name of await readdir(path.join(worker.artifactDir, "checkpoints"))) { + const checkpoint = JSON.parse(await readFile(path.join(worker.artifactDir, "checkpoints", name), "utf8")); + assert.equal(Object.hasOwn(checkpoint, "sourceCoverage"), selectedRecovery, "checkpoint coverage follows the accepted workflow version"); + } + } + } + await runWorkbench(["complete-scan", "--scan-id", run.scanId]); + for (const [file, bytes] of rawSources) assert.equal(await readFile(file, "utf8"), bytes); + return { scanDir: run.scanDir, threadId, terminal }; +} + +export const materialRemediations = [ + "Check the destination before writing the archive entry.", + "Reject symbolic links before opening the destination.", +]; +export const materialRemediationTests = [ + "Reject an archive entry outside the destination.", + "Reject a symbolic link inside the destination.", +]; + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const result = await publishCoverageFixture(process.argv[2], process.argv[3], { resume: process.argv[4] === "true", continueAfterResume: process.argv[5] === "true" }); + process.stdout.write(JSON.stringify(result)); +} diff --git a/plugins/codex-security/mcp-app/tests/deep_scan_publication_cases.mjs b/plugins/codex-security/mcp-app/tests/deep_scan_publication_cases.mjs index 5e44d1796..6c192c1b7 100644 --- a/plugins/codex-security/mcp-app/tests/deep_scan_publication_cases.mjs +++ b/plugins/codex-security/mcp-app/tests/deep_scan_publication_cases.mjs @@ -55,7 +55,7 @@ export async function testDeepScanPublication({ assert.deepEqual(completed[0].findings, [], "late worker findings are not appended to the saturated aggregate"); } - async function testSuccessfulDeepCoverageIgnoresWorkerAndReducerReviewStatus() { + async function testSuccessfulDeepCoveragePreservesWorkerReviewStatus() { const fixture = await fixtureRun({ workers: 2, subagents: 0, stopAfterNoNew: 2, maxDiscoveryRuns: 2 }); const store = new FakeStore(fixture.run); const executor = new FakeExecutor(); @@ -87,9 +87,16 @@ export async function testDeepScanPublication({ const terminal = await coordinator.wait(undefined, 5_000); assert.equal(terminal?.status, "succeeded", terminal?.error); assert.equal(completed.length, 1); - assert.deepEqual(completed[0].coverage, { - completeness: "complete", surfaces: [], explicitExclusions: [], deferred: [], - }); + assert.equal(completed[0].coverage.completeness, "partial"); + assert.equal(completed[0].coverage.deferred.length, 2); + assert.equal(completed[0].coverage.surfaces.length, 4); + assert.equal(completed[0].coverage.reviews.length, 2); + assert.deepEqual(new Set(completed[0].coverage.reviews.map((review) => review.completeness)), + new Set(["partial", "unknown"])); + for (const item of completed[0].coverage.deferred) { + assert.equal(item.provenance.attempt, 1); + assert.ok(store.workers.has(item.provenance.workerId)); + } for (const worker of store.workers.values()) { if (worker.kind !== "discovery") continue; const draft = JSON.parse(await readFile(worker.resultManifestPath, "utf8")); @@ -141,6 +148,7 @@ export async function testDeepScanPublication({ worker.kind === "dedup" && worker.status === "succeeded" )); const { coverage, ...publishedReduction } = completed[0]; + assert.equal(coverage.reviews.length, 2, "both completed audits retain source coverage in the publication"); assert.deepEqual( publishedReduction, JSON.parse(await readFile(acceptedReducer.resultManifestPath, "utf8")), @@ -150,6 +158,7 @@ export async function testDeepScanPublication({ async function testPublicationUsesAcceptedReducerSnapshot() { const fixture = await fixtureRun({ workers: 1, subagents: 0, stopAfterNoNew: 1, maxDiscoveryRuns: 1 }); + fixture.run.coordinatorGeneration = 3; const store = new FakeStore(fixture.run); const commitDedup = store.commitDedup.bind(store); store.commitDedup = async (commit) => { @@ -163,21 +172,30 @@ export async function testDeepScanPublication({ return structuredClone(store.run); }; const completed = []; + const published = []; const coordinator = new DeepScanCoordinator({ run: fixture.run, store, executor: new FakeExecutor({ discoveryCandidateId: "accepted-finding" }), pluginRoot: fixture.pluginRoot, clock: immediateClock, - onComplete: async (draft) => completed.push(structuredClone(draft)), + onComplete: async (draft, _signal, publication) => { + completed.push(structuredClone(draft)); + published.push(publication); + }, }); coordinator.start(); const terminal = await coordinator.wait(undefined, 5_000); assert.equal(terminal?.status, "succeeded", terminal?.error); assert.equal(completed[0].findings[0].provenance.candidateId, "accepted-finding"); assert.equal(completed[0].coverage.completeness, "complete"); + const reducer = [...store.workers.values()].find((worker) => worker.kind === "dedup"); + assert.deepEqual(published, [{ + coordinatorGeneration: 3, + resultPath: reducer.resultManifestPath, + }]); } await testSaturationOmitsWorkerAcceptedDuringCancellation(); - await testSuccessfulDeepCoverageIgnoresWorkerAndReducerReviewStatus(); + await testSuccessfulDeepCoveragePreservesWorkerReviewStatus(); await testSaturationIgnoresDiscoveryCancellationWriteFailure(); await testPublicationUsesAcceptedReducerSnapshot(); } diff --git a/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_bank.mjs b/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_bank.mjs new file mode 100644 index 000000000..0fed15b23 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_bank.mjs @@ -0,0 +1,180 @@ +// Synthetic accepted artifacts. Expected fixes are independent of display identities. +export const scanId = "7fc17317-9594-49e0-b06a-d72fd7e14bba"; +export const bankVersion = "accepted-source-bank/v1"; + +export const fixes = { + owner: "Check document ownership before returning document contents.", + path: "Resolve archive paths and reject entries outside the extraction directory.", + sql: "Bind the search term as a SQL parameter.", + html: "HTML-encode the search term before rendering the response.", +}; + +function finding(id, remediation, extra = {}) { + return { + ruleId: "synthetic." + id, + identity: { anchor: id }, + title: "Request boundary " + id, + summary: "A request-controlled value crosses an unchecked boundary.", + severity: { level: "high" }, + confidence: { + level: "high", + rationale: "A synthetic local test reaches the sink.", + }, + taxonomy: { category: "input-validation", cwe: ["CWE-20"] }, + locations: [{ path: "src/routes.py", startLine: 12, endLine: 14 }], + remediation, + provenance: { source: "local_plugin" }, + ...extra, + }; +} + +function coverage(completeness = "complete", deferred = []) { + return { completeness, surfaces: [], explicitExclusions: [], deferred }; +} + +function worker(id, findings, scanCoverage = coverage()) { + return { + id, + result: { scanId, complete: true, findings, coverage: scanCoverage }, + }; +} + +export const workers = [ + worker( + "worker-owner", + [ + finding("owner", fixes.owner, { + summary: + "An authenticated user can read another user's document by changing its ID.", + validation: { + summary: "The ownership check is absent; authentication is required.", + }, + }), + ], + coverage("partial", [ + { + candidateId: "candidate-1", + reason: "Check the alternate document handler.", + paths: ["src/alternate.py"], + }, + ]), + ), + worker( + "worker-path", + [finding("path", fixes.path)], + coverage("partial", [ + { + candidateId: "candidate-1", + reason: "Check symlink extraction separately.", + paths: ["src/archive.py"], + }, + ]), + ), + worker("worker-owner-duplicate", [ + finding("owner-copy", fixes.owner, { + summary: "Document contents may be reachable without authentication.", + severity: { level: "critical" }, + confidence: { + level: "low", + rationale: "Authentication middleware was not examined.", + }, + validation: { + summary: + "The unauthenticated claim is untested; ownership check is absent.", + }, + }), + ]), + worker("worker-bundled", [ + finding("search-bundle", fixes.sql + " " + fixes.html, { + summary: + "The search route interpolates the query into SQL and separately into HTML.", + locations: [{ path: "src/search.py", startLine: 5, endLine: 9 }], + }), + ]), +]; + +export const sourceGroups = { + "worker-owner:0": "owner", + "worker-path:0": "path", + "worker-owner-duplicate:0": "owner", + "worker-bundled:0": "search-bundle", +}; + +export const sourceFixes = { + "worker-owner:0": ["owner"], + "worker-path:0": ["path"], + "worker-owner-duplicate:0": ["owner"], + "worker-bundled:0": ["sql", "html"], +}; + +// This history is distinct from the immutable accepted terminal bank above. +// A newer rejection is authoritative for this logical worker's final result. +export const rejectionHistory = { + workerId: "worker-rejected", + earlier: worker("worker-rejected", [finding("safe-query", fixes.sql)]).result, + latest: worker("worker-rejected", [], { + ...coverage(), + surfaces: [ + { + label: "Search SQL", + disposition: "rejected", + notes: + "The driver binds parameters; the earlier interpolation claim was disproved.", + }, + ], + }).result, +}; + +export function permutations(items) { + if (!items.length) return [[]]; + return items.flatMap((item, index) => + permutations(items.filter((_, i) => i !== index)).map((rest) => [ + item, + ...rest, + ]), + ); +} + +export function partitions(items) { + if (!items.length) return [[]]; + return items.flatMap((_, index) => + partitions(items.slice(index + 1)).map((rest) => [ + items.slice(0, index + 1), + ...rest, + ]), + ); +} + +export function originals(inputs) { + const sources = new Map(); + for (const current of inputs.previous?.findings ?? []) { + for (const source of current.provenance.sourceFindings ?? []) + sources.set(source.id, source.finding); + } + for (const discovery of inputs.discoveries) { + discovery.result.findings.forEach((value, i) => + sources.set(`${discovery.workerId}:${i}`, value), + ); + } + return sources; +} + +// Scripted proposals isolate host reconciliation from model variability. +export function proposal(inputs, groupFor = (id) => sourceGroups[id]) { + const grouped = new Map(); + for (const [id, value] of originals(inputs)) { + const group = groupFor(id); + if (!grouped.has(group)) grouped.set(group, { value, refs: [] }); + grouped.get(group).refs.push(id); + } + return { + scanId, + complete: true, + findings: [...grouped].map(([group, { value, refs }]) => ({ + ...structuredClone(value), + ruleId: "synthetic." + group, + identity: { anchor: group }, + provenance: { source: "local_plugin", sourceFindingIds: refs }, + })), + }; +} diff --git a/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_replay.mjs b/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_replay.mjs new file mode 100644 index 000000000..26b6a6163 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_replay.mjs @@ -0,0 +1,105 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { build } from "esbuild"; +import { proposal, scanId, workers } from "./accepted_source_bank.mjs"; + +export async function loadReducer() { + const bundled = await build({ + bundle: true, + entryPoints: [ + fileURLToPath( + new URL("../../src/artifact-deep-reducer.ts", import.meta.url), + ), + ], + format: "esm", + platform: "node", + write: false, + }); + return import( + "data:text/javascript;base64," + + Buffer.from(bundled.outputFiles[0].contents).toString("base64") + ); +} + +export async function prepareReplay(root, bank = workers) { + const workerRoot = path.join(root, "artifacts", "deep_discovery", "workers"); + const records = new Map(); + for (const [i, worker] of bank.entries()) { + const output = path.join(workerRoot, worker.id, "output"); + await mkdir(output, { recursive: true }); + const resultPath = path.join(output, "result.json"); + await writeFile(resultPath, JSON.stringify(worker.result) + "\n"); + records.set(worker.id, { + id: worker.id, + resultPath, + completionSequence: i + 1, + }); + } + return records; +} + +export async function runBatch( + reducer, + root, + records, + ids, + index, + previous, + propose = proposal, +) { + const output = path.join( + root, + "artifacts", + "deep_discovery", + "dedup", + `dedup-${index}`, + "output", + ); + await mkdir(output, { recursive: true }); + const context = { + root: output, + repoRoot: root, + scanId, + layout: "reducer", + deepReducer: { + scanRoot: root, + claimedWorkers: ids.map((id) => records.get(id)), + ...(previous ? { previousReducerResultPath: previous } : {}), + }, + }; + const inputs = await reducer.getCodexSecurityDeepReducerInputs(context); + const submitted = propose(inputs); + const receipt = await reducer.recordCodexSecurityDeepReduction( + context, + submitted, + ); + const resultPath = path.join(output, "result.json"); + return { + inputs, + submitted, + receipt, + resultPath, + result: JSON.parse(await readFile(resultPath, "utf8")), + }; +} + +export async function replay(reducer, root, batches, propose = proposal) { + const records = await prepareReplay(root); + const steps = []; + let previous; + for (const [index, batch] of batches.entries()) { + const step = await runBatch( + reducer, + root, + records, + batch, + index, + previous, + propose, + ); + steps.push(step); + previous = step.resultPath; + } + return { steps, result: steps.at(-1).result }; +} diff --git a/plugins/codex-security/mcp-app/tests/test_accepted_source_replay.mjs b/plugins/codex-security/mcp-app/tests/test_accepted_source_replay.mjs new file mode 100644 index 000000000..2b8709773 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_accepted_source_replay.mjs @@ -0,0 +1,69 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, realpath, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { + partitions, + permutations, + sourceFixes, + workers, +} from "./fixtures/accepted_source_bank.mjs"; +import { loadReducer, replay } from "./fixtures/accepted_source_replay.mjs"; + +test("accepted sources survive completion orders and eligible batch partitions", async () => { + const reducer = await loadReducer(); + const root = await realpath( + await mkdtemp(path.join(tmpdir(), "accepted-source-replay-")), + ); + let count = 0; + try { + for (const order of permutations(workers.map((worker) => worker.id))) { + for (const batches of partitions(order)) { + // Ordinary rolling runs require two successes for the first merge. + if (batches[0].length < 2) continue; + const runRoot = path.join(root, String(count++)); + const { result, steps } = await replay(reducer, runRoot, batches); + assert.equal(result.findings.length, 3); + const refs = result.findings.flatMap( + (finding) => finding.provenance.sourceFindingIds, + ); + assert.deepEqual(refs.toSorted(), Object.keys(sourceFixes).toSorted()); + for (const step of steps) { + assert.deepEqual( + step.receipt.consumedWorkerIds, + batches[steps.indexOf(step)], + ); + } + for (const finding of result.findings) { + for (const source of finding.provenance.sourceFindings) { + const [workerId, index] = source.id.split(":"); + const original = workers.find((worker) => worker.id === workerId) + .result.findings[Number(index)]; + assert.deepEqual(source.finding, original); + } + } + for (const worker of workers) { + const persisted = JSON.parse( + await readFile( + path.join( + runRoot, + "artifacts", + "deep_discovery", + "workers", + worker.id, + "output", + "result.json", + ), + "utf8", + ), + ); + assert.deepEqual(persisted, worker.result); + } + } + } + assert.equal(count, 96); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_deep_reducer.mjs b/plugins/codex-security/mcp-app/tests/test_artifact_deep_reducer.mjs index 5466999cc..c9cb7533d 100644 --- a/plugins/codex-security/mcp-app/tests/test_artifact_deep_reducer.mjs +++ b/plugins/codex-security/mcp-app/tests/test_artifact_deep_reducer.mjs @@ -14,7 +14,8 @@ const bundled = await build({ const { deepReducerInputsInputSchema, deepReductionInputSchema, - getCodexSecurityDeepReducerInputs, + getCodexSecurityDeepReducerInputs: getModelInputs, + readDeepReductionSources: getCodexSecurityDeepReducerInputs, recordCodexSecurityDeepReduction } = await import( "data:text/javascript;base64," @@ -99,6 +100,7 @@ try { layout: "reducer", deepReducer: { scanRoot, + persistSourceCoverage: true, claimedWorkers: [first, second] } }; @@ -127,13 +129,23 @@ try { /evidenceRefs must refer/, "live reducer submissions reject unknown evidence references instead of silently removing them", ); - assert.deepEqual(inputs, { + assert.deepEqual({ ...inputs, discoveries: inputs.discoveries.map(({ coverage, ...source }) => source) }, { discoveries: [ { workerId: first.id, result: withSourceRefs(first) }, { workerId: second.id, result: withSourceRefs(second) } ], previous: null }); + assert.equal(inputs.discoveries[0].coverage.completeness, "partial"); + assert.equal(inputs.discoveries[1].coverage.completeness, "unknown"); + assert.deepEqual(await getModelInputs(context), { + discoveries: inputs.discoveries.map(({ workerId, result }) => ({ workerId, result })), + previous: null, + }, "coverage accounting does not change reducer model inputs"); + assert.deepEqual(inputs.discoveries[0].coverage.deferred[0].provenance, + { workerId: first.id, candidateId: "candidate-upload" }); + assert.deepEqual(inputs.discoveries[0].coverage.surfaces[0].receiptRefs, + ["artifacts/deep_discovery/workers/discovery-0001/output/artifacts/missing-worker-receipt.md"]); assert.equal(JSON.stringify(inputs).includes(root), false); assert.equal(JSON.stringify(inputs).includes("result.json"), false); @@ -158,6 +170,10 @@ try { const outcome = await recordCodexSecurityDeepReduction(context, merged); const mergedWithSources = { ...merged, + sourceCoverage: { + ...inputs.discoveries[0].coverage, + reviews: [...inputs.discoveries[0].coverage.reviews, ...inputs.discoveries[1].coverage.reviews], + }, findings: [ retainedFinding(shared, [{ id: "worker-001:0", finding: shared }, { id: "worker-002:0", finding: shared }]), retainedFinding(independent, [{ id: "worker-002:1", finding: independent }]), @@ -176,7 +192,7 @@ try { assert.deepEqual( JSON.parse(await readFile(path.join(outputRoot, "checkpoints", checkpointNames[0]), "utf8")), mergedWithSources, - "reducer checkpoints retain the accepted findings and scope without coverage", + "reducer checkpoints retain accepted findings, scope and source coverage", ); assert.deepEqual( @@ -234,12 +250,16 @@ try { layout: "reducer", deepReducer: { scanRoot, + persistSourceCoverage: true, claimedWorkers: [third], previousReducerResultPath: path.join(outputRoot, "result.json") } }; const nextInputs = await getCodexSecurityDeepReducerInputs(nextContext); - assert.deepEqual(nextInputs, { + const { sourceCoverage, ...previousModelInput } = mergedWithSources; + assert.deepEqual((await getModelInputs(nextContext)).previous, previousModelInput, + "host coverage metadata is excluded from the previous model input too"); + assert.deepEqual({ ...nextInputs, discoveries: nextInputs.discoveries.map(({ coverage, ...source }) => source) }, { discoveries: [{ workerId: third.id, result: withSourceRefs(third) }], previous: mergedWithSources }); @@ -255,6 +275,10 @@ try { JSON.parse(await readFile(path.join(nextOutputRoot, "result.json"), "utf8")), { ...mergedWithSources, + sourceCoverage: { + ...mergedWithSources.sourceCoverage, + reviews: [...mergedWithSources.sourceCoverage.reviews, ...nextInputs.discoveries[0].coverage.reviews], + }, findings: [ retainedFinding(shared, [ { id: "worker-003:0", finding: shared }, diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_scan_draft.mjs b/plugins/codex-security/mcp-app/tests/test_artifact_scan_draft.mjs index 72f815fc3..aa5859ad2 100644 --- a/plugins/codex-security/mcp-app/tests/test_artifact_scan_draft.mjs +++ b/plugins/codex-security/mcp-app/tests/test_artifact_scan_draft.mjs @@ -591,6 +591,10 @@ try { const obsoleteCheckpointPath = path.join(deepParentRoot, "checkpoints", "obsolete.json"); await writeFile(obsoleteCheckpointPath, "{malformed obsolete checkpoint\n"); let deepWorkbenchWrites = 0; + const deepPublication = { + coordinatorGeneration: 3, + resultPath: path.join(deepParentRoot, "workers", "reducer", "result.json"), + }; await recordCodexSecurityScanDraftViaWorkbench( deepParentContext, acceptedDeepDraft, @@ -603,11 +607,15 @@ try { const checkpointPath = arguments_[arguments_.indexOf("--checkpoint-path") + 1]; const staged = JSON.parse(await readFile(draftPath, "utf8")); const stagedCheckpoint = JSON.parse(await readFile(checkpointPath, "utf8")); + assert.deepEqual(staged.deepScanPublication, deepPublication); + assert.equal(stagedCheckpoint.deepScanPublication, undefined); assert.deepEqual(staged.findings, acceptedDeepFindings); assert.deepEqual(staged.coverage, acceptedDeepCoverage); assert.deepEqual(stagedCheckpoint.findings, acceptedDeepDraft.findings); assert.equal(stagedCheckpoint.handoffClaimToken, undefined); }, + undefined, + deepPublication, ); assert.equal(deepWorkbenchWrites, 1, "terminal Deep drafts still publish through the workbench lock despite obsolete malformed checkpoints"); assert.deepEqual(await readdir(path.join(deepParentRoot, "drafts")), []); diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_storage.mjs b/plugins/codex-security/mcp-app/tests/test_artifact_storage.mjs index 0e5b33efd..3a3009955 100644 --- a/plugins/codex-security/mcp-app/tests/test_artifact_storage.mjs +++ b/plugins/codex-security/mcp-app/tests/test_artifact_storage.mjs @@ -21,6 +21,7 @@ try { await writeFile(path.join(repository, "example.py"), "value = 1\n"); await build({ bundle: true, + nodePaths: [fileURLToPath(new URL("../node_modules", import.meta.url))], define: { __dirname: JSON.stringify(applicationRoot), "import.meta.url": "__filename" }, entryPoints: [path.join(applicationRoot, "main.ts")], external: ["fsevents"], diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_storage_regressions.mjs b/plugins/codex-security/mcp-app/tests/test_artifact_storage_regressions.mjs index 3ce725944..53059a459 100644 --- a/plugins/codex-security/mcp-app/tests/test_artifact_storage_regressions.mjs +++ b/plugins/codex-security/mcp-app/tests/test_artifact_storage_regressions.mjs @@ -29,6 +29,7 @@ await fs.mkdir(repository); await fs.writeFile(path.join(repository, "example.py"), "value = 1\n"); await build({ bundle: true, + nodePaths: [fileURLToPath(new URL("../node_modules", import.meta.url))], define: { __dirname: JSON.stringify(applicationRoot), "import.meta.url": "__filename" }, entryPoints: [path.join(applicationRoot, "main.ts")], external: ["fsevents"], format: "cjs", loader: { ".md": "text" }, diff --git a/plugins/codex-security/mcp-app/tests/test_audit_acceptance_contract.mjs b/plugins/codex-security/mcp-app/tests/test_audit_acceptance_contract.mjs new file mode 100644 index 000000000..8d1e108d1 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_audit_acceptance_contract.mjs @@ -0,0 +1,122 @@ +import assert from "node:assert/strict"; +import { mkdtemp, mkdir, readFile, readdir, realpath, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { build } from "esbuild"; + +const bundle = await build({ + stdin: { + contents: `export * from "./src/artifact-scan-draft.ts"; + export * from "./src/deep-scan/artifact-validation.ts"; + export * from "./src/deep-scan/artifacts.ts"; + export * from "../../../sdk/typescript/src/accepted-audit.ts";`, + resolveDir: path.resolve(import.meta.dirname, ".."), + }, + bundle: true, format: "esm", platform: "node", write: false, + footer: { js: "//# sourceURL=audit-acceptance-contract.js" }, +}); +const { + createDeepScanArtifacts, recordCodexSecurityScanDraft, + recordCodexSecurityWorkerScanDraft, validateDiscoveryArtifacts, + readDiscoveryAuditDraft, auditEvidence, runAcceptedAudit, +} = await import(`data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}`); + +const scanId = "811aef98-3709-4c2d-8b7a-742977521865"; +const finding = { + ruleId: "path-traversal.archive-extraction", title: "Unsafe archive extraction", + summary: "An archive entry reaches a filesystem write.", + severity: { level: "high" }, confidence: { level: "high", rationale: "Source review." }, + taxonomy: { category: "path-traversal", cwe: ["CWE-22"] }, + locations: [{ path: "src/extract.py", startLine: 4 }], + remediation: "Validate the resolved output path before writing.", + provenance: { source: "local_plugin", candidateId: "archive-entry" }, +}; + +for (const completeness of ["complete", "partial", "unknown"]) { + test(`Standard and Deep retain accepted semantic evidence with ${completeness} coverage`, async () => { + const root = await realpath(await mkdtemp(path.join(tmpdir(), "audit-contract-"))); + try { + const repository = path.join(root, "repository"); + const scanDir = path.join(root, "scan"); + const artifacts = createDeepScanArtifacts(scanDir); + const workerRoot = path.join(artifacts.workersRoot, "discovery-0001", "output"); + await Promise.all([mkdir(repository), mkdir(workerRoot, { recursive: true })]); + const semantic = { + scanId, complete: true, + threatModel: { summary: "An untrusted caller supplies archive entries." }, + findings: [finding], + coverage: { + completeness, + surfaces: [{ id: "archive", label: "Archive extraction", disposition: "reported", receiptRefs: [] }], + explicitExclusions: [], + deferred: completeness === "complete" ? [] : [{ id: "deployment-controls", reason: "Deployment controls remain unverified." }], + }, + }; + const standard = { + root: scanDir, repoRoot: repository, layout: "scan", scanId, + mode: "standard", status: "running", scope: ".", + targetContract: { + target: { + allowedKinds: ["directory_snapshot"], targetId: "target_example", displayName: "example", + requiredSnapshotDigest: `codex-security-snapshot/v1:sha256:${"a".repeat(64)}`, + }, + scope: { requiredIncludePaths: ["."], requiredExcludePaths: [] }, diffTarget: null, + }, + }; + const worker = { root: workerRoot, repoRoot: repository, layout: "worker", scanId }; + const checkpoint = { ...semantic, complete: false }; + await recordCodexSecurityScanDraft(standard, checkpoint); + await recordCodexSecurityWorkerScanDraft(worker, checkpoint); + await assert.rejects(validateDiscoveryArtifacts(artifacts, path.join(workerRoot, "result.json"), scanId), /checkpoint/); + assert.equal(JSON.parse(await readFile(path.join(scanDir, "scan-manifest.json"))).scan.complete, false); + const controller = new AbortController(); + const execute = async () => ({ threadId: "audit-conversation", usage: null }); + const accept = async () => auditEvidence(await readDiscoveryAuditDraft( + artifacts, path.join(workerRoot, "result.json"), scanId, + )); + const unfinished = await runAcceptedAudit({ signal: controller.signal, execute, accept }); + assert.equal(unfinished.status, "checkpoint"); + assert.equal(unfinished.checkpoint.complete, false); + assert.equal(unfinished.accepted, undefined); + assert.equal(unfinished.execution.usage, null); + const standardWrite = await recordCodexSecurityScanDraft(standard, semantic); + const deepWrite = await recordCodexSecurityWorkerScanDraft(worker, semantic); + assert.equal(standardWrite.status, "draft_written"); + assert.equal(deepWrite.status, "draft_written"); + const accepted = await validateDiscoveryArtifacts(artifacts, path.join(workerRoot, "result.json"), scanId); + const manifest = JSON.parse(await readFile(path.join(scanDir, "scan-manifest.json"))); + const findings = JSON.parse(await readFile(path.join(scanDir, "findings.json"))); + const coverage = JSON.parse(await readFile(path.join(scanDir, "coverage.json"))); + assert.deepEqual(accepted.findings, [finding]); + for (const [key, value] of Object.entries(finding)) { + assert.deepEqual(findings.findings[0][key], value); + } + assert.ok(findings.findings[0].identity.anchor); + assert.deepEqual(accepted.threatModel, manifest.scan.threatModel); + for (const field of ["completeness", "surfaces", "explicitExclusions", "deferred"]) { + assert.deepEqual(coverage[field], accepted.coverage[field]); + } + assert.equal(accepted.scanId, scanId); + const audit = await runAcceptedAudit({ signal: controller.signal, execute, accept }); + assert.equal(audit.status, "accepted"); + assert.deepEqual(audit.accepted, accepted); + assert.deepEqual(audit.checkpoint, accepted); + const failure = new Error("Synthetic execution failure"); + await assert.rejects(runAcceptedAudit({ signal: controller.signal, + execute: async () => { throw failure; }, + accept: async () => { assert.fail("An execution failure cannot accept old output"); }, + }), (error) => error === failure); + await assert.rejects(runAcceptedAudit({ signal: controller.signal, execute, + accept: async () => { const evidence = await accept(); controller.abort("user canceled"); return evidence; }, + }), (error) => error === controller.signal.reason); + assert.equal(manifest.scan.sealedAt, undefined); + assert.equal(manifest.scan.artifacts, undefined); + assert.equal((await readdir(workerRoot)).includes("scan-manifest.json"), false); + assert.equal((await readdir(scanDir)).includes("report.md"), false); + await assert.rejects(validateDiscoveryArtifacts(artifacts, path.join(workerRoot, "result.json"), "b4c84677-5aaf-410c-88d2-3e97e6f8c4d8"), /scan/); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_compact_artifact_server.mjs b/plugins/codex-security/mcp-app/tests/test_compact_artifact_server.mjs index aa7937bd5..af2138c5b 100644 --- a/plugins/codex-security/mcp-app/tests/test_compact_artifact_server.mjs +++ b/plugins/codex-security/mcp-app/tests/test_compact_artifact_server.mjs @@ -1229,6 +1229,7 @@ async function testReducerWorkerToolList(bundle) { async function bundleEntrypoint(entrypoint, outfile) { await build({ bundle: true, + nodePaths: [fileURLToPath(new URL("../node_modules", import.meta.url))], define: { __dirname: JSON.stringify(applicationRoot), "import.meta.url": "__filename" diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.mjs index b80267ac4..ed3235dde 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.mjs +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.mjs @@ -217,8 +217,9 @@ async function testReducerValidation(root) { discoveries: [{ workerId: first.id, result: draft([firstFinding, secondFinding]) }], previous: null, }; - const validateSnapshot = () => validateReducerArtifacts({ + const validateSnapshot = (persistSourceCoverage = false) => validateReducerArtifacts({ artifacts, artifactDir, resultPath, reducerId: "dedup-0001", sources, + persistSourceCoverage, }, scanId); await assert.rejects(validateSnapshot(), /unaccounted source findings/); await writeResult(resultPath, draft([firstFinding, secondFinding])); @@ -226,11 +227,15 @@ async function testReducerValidation(root) { const validatedSnapshot = await validateSnapshot(); assert.equal(validatedSnapshot.newFindings, 2); const admitted = JSON.parse(await readFile(resultPath, "utf8")); + const { sourceCoverage, ...legacySnapshot } = validatedSnapshot.result; + assert.equal(sourceCoverage.completeness, "unknown"); assert.deepEqual( - validatedSnapshot.result, + legacySnapshot, admitted, - "validation returns the same reconciled result that was accepted on disk", + "v1 preserves host coverage in memory while retaining the legacy persisted shape", ); + const versionedSnapshot = await validateSnapshot(true); + assert.deepEqual(versionedSnapshot.result, JSON.parse(await readFile(resultPath, "utf8")), "v2 persists the full host projection"); assert.equal(Object.hasOwn(admitted, "coverage"), false); assert.deepEqual(admitted.findings[1].provenance.sourceFindingIds, ["worker-001:1"]); sources.previous = structuredClone(admitted); diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_checkpoint_coverage.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_checkpoint_coverage.mjs new file mode 100644 index 000000000..1df47d525 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_checkpoint_coverage.mjs @@ -0,0 +1,33 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { publishCoverageFixture } from "./deep_scan_coverage_fixture.mjs"; + +for (const continueAfterResume of [false, true]) { + test(`immutable discovery receipts survive resumed publication (continued: ${continueAfterResume})`, async () => { + const root = await mkdtemp(path.join(tmpdir(), "deep-checkpoint-coverage-")); + try { + const fixture = path.join(root, "fixture"); + await mkdir(fixture, { mode: 0o700 }); + const { scanDir } = await publishCoverageFixture(fixture, "partial", { + resume: true, continueAfterResume, immutableInputs: true, + }); + const coverage = JSON.parse(await readFile(path.join(scanDir, "coverage.json"), "utf8")); + assert.equal(coverage.completeness, "partial"); + assert.deepEqual(coverage.reviews.map((review) => review.completeness), ["partial", "complete", "unknown"]); + assert.equal(coverage.reviews[0].attempt, 2); + assert.equal(new Set(coverage.deferred.map((item) => item.candidateId)).size, 2); + const report = await readFile(path.join(scanDir, "report.md"), "utf8"); + for (const item of coverage.deferred) assert.ok(report.includes(item.reason)); + for (const surface of coverage.surfaces) { + for (const receipt of surface.receiptRefs) { + assert.equal(await readFile(path.join(scanDir, receipt), "utf8"), "Synthetic review evidence.\n"); + } + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_compatibility.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_compatibility.mjs new file mode 100644 index 000000000..d001192dd --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_compatibility.mjs @@ -0,0 +1,147 @@ +import assert from "node:assert/strict"; +import { build } from "esbuild"; + +const bundle = await build({ + bundle: true, + entryPoints: [new URL("../src/deep-scan/registry.ts", import.meta.url).pathname], + format: "esm", + loader: { ".md": "text" }, + platform: "node", + write: false +}); +const { startOrJoinDeepScanCoordinator, DeepScanRemoteCoordinator } = await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}` +); + +await testUnsupportedWorkflowDoesNotAcquireOwnership(); +await testUnsupportedSelectionDoesNotAcquireOwnership(); + +async function testUnsupportedSelectionDoesNotAcquireOwnership() { + for (const selection of [ + { workflowVersion: "deep-security-scan/v1", finalizationInput: { version: 1 } }, + { workflowVersion: "deep-security-scan/v2", finalizationInput: { version: 99 } } + ]) { + await assert.rejects(startOrJoinDeepScanCoordinator({ + begin: { run: { scanId: "fixture", schemaVersion: 1, ...selection }, shouldStart: false }, + registry: { + get: () => assert.fail("unsupported selection inspected a live coordinator"), + start: () => assert.fail("unsupported selection started a coordinator") + }, + options: { + threadId: "fixture-thread", + prepareExecutor: async () => assert.fail("unsupported selection resolved settings"), + store: { claimCoordinator: async () => assert.fail("unsupported selection acquired ownership") } + } + }), /finalization input version/); + } +} + +async function testUnsupportedWorkflowDoesNotAcquireOwnership() { + for (const version of [ + { schemaVersion: 99, workflowVersion: "deep-scan-mcp/v1" }, + { schemaVersion: 1, workflowVersion: "future/v99" } + ]) { + let mutations = 0; + await assert.rejects(startOrJoinDeepScanCoordinator({ + begin: { run: { scanId: "fixture", ...version }, shouldStart: false }, + registry: { + get: () => undefined, + start: () => { mutations += 1; } + }, + options: { + threadId: "fixture-thread", + store: { claimCoordinator: async () => { mutations += 1; } } + } + }), /unsupported workflow or schema version/); + assert.equal(mutations, 0); + } +} + + +// A joining client must not resolve or replace the live executor's settings. +for (const workflowVersion of ["deep-scan-mcp/v1", "deep-security-scan/v1", "deep-security-scan/v2"]) { + for (const local of [true, false]) { + let preparations = 0; + const run = { scanId: "fixture", status: "running", workflowVersion }; + const options = { + threadId: "fixture-thread", + executor: { marker: "observer" }, + prepareExecutor: async () => { preparations += 1; return {}; }, + store: { claimCoordinator: async () => ({ run, acquired: false }) } + }; + await startOrJoinDeepScanCoordinator({ + begin: { run, shouldStart: false }, + registry: { get: () => local ? {} : undefined, start: () => assert.fail("observer started") }, + options + }); + assert.equal(preparations, 0); + } +} + +// Selected publication has no worker launch and must not need current settings. +for (const selected of [false, true]) { + const run = { + scanId: "fixture", status: "running", workflowVersion: "deep-security-scan/v2", + ...(selected ? { finalizationInput: { version: 1 } } : {}) + }; + let preparations = 0; + const fallback = {}; + const restored = {}; + await startOrJoinDeepScanCoordinator({ + begin: { run, shouldStart: false }, + registry: { + get: () => undefined, + start: (options) => { + assert.equal(options.executor, selected ? fallback : restored); + assert.equal(options.run, run); + return {}; + } + }, + options: { + threadId: "fixture-thread", + executor: fallback, + prepareExecutor: async () => { preparations += 1; return restored; }, + store: { claimCoordinator: async () => ({ run, acquired: true }) } + } + }); + assert.equal(preparations, selected ? 0 : 1); +} + +const originalNow = Date.now; +try { + let now = 0; + Date.now = () => now; + const run = { scanId: "fixture", status: "running", updatedAt: "1970-01-01T00:00:00Z" }; + const acquired = { ...run, model: "original-model", coordinatorGeneration: 3 }; + let preparations = 0; + const executor = { marker: "restored" }; + const registry = { + get: () => undefined, + start: (options) => { + assert.equal(options.run, acquired); + assert.equal(options.executor, executor); + return { wait: async () => ({ ...acquired, status: "succeeded" }) }; + } + }; + const options = { + threadId: "fixture-thread", + executor: { marker: "observer" }, + prepareExecutor: async (state) => { + assert.equal(state, acquired); + preparations += 1; + return executor; + }, + store: { + get: async () => run, + claimCoordinator: async () => ({ run: acquired, acquired: true }) + } + }; + await startOrJoinDeepScanCoordinator({ begin: { run, shouldStart: true }, registry, options }); + assert.equal(preparations, 1); + const remote = new DeepScanRemoteCoordinator({ run, registry, options }); + now = 60_000; + assert.equal((await remote.wait(undefined, 1_000)).status, "succeeded"); + assert.equal(preparations, 2, "takeover resolves settings from the newly acquired run"); +} finally { + Date.now = originalNow; +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.mjs index 0625bbbb1..3625954d9 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.mjs +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.mjs @@ -1362,12 +1362,13 @@ async function testFinishPersistenceFailureRewritesManifestAsFailure() { await assertFailureManifest(terminal, "terminal"); } -async function testLostFinishResponseReplaysWithoutOverwritingSuccessManifest() { +async function testLostFinishResponseObservesCommitWithoutOverwritingSuccessManifest() { const fixture = await fixtureRun({ workers: 1, subagents: 0, stopAfterNoNew: 1, maxDiscoveryRuns: 1 }); const store = new FakeStore(fixture.run); store.loseFirstFinishResponseAfterCommit = true; const coordinator = new DeepScanCoordinator({ run: fixture.run, + threadId: "original-parent-thread", store, executor: new FakeExecutor({ dedupNewFindings: [0] }), pluginRoot: fixture.pluginRoot, @@ -1377,36 +1378,12 @@ async function testLostFinishResponseReplaysWithoutOverwritingSuccessManifest() const terminal = await coordinator.wait(undefined, 5_000); assert.equal(terminal?.status, "succeeded"); - assert.equal(store.finishCalls.length, 2); - assert.deepEqual(store.finishCalls[1], store.finishCalls[0]); + assert.equal(store.finishCalls.length, 1); assert.equal(store.failCalls, 0); const manifest = JSON.parse(await readFile(terminal.manifestPath, "utf8")); assert.equal(manifest.scan.scanId, fixture.run.scanId); } -async function testLostWorkerCommitResponsesReplayIdempotently() { - const fixture = await fixtureRun({ workers: 1, subagents: 0, stopAfterNoNew: 1, maxDiscoveryRuns: 1 }); - const store = new FakeStore(fixture.run); - store.loseFirstDiscoveryAcceptanceResponseAfterCommit = true; - store.loseFirstDedupCommitResponseAfterCommit = true; - const coordinator = new DeepScanCoordinator({ - run: fixture.run, - store, - executor: new FakeExecutor({ dedupNewFindings: [0] }), - pluginRoot: fixture.pluginRoot, - clock: immediateClock - }); - coordinator.start(); - - const terminal = await coordinator.wait(undefined, 5_000); - assert.equal(terminal?.status, "succeeded"); - assert.equal(store.discoveryAcceptanceResponseLosses, 1); - assert.equal(store.dedupCommitResponseLosses, 1); - assert.equal(store.dedupCommitCalls.length, 2); - assert.equal(store.dedupCommits.length, 1); - assert.equal(store.failCalls, 0); -} - async function testCommittedReducerIsReconciledBeforeDiscoveryFailureManifest() { const fixture = await fixtureRun({ workers: 3, subagents: 0, stopAfterNoNew: 10, maxDiscoveryRuns: 3 }); const thirdWorkerGate = deferred(); @@ -2910,6 +2887,8 @@ async function testPausedDiscoverySurvivesCoordinatorRestart() { store.heartbeatCoordinator = async () => structuredClone(store.run); const replacementExecutor = new FakeExecutor({ dedupNewFindings: [0] }); const acceptedResult = await readFile(accepted.resultManifestPath, "utf8"); + const acceptedWorkerId = await workerIdFromPrompt(accepted.promptPath); + await Promise.all(persistedWorkers.map((worker) => rm(worker.promptPath, { force: true }))); const resumed = await startOrJoinDeepScanCoordinator({ begin: { run: structuredClone(store.run), shouldStart: false }, registry: new DeepScanCoordinatorRegistry(), @@ -2926,11 +2905,11 @@ async function testPausedDiscoverySurvivesCoordinatorRestart() { assert.equal(continuationClaims.length, 1); assert.equal(continuationClaims[0].handoffClaimToken, handoffClaimToken); - assert.equal(terminal?.status, "succeeded"); + assert.equal(terminal?.status, "succeeded", terminal?.error); assert.equal(store.failCalls, 0); assert.equal(replacementExecutor.logicalDiscoveryWorkers.size, 1); assert.equal( - replacementExecutor.logicalDiscoveryWorkers.has(await workerIdFromPrompt(accepted.promptPath)), + replacementExecutor.logicalDiscoveryWorkers.has(acceptedWorkerId), false ); assert.equal(store.dedupClaims.length, 1); @@ -3995,8 +3974,7 @@ try { await testConfigurationFailureDoesNotRetry(); await testFailureManifestWriteDoesNotMaskOriginalError(); await testFinishPersistenceFailureRewritesManifestAsFailure(); - await testLostFinishResponseReplaysWithoutOverwritingSuccessManifest(); - await testLostWorkerCommitResponsesReplayIdempotently(); + await testLostFinishResponseObservesCommitWithoutOverwritingSuccessManifest(); await testCommittedReducerIsReconciledBeforeDiscoveryFailureManifest(); await testLongWorkerErrorIsBoundedOnlyAtPersistenceBoundary(); await testDiscoveryPhasePersistenceFailureStopsDispatch(); diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.mjs index 76112891d..170a6e9fc 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.mjs +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.mjs @@ -6,16 +6,18 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { build } from "esbuild"; +import { parse as parseToml } from "smol-toml"; const executorSource = new URL("../src/deep-scan/executor.ts", import.meta.url); const bundle = await build({ bundle: true, + nodePaths: [fileURLToPath(new URL("../node_modules", import.meta.url))], define: { "import.meta.url": JSON.stringify(executorSource.href) }, stdin: { // Test the environment snapshot without adding a production export. - contents: `${await readFile(executorSource, "utf8")}\nexport { snapshotWorkerEnvironment };`, + contents: `${await readFile(executorSource, "utf8")}\nexport { snapshotWorkerEnvironment };\nexport { captureDeepScanExecutionSettings, loadDeepScanExecutionSettings, restoredDeepScanWorkerSettings } from "./recovery-settings.js";\nexport { WorkbenchDeepScanStore } from "./store.js";`, loader: "ts", resolveDir: path.dirname(fileURLToPath(executorSource)), sourcefile: fileURLToPath(executorSource) @@ -24,7 +26,7 @@ const bundle = await build({ platform: "node", write: false }); -const { CodexSdkWorkerExecutor, resolveCodexPath, snapshotWorkerEnvironment } = await import( +const { WorkbenchDeepScanStore, CodexSdkWorkerExecutor, resolveCodexPath, snapshotWorkerEnvironment, captureDeepScanExecutionSettings, loadDeepScanExecutionSettings, restoredDeepScanWorkerSettings } = await import( `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}` ); const errorsBundle = await build({ @@ -74,6 +76,9 @@ const deniedWorkerPermissionProfile = { try { await testOpenAiCredentialsReachWorker(); await testWorkerReasoningSummaries(); + await testWorkerProviderSelection(); + await testIsolatedReconstructedWorkers(); + if (process.platform !== "win32") await testNullUsageCompletion(); if (process.platform !== "win32") { await testMissingParentSandboxFailsBeforeWorkerLaunch(); await testDisallowedWorkerProfileFailsBeforeWorkerLaunch(); @@ -83,6 +88,7 @@ try { await testSdkInvocationAndThreadCapture(); await testBedrockCredentialsReachWorker(); await testArtifactServerUsesExtendedStartupTimeout(); + await testReducerCoveragePersistenceBinding(); await testZeroSubagentsPreservesHostRestrictions(); await testSdkResumesExistingThread(); await testRetryNotificationDoesNotInterruptTurn(); @@ -737,6 +743,318 @@ async function testOpenAiCredentialsReachWorker() { } } +async function testIsolatedReconstructedWorkers() { + const launchFailures = []; + const previousMarker = process.env.FAKE_CODEX_MARKER; + const originalSpawn = childProcess.spawn; + const scans = []; + try { + for (const name of ["first", "second"]) { + const uncappedSandbox = { filesystemDenies: trustedParentSandboxWithDenials.filesystemDenies }; + const currentParentSandbox = name === "first" ? uncappedSandbox : trustedParentSandboxWithDenials; + const expectedProfile = structuredClone(deniedWorkerPermissionProfile); + delete expectedProfile.filesystem.glob_scan_max_depth; + const fixture = await fakeCodexFixture(expectedProfile); + const codexHome = path.join(fixture.root, "home"); + const configPath = path.join(fixture.root, "scan config.toml"); + const promptPath = path.join(fixture.root, "prompt.md"); + await mkdir(codexHome); + const config = { + model: `fixture-${name}-inherited`, + model_provider: name === "first" ? "openrouter" : "amazon-bedrock", + model_reasoning_effort: "medium", + model_reasoning_summary: "concise", + service_tier: name === "first" ? "default" : "fast" + }; + await writeFile(configPath, Object.entries(config).filter(([key]) => name !== "first" + || !["model_provider", "model_reasoning_summary", "service_tier"].includes(key)) + .map(([key, value]) => `${key} = ${JSON.stringify(value)}\n`).join("") + + (name === "second" ? '[model_providers.amazon-bedrock.aws]\nregion = "us-west-2"\nprofile = "fixture-profile"\n' : "")); + const providerKeys = name === "first" ? ["OPENROUTER_API_KEY"] : ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]; + await writeFile(promptPath, "CAPTURE_SYNTHETIC_OPENAI_AUTH CAPTURE_SYNTHETIC_PROVIDER_AUTH NULL_USAGE\n"); + const executable = path.join(fixture.root, process.platform === "win32" ? "node.exe" : "node"); + // Keep dynamically linked Node beside its libraries on Unix. Each scan + // still selects a distinct executable path at the spawn boundary. + if (process.platform === "win32") await copyFile(process.execPath, executable); + else await symlink(process.execPath, executable); + const codexOptions = { + codexPathOverride: executable, + baseUrl: `https://${name}.example.invalid/v1`, + env: { + PATH: path.dirname(process.execPath), + ...(process.env.SystemRoot ? { SystemRoot: process.env.SystemRoot } : {}), + CODEX_HOME: codexHome, + CODEX_SECURITY_CONFIG_PATH: configPath, + CODEX_API_KEY: `synthetic-${name}-credential`, + FAKE_CODEX_MARKER: fixture.markerPath, + FAKE_CODEX_PROVIDER_ENV_KEYS: JSON.stringify(providerKeys), + FAKE_CODEX_SCAN_VALUE: name + } + }; + const settings = { + codexOptions, + model: `fixture-${name}-override`, + reasoningEffort: "ultra", + usageOwner: { threadId: `fixture-${name}-owner`, turnId: "original-turn", startedAt: "2026-01-01T00:00:00Z" }, + parentSandbox: name === "first" ? trustedParentSandboxWithDenials : uncappedSandbox + }; + await mkdir(path.join(codexHome, "sessions")); + await writeFile(path.join(codexHome, "sessions", "owner.jsonl"), [ + { type: "session_meta", timestamp: "2026-01-01T00:00:00Z", + payload: { id: `fixture-${name}-owner`, model_provider: config.model_provider } }, + { type: "event_msg", timestamp: "2026-01-01T00:00:00Z", + payload: { type: "thread_settings_applied", thread_id: `fixture-${name}-owner`, thread_settings: { + model: "native-parent-model", model_provider_id: config.model_provider, + reasoning_effort: "medium", reasoning_summary: config.model_reasoning_summary + } } }, + { type: "turn_context", timestamp: "2026-01-01T00:00:01Z", + payload: { turn_id: "original-turn", model: "native-parent-model", effort: "medium", summary: "none" } }, + { type: "turn_context", timestamp: "2026-01-01T00:02:00Z", + payload: { turn_id: "later-turn", model: "later-parent-model", effort: "low", summary: "detailed" } } + ].map(JSON.stringify).join("\n") + "\n"); + await writeFile(path.join(codexHome, "sessions", "observer.jsonl"), JSON.stringify({ + type: "session_meta", timestamp: "2026-01-01T00:00:00Z", + payload: { id: `fixture-${name}-observer`, model_provider: "observer-provider" } + }) + "\n"); + const saved = await captureDeepScanExecutionSettings(settings, settings.parentSandbox, + { ...codexOptions.env, CODEX_CLI_PATH: executable }, + { threadId: `fixture-${name}-observer`, startedAt: "2026-01-01T00:01:00Z" }); + assert.equal(saved.nativeServiceTierAbsent, name === "first" ? true : undefined); + const targetPath = path.join(fixture.root, "target"); + await mkdir(targetPath); + const workbenchPath = fileURLToPath(new URL("../../scripts/workbench_db.py", import.meta.url)); + const runWorkbench = async (args, input, _selectFinalization, withExecutionSettings) => { + const pythonArgs = withExecutionSettings ? ["-c", + "import runpy, sys; script = sys.argv.pop(1); runpy.run_path(script)['main'](with_execution_settings=True)", + workbenchPath, ...args] : [workbenchPath, ...args]; + const result = spawnSync(process.env.PYTHON?.trim() || "python3", pythonArgs, { + env: { ...process.env, CODEX_HOME: codexHome, + CODEX_SECURITY_STATE_DIR: path.join(fixture.root, "state") }, + input, encoding: "utf8", timeout: 30_000 + }); + assert.equal(result.status, 0, result.stderr); + return JSON.parse(result.stdout); + }; + const store = new WorkbenchDeepScanStore(runWorkbench); + const beginInput = { targetPath, threadId: settings.usageOwner.threadId, + model: settings.model, reasoningEffort: settings.reasoningEffort, + scanRoot: path.join(fixture.root, "scans") }; + const { run } = await store.begin(beginInput); + const recordedScanDir = run.scanDir; + const snapshotPath = path.join(recordedScanDir, "artifacts", "deep_discovery", "execution-settings.json"); + await assert.rejects(readFile(snapshotPath), { code: "ENOENT" }); + assert.equal(run.workflowVersion, "deep-security-scan/v1"); + // Import the eventual writer's row and projection; this release must not create them. + const envelope = { version: 1, settings: saved }; + const imported = spawnSync(process.env.PYTHON?.trim() || "python3", ["-c", + "import json, sqlite3, sys; c=sqlite3.connect(sys.argv[1]); c.execute(\"UPDATE deep_scan_runs SET execution_settings_json = ? WHERE scan_id = ?\", (sys.argv[3], sys.argv[2])); c.commit()", + path.join(fixture.root, "state", "workbench.sqlite3"), run.scanId, JSON.stringify(envelope) + ], { encoding: "utf8" }); + assert.equal(imported.status, 0, imported.stderr); + await mkdir(path.dirname(snapshotPath), { recursive: true }); + await writeFile(snapshotPath, JSON.stringify(envelope) + "\n"); + const snapshot = await readFile(snapshotPath, "utf8"); + const expectedProvider = name === "first" ? undefined + : { "amazon-bedrock": { aws: { region: "us-west-2", profile: "fixture-profile" } } }; + assert.deepEqual(JSON.parse(snapshot).settings.providerConfig, expectedProvider, + "recorded provider selections need no persisted catalog definitions"); + const claim = await store.claimCoordinator({ scanId: run.scanId, threadId: beginInput.threadId }); + assert.equal(claim.acquired, true); + assert.deepEqual(await loadDeepScanExecutionSettings(recordedScanDir, claim.run), saved); + const observer = await new WorkbenchDeepScanStore(runWorkbench).begin({ + ...beginInput, model: "observer-model", reasoningEffort: "low" + }); + assert.equal(observer.shouldStart, false); + assert.equal(await readFile(snapshotPath, "utf8"), snapshot); + assert.equal(observer.run.model, settings.model); + assert.equal(snapshot.includes("synthetic-"), false); + const runtimeEnvironment = { ...codexOptions.env }; + const restored = restoredDeepScanWorkerSettings(saved, currentParentSandbox, () => runtimeEnvironment); + restored.codexOptions.baseUrl = codexOptions.baseUrl; + scans.push({ name, fixture, run, readRun: async () => (await new WorkbenchDeepScanStore(runWorkbench).claimCoordinator({ scanId: run.scanId, threadId: beginInput.threadId })).run, recordedScanDir, currentParentSandbox, config, configPath, promptPath, settings, runtimeEnvironment, snapshotPath, snapshot, providerKeys, expectedProvider, + executor: new CodexSdkWorkerExecutor(restored) }); + } + childProcess.spawn = (command, args, options) => { + const scan = scans.find((scan) => options?.env?.FAKE_CODEX_MARKER === scan.fixture.markerPath); + return originalSpawn(command, scan ? [scan.fixture.executablePath, ...args] : args, options); + }; + syncBuiltinESMExports(); + + for (const phase of ["fresh", "resume", "reconstructed-fresh", "reconstructed", "incomplete"]) { + if (phase.startsWith("reconstructed") || phase === "incomplete") { + for (const scan of scans) { + scan.run = await scan.readRun(); + // The caller restores recorded selections. Its old config file need + // not exist; current credentials still come from the selected home/env. + if (phase === "reconstructed-fresh") await rm(scan.configPath); + if (phase.startsWith("reconstructed")) { + // The managed parent can edit its output files. Neither a substituted + // executable/home nor other settings in that file are launch authority. + const rewritten = JSON.parse(scan.snapshot); + rewritten.settings.codexPath = process.execPath; + rewritten.settings.codexHome = scans.find((other) => other !== scan).settings.codexOptions.env.CODEX_HOME; + await writeFile(scan.snapshotPath, JSON.stringify(rewritten)); + } + if (phase === "incomplete") { + const saved = JSON.parse(scan.snapshot); + for (const key of ["model", "reasoningEffort", "reasoningSummary"]) delete saved.settings[key]; + // Native history restores the first provider. The second snapshot + // retains the provider binding for its saved AWS selectors. + if (scan.name === "first") delete saved.settings.modelProvider; + if (scan.name === "first") delete saved.settings.serviceTier; + await writeFile(scan.snapshotPath, JSON.stringify(saved)); + // Emulate an older trusted record with missing optional selections. + scan.run.executionSettings = saved; + } + const snapshotBeforeRead = await readFile(scan.snapshotPath, "utf8"); + const recorded = await loadDeepScanExecutionSettings(scan.recordedScanDir, { + ...scan.run, ...scan.settings, createdAt: "2026-01-01T00:01:00Z" + }); + const restored = restoredDeepScanWorkerSettings(recorded, scan.currentParentSandbox, () => scan.runtimeEnvironment); + restored.codexOptions.baseUrl = scan.settings.codexOptions.baseUrl; + scan.executor = new CodexSdkWorkerExecutor(restored); + assert.equal(await readFile(scan.snapshotPath, "utf8"), snapshotBeforeRead, + "restoring original worker selections must not rewrite saved settings"); + } + } + for (const scan of scans) { + scan.runtimeEnvironment.CODEX_API_KEY = `synthetic-${scan.name}-${phase}`; + for (const key of scan.providerKeys) scan.runtimeEnvironment[key] = `synthetic-${scan.name}-${phase}-${key}`; + scan.runtimeEnvironment.FAKE_CODEX_SCAN_VALUE = `${scan.name}-${phase}`; + scan.runtimeEnvironment.CODEX_HOME = path.join(scan.fixture.root, "observer-home"); + scan.runtimeEnvironment.CODEX_CLI_PATH = path.join(scan.fixture.root, "observer-codex"); + } + for (const kind of ["discovery", "dedup"]) { + const launches = await Promise.allSettled(scans.map(async (scan) => { + const resumeThreadId = ["fresh", "reconstructed-fresh"].includes(phase) ? undefined : `fixture-${scan.name}-resumed`; + const result = await scan.executor.run({ + kind, promptPath: scan.promptPath, workingDirectory: scan.fixture.root, + subagents: scan.name === "first" ? 0 : 2, + resumeThreadId, continuationPrompt: "CAPTURE_SYNTHETIC_OPENAI_AUTH CAPTURE_SYNTHETIC_PROVIDER_AUTH NULL_USAGE continuation", + signal: new AbortController().signal + }); + assert.equal(result.threadId, resumeThreadId ?? "fixture-thread-id"); + const child = JSON.parse(await readFile(scan.fixture.markerPath, "utf8")); + const preflight = JSON.parse(await readFile(scan.fixture.preflightMarkerPath, "utf8")); + assert.equal(await realpath(child.executable), await realpath(scan.settings.codexOptions.codexPathOverride)); + assert.equal(child.codexCliPath, scan.settings.codexOptions.codexPathOverride); + assert.equal(child.codexHome, scan.settings.codexOptions.env.CODEX_HOME); + assert.equal(preflight.codexHome, child.codexHome); + assert.equal(child.scanValue, `${scan.name}-${phase}`); + assert.equal(child.configPath, scan.configPath); + assert.deepEqual(child.openaiAuthentication, { CODEX_API_KEY: `synthetic-${scan.name}-${phase}` }); + assert.deepEqual(child.providerAuthentication, Object.fromEntries(scan.providerKeys + .map((key) => [key, `synthetic-${scan.name}-${phase}-${key}`]))); + assertFlagPair(child.argv, "--model", scan.settings.model); + for (const key of ["model_provider", "model_reasoning_summary", "service_tier"]) { + const override = `${key}=${JSON.stringify(scan.config[key])}`; + assert.equal(child.argv.includes(override), true, override); + assert.equal(preflight.argv.includes(override), true, override); + } + assert.equal(child.argv.includes('model_reasoning_effort="ultra"'), true); + assert.equal(preflight.argv.includes('model_reasoning_effort="ultra"'), true); + assert.equal(preflight.argv.includes(`model=${JSON.stringify(scan.settings.model)}`), true); + const baseUrl = `openai_base_url=${JSON.stringify(scan.settings.codexOptions.baseUrl)}`; + assert.equal(child.argv.includes(baseUrl), true); + assert.equal(preflight.argv.includes(baseUrl), true); + for (const launch of [child, preflight]) { + const provider = launch.argv.filter((argument) => /^model_providers[.=]/u.test(argument)); + assert.ok(provider.length > 0, "both preflight and worker launch receive provider configuration"); + const providers = parseToml(provider.join("\n")).model_providers; + if (scan.expectedProvider) assert.deepEqual(providers, scan.expectedProvider); + else assert.deepEqual(Object.keys(providers.openrouter).sort(), ["base_url", "env_key", "name", "wire_api"]); + assert.equal(workerPermissionProfileOverride(launch.argv).includes("glob_scan_max_depth"), false, + "a resumed bounded cap must not truncate an original uncapped deny glob, in either order"); + } + assertReadOnlyWorkerPolicy(child.argv); + assertWorkerSubagentPolicy(child.argv, scan.name === "first" ? 0 : 2); + assert.equal(workerPermissionProfileOverride(child.argv).includes('"/repo/.env"="deny"'), true); + assert.equal(child.argv.includes("resume"), resumeThreadId !== undefined); + assert.equal(child.stdin.includes("continuation"), resumeThreadId !== undefined); + })); + for (const [index, launch] of launches.entries()) { + if (launch.status === "rejected") { + launchFailures.push(`${scans[index].name}/${phase}/${kind}: ${launch.reason.message}`); + } + } + } + if (phase === "fresh") { + for (const scan of scans) { + await writeFile(scan.configPath, 'model_provider = "changed-provider"\nmodel_reasoning_summary = "detailed"\n'); + } + } + } + assert.deepEqual(launchFailures, [], "every actual preflight and worker must retain the original launch selection"); + } finally { + childProcess.spawn = originalSpawn; + syncBuiltinESMExports(); + restoreEnv("FAKE_CODEX_MARKER", previousMarker); + } +} + +async function testWorkerProviderSelection() { + const fixture = await fakeCodexFixture(); + const saved = Object.fromEntries( + ["CODEX_CLI_PATH", "CODEX_SECURITY_CONFIG_PATH", "OPENAI_API_KEY", "CODEX_API_KEY"].map((name) => [name, process.env[name]]) + ); + const originalSpawn = childProcess.spawn; + try { + delete process.env.OPENAI_API_KEY; + delete process.env.CODEX_API_KEY; + const configPath = path.join(fixture.root, "scan config.toml"); + const promptPath = path.join(fixture.root, "prompt.md"); + await writeFile(configPath, 'model_provider = "fixture-provider"\n'); + await writeFile(promptPath, "fixture provider selection"); + process.env.CODEX_CLI_PATH = process.execPath; + process.env.CODEX_SECURITY_CONFIG_PATH = configPath; + childProcess.spawn = (command, args, options) => originalSpawn( + command, + command === process.execPath || command === path.toNamespacedPath(process.execPath) + ? [fixture.executablePath, ...args] + : args, + options + ); + syncBuiltinESMExports(); + const executor = new CodexSdkWorkerExecutor({ parentSandbox: trustedParentSandbox }); + for (const kind of ["discovery", "dedup"]) { + await executor.run({ + kind, promptPath, workingDirectory: fixture.root, subagents: 0, + signal: new AbortController().signal + }); + const invocation = JSON.parse(await readFile(fixture.markerPath, "utf8")); + assert.equal(invocation.argv.includes('model_provider="fixture-provider"'), true); + } + } finally { + childProcess.spawn = originalSpawn; + syncBuiltinESMExports(); + for (const [name, value] of Object.entries(saved)) restoreEnv(name, value); + } +} + +async function testNullUsageCompletion() { + const fixture = await fakeCodexFixture(); + const previousPath = process.env.CODEX_CLI_PATH; + process.env.CODEX_CLI_PATH = fixture.executablePath; + try { + const promptPath = path.join(fixture.root, "prompt.md"); + await writeFile(promptPath, "NULL_USAGE\n"); + for (const kind of ["discovery", "dedup"]) { + for (const resumeThreadId of [undefined, "fixture-resumed-thread"]) { + const result = await new CodexSdkWorkerExecutor({ + parentSandbox: trustedParentSandbox + }).run({ + kind, promptPath, workingDirectory: fixture.root, subagents: 0, + resumeThreadId, signal: new AbortController().signal + }); + assert.equal(result.threadId, resumeThreadId ?? "fixture-thread-id"); + assert.equal(result.finalResponse, "fixture final response"); + } + } + } finally { + restoreEnv("CODEX_CLI_PATH", previousPath); + } +} + async function testWorkerReasoningSummaries() { const cases = [ ["", undefined], @@ -922,6 +1240,56 @@ async function testArtifactServerUsesExtendedStartupTimeout() { } } +async function testReducerCoveragePersistenceBinding() { + const fixture = await fakeCodexFixture(); + const previousPath = process.env.CODEX_CLI_PATH; + const previousMarker = process.env.FAKE_CODEX_MARKER; + process.env.CODEX_CLI_PATH = fixture.executablePath; + try { + const promptPath = path.join(fixture.root, "prompt.md"); + const workingDirectory = path.join(fixture.root, "artifacts"); + await mkdir(workingDirectory); + await writeFile(promptPath, "fixture reducer prompt\n"); + const launches = []; + for (const resume of [false, true]) { + for (const persistSourceCoverage of [false, true]) { + const markerPath = path.join(fixture.root, `coverage-${resume}-${persistSourceCoverage}.json`); + process.env.FAKE_CODEX_MARKER = markerPath; + const scanRoot = path.join(fixture.root, `scan-${resume}-${persistSourceCoverage}`); + const deepReducer = { + scanRoot, + claimedWorkers: [{ + id: "worker-1", attempt: 2, + resultPath: path.join(scanRoot, "worker", "checkpoints", "accepted.json"), + artifactDir: path.join(scanRoot, "worker"), + }], + persistSourceCoverage, + }; + const launch = new CodexSdkWorkerExecutor({ + parentSandbox: trustedParentSandbox, + artifactContext: { pluginRoot: fixture.root, scanRoot: deepReducer.scanRoot, repoRoot: fixture.root, scanId: "fixture-scan-id" }, + }).run({ + kind: "dedup", promptPath, workingDirectory, subagents: 0, + signal: new AbortController().signal, + ...(resume ? { resumeThreadId: "fixture-existing-thread", continuationPrompt: "continue the reducer\n" } : {}), + artifactContext: { root: workingDirectory, layout: "reducer", deepReducer }, + }); + launches.push(launch.then(async () => { + const invocation = JSON.parse(await readFile(markerPath, "utf8")); + const prefix = "mcp_servers.cs_artifacts.env.CODEX_SECURITY_REDUCER_CONTEXT_JSON="; + const encoded = invocation.argv.find((arg) => arg.startsWith(prefix)); + assert.ok(encoded, "the launched reducer receives its host-bound artifact context"); + assert.deepEqual(JSON.parse(JSON.parse(encoded.slice(prefix.length))), deepReducer); + })); + } + } + await Promise.all(launches); + } finally { + restoreEnv("CODEX_CLI_PATH", previousPath); + restoreEnv("FAKE_CODEX_MARKER", previousMarker); + } +} + async function testSdkResumesExistingThread() { const fixture = await fakeCodexFixture(); const previousPath = process.env.CODEX_CLI_PATH; @@ -1400,22 +1768,27 @@ async function testDisallowedWorkerProfileFailsBeforeWorkerLaunch() { await mkdir(workingDirectory); await writeFile(promptPath, "fixture blocked worker prompt\n"); - await assert.rejects( - new CodexSdkWorkerExecutor({ - parentSandbox: trustedParentSandbox - }).run({ - kind: "discovery", - promptPath, - workingDirectory, - subagents: 0, - signal: new AbortController().signal - }), - (error) => error?.name === "DeepScanNonRetryableError" - && error.message.includes("codex_security_deep_scan_worker") - && error.message.includes("[allowed_permission_profiles]") - && error.message.includes("codex_security_deep_scan_worker = true") - && error.message.includes("Deep Scan did not run.") - ); + for (const kind of ["discovery", "dedup"]) { + for (const resumeThreadId of [undefined, "fixture-resumed-worker"]) { + await assert.rejects( + new CodexSdkWorkerExecutor({ + parentSandbox: trustedParentSandbox + }).run({ + kind, + resumeThreadId, + promptPath, + workingDirectory, + subagents: 0, + signal: new AbortController().signal + }), + (error) => error?.name === "DeepScanNonRetryableError" + && error.message.includes("codex_security_deep_scan_worker") + && error.message.includes("[allowed_permission_profiles]") + && error.message.includes("codex_security_deep_scan_worker = true") + && error.message.includes("Deep Scan did not run.") + ); + } + } await assert.rejects( readFile(fixture.markerPath, "utf8"), (error) => error?.code === "ENOENT" @@ -1480,7 +1853,7 @@ async function fakeCodexFixture( `const accountResult = ${JSON.stringify(accountResult)};`, `const preflightMarkerPath = ${JSON.stringify(preflightMarkerPath)};`, "if (process.argv.includes('app-server')) {", - " const preflight = { cwd: process.cwd(), codexHome: process.env.CODEX_HOME, requests: [] };", + " const preflight = { argv: process.argv.slice(2), cwd: process.cwd(), codexHome: process.env.CODEX_HOME, requests: [] };", " writeFileSync(preflightMarkerPath, JSON.stringify(preflight));", " let buffer = '';", " process.stdin.setEncoding('utf8');", @@ -1522,7 +1895,8 @@ async function fakeCodexFixture( "for await (const chunk of process.stdin) stdin += chunk;", "const openaiAuthentication = stdin.includes('CAPTURE_SYNTHETIC_OPENAI_AUTH') ? { OPENAI_API_KEY: process.env.OPENAI_API_KEY, CODEX_API_KEY: process.env.CODEX_API_KEY } : undefined;", "const bedrockAuthentication = stdin.includes('CAPTURE_SYNTHETIC_BEDROCK_AUTH') ? Object.fromEntries(JSON.parse(process.env.FAKE_CODEX_BEDROCK_ENV_KEYS).map((name) => [name, process.env[name]])) : undefined;", - "writeFileSync(process.env.FAKE_CODEX_MARKER, JSON.stringify({ argv: process.argv.slice(2), stdin, cwd: process.cwd(), codexHome: process.env.CODEX_HOME, configPath: process.env.CODEX_SECURITY_CONFIG_PATH, deepConfigPath: process.env.CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH, originator: process.env.CODEX_INTERNAL_ORIGINATOR_OVERRIDE, ...(stdin.includes('COMPLETE_THEN_HANG') ? { pid: process.pid } : {}), ...(openaiAuthentication ? { openaiAuthentication } : {}), ...(bedrockAuthentication ? { bedrockAuthentication } : {}) }));", + "const providerAuthentication = stdin.includes('CAPTURE_SYNTHETIC_PROVIDER_AUTH') ? Object.fromEntries(JSON.parse(process.env.FAKE_CODEX_PROVIDER_ENV_KEYS).map((name) => [name, process.env[name]])) : undefined;", + "writeFileSync(process.env.FAKE_CODEX_MARKER, JSON.stringify({ executable: process.execPath, argv: process.argv.slice(2), stdin, cwd: process.cwd(), codexHome: process.env.CODEX_HOME, codexCliPath: process.env.CODEX_CLI_PATH, configPath: process.env.CODEX_SECURITY_CONFIG_PATH, scanValue: process.env.FAKE_CODEX_SCAN_VALUE, deepConfigPath: process.env.CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH, originator: process.env.CODEX_INTERNAL_ORIGINATOR_OVERRIDE, ...(stdin.includes('COMPLETE_THEN_HANG') ? { pid: process.pid } : {}), ...(openaiAuthentication ? { openaiAuthentication } : {}), ...(bedrockAuthentication ? { bedrockAuthentication } : {}), ...(providerAuthentication ? { providerAuthentication } : {}) }));", "if (stdin.includes('COMPLETE_THEN_HANG')) process.on('SIGTERM', () => setTimeout(() => process.exit(0), 100));", "if (stdin.includes('THREAD_START_CONFIG_ERROR')) { console.error('Error: thread/start: thread/start failed: agents.max_threads cannot be set when features.multi_agent_v2 is enabled (code -32600)'); process.exit(1); }", "if (stdin.includes('CONFIG_ERROR')) { console.error('failed to load configuration: invalid value'); process.exit(2); }", @@ -1562,7 +1936,7 @@ async function fakeCodexFixture( " console.log(JSON.stringify({ type: 'item.completed', item }));", "}", "console.log(JSON.stringify({ type: 'item.completed', item: { id: 'message-1', type: 'agent_message', text: 'fixture final response' } }));", - "console.log(JSON.stringify({ type: 'turn.completed', usage: { input_tokens: 1, cached_input_tokens: 0, output_tokens: 1 } }));", + "console.log(JSON.stringify({ type: 'turn.completed', usage: stdin.includes('NULL_USAGE') ? null : { input_tokens: 1, cached_input_tokens: 0, output_tokens: 1 } }));", "if (stdin.includes('COMPLETE_THEN_HANG')) { setInterval(() => {}, 1_000); await new Promise(() => {}); }", "}", "" diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_finalization.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_finalization.mjs new file mode 100644 index 000000000..2a96eb5ea --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_finalization.mjs @@ -0,0 +1,151 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, readdir, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { build } from "esbuild"; + +const bundle = await build({ + stdin: { + contents: `export * from "./src/deep-scan/finalization.ts"; + export * from "./src/deep-scan/artifacts.ts"; + export * from "./src/artifact-scan-draft.ts";`, + resolveDir: path.resolve(import.meta.dirname, ".."), + }, + bundle: true, format: "esm", platform: "node", write: false, + footer: { js: "//# sourceURL=deep-scan-finalization-contract.js" }, +}); +const { publishSelectedDeepScan, readSelectedDeepScanDraft, createDeepScanArtifacts, saveScanDraftCheckpoint } = await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}`, +); +const scanId = "4e7b4acb-ac80-4d68-98cd-3d5ac5581cd1"; + +for (const terminalReason of ["capped", "saturated"]) { + test(`replays the selected ${terminalReason} input after replaceable results change`, async () => { + const scanDir = await realpath(await mkdtemp(path.join(tmpdir(), "selected-finalization-"))); + try { + const artifacts = createDeepScanArtifacts(scanDir); + const root = path.join(artifacts.dedupRoot, "dedup-0001", "output"); + await mkdir(root, { recursive: true }); + const draft = { + scanId, complete: true, findings: [], + coverage: { + completeness: "partial", surfaces: [], explicitExclusions: [], + deferred: [{ id: "review", reason: "A dependency remains unreviewed." }], + }, + }; + const { coverage, ...reduction } = draft; + await saveScanDraftCheckpoint({ root, repoRoot: scanDir, layout: "reducer" }, { + ...reduction, sourceCoverage: coverage, + }); + const [name] = await readdir(path.join(root, "checkpoints")); + const checkpoint = path.join(root, "checkpoints", name); + const contents = await readFile(checkpoint); + const selection = { + version: 1, + resultPath: path.relative(scanDir, checkpoint), + resultSha256: createHash("sha256").update(contents).digest("hex"), + terminalReason, omittedWorkerIds: [], selectedAt: "2026-01-01T00:00:00Z", + }; + // A replacement result is not a new finalization selection. + await writeFile(path.join(root, "result.json"), JSON.stringify({ ...draft, complete: false })); + assert.deepEqual(await readSelectedDeepScanDraft(artifacts, scanId, selection), draft); + assert.deepEqual(await readSelectedDeepScanDraft(artifacts, scanId, selection), draft); + await assert.rejects( + readSelectedDeepScanDraft(artifacts, "e14e9229-653a-4385-bec0-8745f0b037cb", selection), + /complete result for this scan/, + ); + await writeFile(checkpoint, JSON.stringify({ ...draft, findings: [] })); + await assert.rejects(readSelectedDeepScanDraft(artifacts, scanId, selection), /changed after acceptance/); + } finally { + await rm(scanDir, { recursive: true, force: true }); + } + }); +} + +test("recreates only partial coverage for a persisted zero-success deadline selection", async () => { + const selection = { + version: 1, resultPath: null, resultSha256: null, terminalReason: "capped", + omittedWorkerIds: [], selectedAt: "2026-01-01T00:00:00Z", + }; + const result = await readSelectedDeepScanDraft(createDeepScanArtifacts("unused"), scanId, selection); + assert.equal(result.scanId, scanId); + assert.deepEqual(result.findings, []); + assert.equal(result.coverage.completeness, "partial"); + assert.equal(result.coverage.deferred.length, 1); + await assert.rejects( + readSelectedDeepScanDraft(createDeepScanArtifacts("unused"), scanId, { ...selection, terminalReason: "saturated" }), + /recorded discovery deadline/, + ); +}); + +for (const status of ["failed", "canceled", "interrupted"]) { + test(`saved selection does not turn a ${status} scan into success`, async () => { + await assert.rejects(publishSelectedDeepScan({ + run: { scanId, scanDir: "unused", workflowVersion: "deep-security-scan/v2", status, finalizationInput: { + version: 1, resultPath: null, resultSha256: null, terminalReason: "capped", + omittedWorkerIds: [], selectedAt: "2026-01-01T00:00:00Z", + } }, + artifacts: createDeepScanArtifacts("unused"), signal: new AbortController().signal, + publish: async () => assert.fail("Stopped work cannot publish successful results"), + finish: async () => assert.fail("Stopped work cannot finish successfully"), + }), /Stopped Deep Scan/); + }); +} + +test("cancellation prevents selected publication and preserves its input", async () => { + const controller = new AbortController(); + controller.abort("cost limit or user cancellation"); + const selection = { version: 1, resultPath: null, resultSha256: null, terminalReason: "capped", + omittedWorkerIds: [], selectedAt: "2026-01-01T00:00:00Z" }; + await assert.rejects(publishSelectedDeepScan({ + run: { scanId, scanDir: "unused", workflowVersion: "deep-security-scan/v2", status: "running", finalizationInput: selection }, + artifacts: createDeepScanArtifacts("unused"), signal: controller.signal, + publish: async () => assert.fail("Canceled work cannot publish"), + finish: async () => assert.fail("Canceled work cannot finish"), + }), (error) => error === controller.signal.reason); + assert.equal(selection.terminalReason, "capped"); +}); + +for (const parentStatus of ["running", "complete", "invalid-seal"]) { + test(`recovery verifies the ${parentStatus} parent after a succeeded child`, async () => { + const root = await realpath(await mkdtemp(path.join(tmpdir(), "selected-parent-"))); + const { resumeSelectedDeepScan } = await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}`, + ); + const resultPath = "selected.json"; + await writeFile(path.join(root, resultPath), "{}\n"); + const calls = []; + const runWorkbench = async (args) => { + const [command] = args; + calls.push(command); + if (command === "get-deep-scan") return { deepScan: { + scanId, scanDir: root, targetPath: root, scope: ".", + workflowVersion: "deep-security-scan/v2", status: "succeeded", phase: "terminal", + coordinatorGeneration: 2, dispatchedCount: 2, noNewStreak: 0, + config: { workers: 1, subagents: 0, stopAfterNoNew: 1, maxDiscoveryRuns: 2 }, + finalizationInput: { version: 1, resultPath, resultSha256: "0".repeat(64), + terminalReason: "capped", omittedWorkerIds: [], selectedAt: "2026-01-01T00:00:00Z" }, + } }; + if (command === "get-scan") return { scan: { + scanId, scanDir: root, targetPath: root, + progress: { status: parentStatus === "running" ? "running" : "complete" }, + } }; + assert.deepEqual(args, ["prepare-scan-completion", "--scan-id", scanId, "--claim-token", "current-claim"]); + if (parentStatus === "running") throw new Error("Selected input changed after acceptance"); + if (parentStatus === "invalid-seal") throw new Error("Recorded seal does not match"); + return {}; + }; + try { + const recover = () => resumeSelectedDeepScan({ scanId, threadId: "original-parent", + pluginRoot: root, runWorkbench, handoffClaimToken: "current-claim", signal: new AbortController().signal }); + if (parentStatus === "complete") await recover(); + else await assert.rejects(recover(), parentStatus === "running" ? /changed after acceptance/ : /Recorded seal/); + assert.equal(calls.includes("prepare-scan-completion"), true); + assert.equal(await readFile(path.join(root, resultPath), "utf8"), "{}\n"); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_material_coverage.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_material_coverage.mjs new file mode 100644 index 000000000..71ea1af4f --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_material_coverage.mjs @@ -0,0 +1,39 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { materialRemediations, materialRemediationTests, publishCoverageFixture } from "./deep_scan_coverage_fixture.mjs"; + +for (const [resume, continueAfterResume] of [[false, false], [true, false], [true, true]]) { + test(`material fixes and unresolved coverage survive canonical publication (resume: ${resume}, continued: ${continueAfterResume})`, async () => { + const root = await mkdtemp(path.join(tmpdir(), "deep-material-coverage-")); + try { + const fixture = path.join(root, "fixture"); + await mkdir(fixture, { mode: 0o700 }); + const { scanDir } = await publishCoverageFixture(fixture, "partial", { + resume, continueAfterResume, immutableInputs: true, materialFindings: true, + }); + const { findings } = JSON.parse(await readFile(path.join(scanDir, "findings.json"), "utf8")); + assert.equal(findings.length, 1); + const sources = findings[0].provenance.sourceFindings; + assert.deepEqual(sources.map((source) => source.finding.remediation), materialRemediations); + assert.equal(new Set(sources.map((source) => source.id)).size, 2); + const report = await readFile(path.join(scanDir, "report.md"), "utf8"); + for (const text of [...materialRemediations, ...materialRemediationTests]) { + assert.equal(report.split(text).length - 1, 1, `canonical report retains ${text}`); + } + const coverage = JSON.parse(await readFile(path.join(scanDir, "coverage.json"), "utf8")); + assert.equal(coverage.completeness, "partial"); + assert.deepEqual(coverage.reviews.map((review) => review.completeness), ["partial", "complete", "unknown"]); + assert.equal(coverage.deferred.length, 2, "a finding does not discharge independent unresolved work"); + assert.equal(new Set(coverage.deferred.map((item) => item.candidateId)).size, 2); + for (const item of coverage.deferred) assert.ok(report.includes(item.reason)); + for (const surface of coverage.surfaces) { + assert.equal(await readFile(path.join(scanDir, surface.receiptRefs[0]), "utf8"), "Synthetic review evidence.\n"); + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_recovery_settings.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_recovery_settings.mjs new file mode 100644 index 000000000..72d0220b7 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_recovery_settings.mjs @@ -0,0 +1,385 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { syncBuiltinESMExports } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { PassThrough } from "node:stream"; +import { fileURLToPath } from "node:url"; +import { build } from "esbuild"; + +const bundle = await build({ + bundle: true, + nodePaths: [fileURLToPath(new URL("../node_modules", import.meta.url))], + define: { "import.meta.url": JSON.stringify(new URL("../src/deep-scan/recovery-settings.ts", import.meta.url).href) }, + entryPoints: [new URL("../src/deep-scan/recovery-settings.ts", import.meta.url).pathname], + format: "esm", + platform: "node", + write: false +}); +const { captureDeepScanExecutionSettings: captureSettings, restoredDeepScanWorkerSettings: restoreSettings, loadDeepScanExecutionSettings: loadRecordedSettings } = await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}` +); +const snapshots = new Map(); +const loadSettings = (directory, original, ...rest) => loadRecordedSettings(directory, { + ...original, executionSettings: snapshots.get(directory) +}, ...rest); +const root = await mkdtemp(join(tmpdir(), "deep-settings-")); +try { + const settings = { + codexPath: "/fixture/runtime/codex", + codexHome: "/fixture/account", + model: "fixture-model", + modelProvider: "fixture-provider", + reasoningEffort: "high", + reasoningSummary: "detailed", + serviceTier: "fast" + }; + const globSandbox = (depth) => ({ + filesystemDenies: ["/fixture/**/*.secret"], + ...(depth === undefined ? {} : { globScanMaxDepth: depth }) + }); + for (const [originalDepth, currentDepth, expectedDepth] of [ + [2, 5, 5], [5, 2, 5], [undefined, 2, undefined], [2, undefined, undefined] + ]) { + const restored = restoreSettings({ ...settings, parentSandbox: globSandbox(originalDepth) }, + globSandbox(currentDepth)); + assert.equal(restored.parentSandbox.globScanMaxDepth, expectedDepth, + `deny expansion must preserve both policies: ${originalDepth}, ${currentDepth}`); + } + assert.equal(restoreSettings(settings, globSandbox(2)).parentSandbox.globScanMaxDepth, 2, + "unavailable historical policy does not establish uncapped glob expansion"); + assert.equal(restoreSettings({ ...settings, parentSandbox: globSandbox(2) }, { + filesystemDenies: ["/fixture/exact-denial"] + }).parentSandbox.globScanMaxDepth, 2, "exact denials do not change glob expansion"); + const writeSnapshot = async (directory, value) => { + const path = join(directory, "artifacts", "deep_discovery", "execution-settings.json"); + await mkdir(join(directory, "artifacts", "deep_discovery"), { recursive: true }); + const snapshot = { version: 1, settings: structuredClone(value) }; + snapshots.set(directory, snapshot); + await writeFile(path, JSON.stringify(snapshot, null, 2) + "\n"); + }; + await assert.rejects(loadSettings(join(root, "missing")), /no recorded original execution settings/); + await writeSnapshot(join(root, "one"), settings); + await writeSnapshot(join(root, "two"), { ...settings, model: "other-model" }); + const savedPath = join(root, "one", "artifacts", "deep_discovery", "execution-settings.json"); + const saved = await readFile(savedPath, "utf8"); + const [recovered, concurrent] = await Promise.all([ + loadSettings(join(root, "one")), loadSettings(join(root, "two")) + ]); + assert.deepEqual(recovered, settings); + assert.equal(concurrent.model, "other-model"); + assert.equal(await readFile(savedPath, "utf8"), saved); + recovered.model = "caller-mutation"; + assert.deepEqual(await loadSettings(join(root, "one")), settings); + const configPath = join(root, "runtime.toml"); + await writeFile(configPath, `model = "inherited-model" +model_provider = "custom" +profile = "scan" +[profiles.scan] +model_reasoning_summary = "concise" +service_tier = "flex" +[model_providers.custom] +name = "Fixture" +http_headers = { Authorization = "synthetic-secret" } +`); + const captured = await captureSettings({ model: "original-model", reasoningEffort: "ultra" }, { + filesystemDenies: ["/fixture/original-deny"], globScanMaxDepth: 3 + }, { CODEX_CLI_PATH: process.execPath, CODEX_HOME: root, CODEX_SECURITY_CONFIG_PATH: configPath }); + assert.equal(captured.model, "original-model"); + assert.equal(captured.modelProvider, "custom"); + assert.equal(captured.reasoningSummary, "concise"); + assert.equal(captured.serviceTier, "flex"); + assert.equal(captured.providerConfig, undefined); + assert.equal(JSON.stringify(captured).includes("synthetic-secret"), false); + for (const modelProvider of ["openrouter", "fireworks", "amazon-bedrock"]) { + await writeFile(configPath, `model_provider = ${JSON.stringify(modelProvider)} +[model_providers.${modelProvider}.aws] +region = "us-west-2" +profile = "fixture-profile" +access_key_id = "synthetic-secret" +`); + const selected = await captureSettings({}, { filesystemDenies: [] }, { + CODEX_CLI_PATH: process.execPath, CODEX_HOME: root, CODEX_SECURITY_CONFIG_PATH: configPath + }); + assert.equal(selected.modelProvider, modelProvider); + const expectedProvider = modelProvider === "amazon-bedrock" + ? { "amazon-bedrock": { aws: { region: "us-west-2", profile: "fixture-profile" } } } : undefined; + assert.deepEqual(selected.providerConfig, expectedProvider, + "saved selections exclude catalog definitions and retain Bedrock selectors"); + const providerDir = join(root, modelProvider); + await writeSnapshot(providerDir, selected); + const path = join(providerDir, "artifacts", "deep_discovery", "execution-settings.json"); + const bytes = await readFile(path, "utf8"); + assert.equal(bytes.includes("synthetic-secret"), false); + const restoredProvider = restoreSettings(await loadSettings(providerDir), { filesystemDenies: [] }) + .codexOptions.config.model_providers; + if (expectedProvider) assert.deepEqual(restoredProvider, expectedProvider); + else assert.deepEqual(Object.keys(restoredProvider[modelProvider]).sort(), + ["base_url", "env_key", "name", "wire_api"]); + assert.equal(await readFile(path, "utf8"), bytes); + // Older snapshots can contain catalog definitions. Reading them must not + // rewrite their bytes or prevent the existing launch projection. + if (!expectedProvider) { + await writeSnapshot(providerDir, { ...selected, providerConfig: restoredProvider }); + const legacyBytes = await readFile(path, "utf8"); + const legacy = await loadSettings(providerDir); + assert.equal(legacy.providerConfig, undefined); + assert.deepEqual(restoreSettings(legacy, { filesystemDenies: [] }).codexOptions.config.model_providers, + restoredProvider); + assert.equal(await readFile(path, "utf8"), legacyBytes); + } + } + let credential = "synthetic-first"; + const restored = restoreSettings(captured, { filesystemDenies: ["/fixture/current-deny"] }, () => ({ + CODEX_API_KEY: credential, CODEX_HOME: "/fixture/observer-home", CODEX_CLI_PATH: "/fixture/observer-codex" + })); + assert.equal(restored.codexOptions.env.CODEX_API_KEY, "synthetic-first"); + credential = "synthetic-refreshed"; + assert.equal(restored.codexOptions.env.CODEX_API_KEY, "synthetic-refreshed"); + assert.equal(restored.codexOptions.env.CODEX_HOME, root); + assert.equal(restored.codexOptions.env.CODEX_CLI_PATH, captured.codexPath); + assert.deepEqual(restored.parentSandbox.filesystemDenies, ["/fixture/original-deny", "/fixture/current-deny"]); + assert.equal(restored.codexOptions.config.model_reasoning_effort, "ultra"); + await writeFile(join(root, "config.toml"), 'model = "native-home-model"\n'); + const native = await captureSettings({}, { filesystemDenies: [] }, { CODEX_CLI_PATH: process.execPath, CODEX_HOME: root }); + assert.equal(native.model, "native-home-model"); + const sessionDirectory = join(root, "sessions"); + await mkdir(sessionDirectory); + await writeFile(join(sessionDirectory, "parent.jsonl"), [ + { type: "session_meta", timestamp: "2026-01-01T00:00:00Z", payload: { id: "fixture-parent", model_provider: "openai" } }, + { type: "turn_context", timestamp: "2026-01-01T00:00:01Z", payload: { turn_id: "original-turn", model: "parent-model", effort: "high", summary: "none" } }, + { type: "turn_context", timestamp: "2026-01-01T00:02:00Z", payload: { turn_id: "later-turn", model: "later-model", effort: "low", summary: "detailed" } } + ].map(JSON.stringify).join("\n") + "\n"); + await writeFile(join(sessionDirectory, "other.jsonl"), JSON.stringify({ + type: "session_meta", payload: { id: "fixture-other", model_provider: "other-provider" } + }) + "\n"); + const parentSettings = await captureSettings({}, { filesystemDenies: [] }, { + CODEX_CLI_PATH: process.execPath, CODEX_HOME: root + }, { threadId: "fixture-parent", startedAt: "2026-01-01T00:01:00Z" }); + assert.equal(parentSettings.model, "native-home-model", "explicit config retains precedence"); + assert.equal(parentSettings.modelProvider, "openai"); + assert.equal(parentSettings.reasoningSummary, "none", "later owner turns are not original discovery settings"); + assert.equal(parentSettings.reasoningEffort, "high"); + await writeFile(join(root, "config.toml"), ""); + const parentEnvironment = { CODEX_CLI_PATH: process.execPath, CODEX_HOME: root }; + const [originalParent, otherParent, unavailableParent] = await Promise.all([ + captureSettings({}, { filesystemDenies: [] }, parentEnvironment, + { threadId: "fixture-parent", startedAt: "2026-01-01T00:00:01Z" }), + captureSettings({}, { filesystemDenies: [] }, parentEnvironment, + { threadId: "fixture-other", startedAt: "2026-01-01T00:00:01Z" }), + captureSettings({ model: "stored-model", reasoningEffort: "ultra" }, { filesystemDenies: [] }, parentEnvironment, + { threadId: "fixture-unavailable", startedAt: "2026-01-01T00:00:01Z" }) + ]); + assert.equal(originalParent.model, "parent-model"); + assert.equal(originalParent.reasoningSummary, "none", "the original turn is included at its timestamp"); + assert.equal(otherParent.modelProvider, "other-provider"); + assert.equal(otherParent.model, undefined, "concurrent scans do not borrow another parent's model"); + assert.equal(otherParent.reasoningSummary, undefined); + assert.equal(unavailableParent.model, "stored-model"); + assert.equal(unavailableParent.reasoningEffort, "ultra"); + assert.equal(unavailableParent.modelProvider, undefined, "missing history does not establish a provider"); + assert.equal(unavailableParent.reasoningSummary, undefined); + assert.equal(unavailableParent.serviceTier, undefined); + assert.equal(unavailableParent.nativeServiceTierAbsent, undefined, "missing history does not prove native absence"); + const originalOwner = { threadId: "fixture-parent", turnId: "original-turn", startedAt: "2026-01-01T00:00:00Z" }; + for (const workflowVersion of ["deep-security-scan/v1", "deep-scan-mcp/v1"]) { + const legacyDir = join(root, workflowVersion.replaceAll("/", "-")); + const legacy = await loadSettings(legacyDir, { + workflowVersion, model: "recorded-model", reasoningEffort: "ultra", + createdAt: "2026-01-01T00:01:00Z", usageOwner: null + }, async () => ({ config: { model_reasoning_summary: "concise", service_tier: "flex" }, + usageOwner: originalOwner }), parentEnvironment); + assert.equal(legacy.model, "recorded-model"); + assert.equal(legacy.reasoningSummary, "concise", "recorded recipe retains precedence"); + assert.equal(legacy.modelProvider, "openai", "recorded original owner supplies native selections"); + assert.equal(legacy.serviceTier, "flex"); + assert.equal(legacy.codexPath, undefined, "legacy metadata did not record an executable"); + assert.equal(legacy.codexHome, undefined, "a history lookup home is not recorded execution provenance"); + const restoredLegacy = restoreSettings(legacy, { filesystemDenies: ["/fixture/current-deny"] }, + () => ({ CODEX_HOME: "/fixture/runtime-home", CODEX_CLI_PATH: "/fixture/runtime-codex", + CODEX_API_KEY: "synthetic-live-key" })); + assert.equal(restoredLegacy.codexOptions.env.CODEX_HOME, "/fixture/runtime-home"); + assert.equal(restoredLegacy.codexOptions.env.CODEX_CLI_PATH, "/fixture/runtime-codex"); + assert.equal(restoredLegacy.codexOptions.config.model_reasoning_summary, "concise"); + await assert.rejects(readFile(join(legacyDir, "artifacts/deep_discovery/execution-settings.json")), + { code: "ENOENT" }); + } + await assert.rejects(loadSettings(join(root, "missing-v2"), { workflowVersion: "deep-security-scan/v2" }, + async () => assert.fail("missing promised v2 settings must not become legacy recovery")), /no recorded original/); + const [rebound, unboundLegacy] = await Promise.all([ + captureSettings({ usageOwner: originalOwner }, { filesystemDenies: [] }, parentEnvironment, + { threadId: "fixture-other", startedAt: "2026-01-01T00:03:00Z" }), + captureSettings({ model: "stored-model", usageOwner: null }, { filesystemDenies: [] }, parentEnvironment, + { threadId: "fixture-other", startedAt: "2026-01-01T00:03:00Z" }) + ]); + assert.equal(rebound.modelProvider, "openai", "takeover uses the recorded owner, not the invoking conversation"); + assert.equal(rebound.model, "parent-model", "the bound turn takes precedence over later turns"); + assert.equal(rebound.reasoningSummary, "none"); + assert.equal(unboundLegacy.model, "stored-model"); + assert.equal(unboundLegacy.modelProvider, undefined, "unrecorded legacy ownership cannot recover caller selections"); + assert.equal(unboundLegacy.reasoningSummary, undefined); + await writeFile(join(sessionDirectory, "legacy-auto.jsonl"), [ + { type: "session_meta", payload: { id: "fixture-legacy-auto", cli_version: "0.132.0", model_provider: "openai" } }, + { type: "turn_context", payload: { turn_id: "legacy-turn", model: "legacy-model", effort: "high", summary: "auto" } } + ].map(JSON.stringify).join("\n") + "\n"); + const legacyAuto = await captureSettings({ usageOwner: { threadId: "fixture-legacy-auto", turnId: "legacy-turn" } }, + { filesystemDenies: [] }, parentEnvironment); + assert.equal(legacyAuto.reasoningSummary, "auto", "older native turn-context selections remain readable"); + for (const version of ["0.133.0", "0.154.0"]) { + const threadId = `fixture-fresh-${version}`; + await writeFile(join(sessionDirectory, `${threadId}.jsonl`), [ + { type: "session_meta", timestamp: "2026-01-01T00:00:00Z", payload: { id: threadId, cli_version: version, model_provider: "openai" } }, + { type: "turn_context", timestamp: "2026-01-01T00:00:01Z", payload: { turn_id: "fresh-turn", model: "fresh-model", effort: "high", summary: "auto" } }, + { type: "event_msg", timestamp: "2026-01-01T00:02:00Z", payload: { type: "thread_settings_applied", thread_id: threadId, + thread_settings: { model: "fresh-model", model_provider_id: "openai", reasoning_summary: "detailed" } } } + ].map(JSON.stringify).join("\n") + "\n"); + const owner = { threadId, turnId: "fresh-turn", startedAt: "2026-01-01T00:01:00Z" }; + const fresh = await captureSettings({ usageOwner: owner }, { filesystemDenies: [] }, parentEnvironment); + assert.equal(fresh.model, "fresh-model"); + assert.equal(fresh.reasoningSummary, undefined, "fresh native compatibility auto is not an original selection"); + assert.equal(restoreSettings(fresh, { filesystemDenies: [] }).codexOptions.config.model_reasoning_summary, undefined); + const freshDir = join(root, threadId); + await writeSnapshot(freshDir, fresh); + const freshPath = join(freshDir, "artifacts", "deep_discovery", "execution-settings.json"); + const freshBytes = await readFile(freshPath, "utf8"); + assert.deepEqual(await loadSettings(freshDir, { usageOwner: owner, createdAt: owner.startedAt }), fresh); + assert.equal(await readFile(freshPath, "utf8"), freshBytes, "unknown summary is not replaced by a compatibility field or a later selection"); + await writeFile(join(root, "config.toml"), 'model_reasoning_summary = "auto"\n'); + const explicit = await captureSettings({ usageOwner: owner }, { filesystemDenies: [] }, parentEnvironment); + assert.equal(explicit.reasoningSummary, "auto", "an explicit original config selection still takes precedence"); + await writeFile(join(root, "config.toml"), ""); + } + await writeFile(join(sessionDirectory, "applied.jsonl"), [ + { type: "session_meta", timestamp: "2026-01-01T00:00:00Z", payload: { id: "fixture-applied", model_provider: "previous-provider" } }, + { type: "event_msg", timestamp: "2026-01-01T00:00:01Z", payload: { type: "thread_settings_applied", thread_id: "fixture-applied", + thread_settings: { model: "applied-model", model_provider_id: "openai", service_tier: "default", reasoning_effort: "high", reasoning_summary: "concise" } } }, + { type: "turn_context", timestamp: "2026-01-01T00:00:02Z", payload: { turn_id: "applied-turn", model: "previous-model", effort: "low", summary: "none" } }, + { type: "event_msg", timestamp: "2026-01-01T00:00:03Z", payload: { type: "thread_settings_applied", thread_id: "fixture-copied-owner", + thread_settings: { model: "copied-model", model_provider_id: "copied-provider", service_tier: "flex", reasoning_summary: "detailed" } } }, + { type: "event_msg", timestamp: "2026-01-01T00:02:00Z", payload: { type: "thread_settings_applied", thread_id: "fixture-applied", + thread_settings: { model: "later-model", model_provider_id: "later-provider", service_tier: "fast", reasoning_summary: "detailed" } } } + ].map(JSON.stringify).join("\n") + "\n"); + const appliedOwner = { threadId: "fixture-applied", turnId: "applied-turn", startedAt: "2026-01-01T00:00:00Z" }; + const applied = await captureSettings({ usageOwner: appliedOwner }, { filesystemDenies: [] }, parentEnvironment, + { threadId: "fixture-other", startedAt: "2026-01-01T00:01:00Z" }); + assert.equal(applied.serviceTier, "default", "original explicit standard routing survives later and copied snapshots"); + assert.equal(applied.reasoningSummary, "concise", "native applied summary overrides the legacy compatibility field"); + assert.equal(applied.modelProvider, "openai", "complete native snapshot replaces the session metadata provider"); + assert.equal(applied.model, "applied-model", "complete native snapshot replaces compatibility turn settings"); + assert.equal(applied.reasoningEffort, "high"); + assert.equal(applied.nativeServiceTierAbsent, undefined, "explicit native standard remains an explicit selection"); + const tierDir = join(root, "missing-tier"); + const { serviceTier: omittedTier, ...withoutTier } = applied; + assert.equal(omittedTier, "default"); + await writeSnapshot(tierDir, withoutTier); + const repairedTier = await loadSettings(tierDir, { + usageOwner: appliedOwner, createdAt: "2026-01-01T00:01:00Z" + }); + assert.equal(repairedTier.serviceTier, "default"); + await writeFile(join(sessionDirectory, "applied.jsonl"), (await readFile(join(sessionDirectory, "applied.jsonl"), "utf8")) + + JSON.stringify({ type: "event_msg", timestamp: "2026-01-01T00:00:04Z", payload: { + type: "thread_settings_applied", thread_id: "fixture-applied", + thread_settings: { model: "applied-model", model_provider_id: "openai", reasoning_effort: "high", service_tier: "priority" } + } }) + "\n"); + const nativeTier = await captureSettings({ usageOwner: appliedOwner }, { filesystemDenies: [] }, parentEnvironment, + { threadId: "fixture-other", startedAt: "2026-01-01T00:01:00Z" }); + assert.equal(nativeTier.serviceTier, "priority", "an effective tier selected by native remains unchanged"); + assert.equal(nativeTier.nativeServiceTierAbsent, undefined); + await writeFile(join(sessionDirectory, "applied.jsonl"), (await readFile(join(sessionDirectory, "applied.jsonl"), "utf8")) + + JSON.stringify({ type: "event_msg", timestamp: "2026-01-01T00:00:05Z", payload: { + type: "thread_settings_applied", thread_id: "fixture-applied", + thread_settings: { model: "applied-model", model_provider_id: "openai", reasoning_effort: "high" } + } }) + "\n"); + const nativeDefaults = await captureSettings({ usageOwner: appliedOwner }, { filesystemDenies: [] }, parentEnvironment, + { threadId: "fixture-other", startedAt: "2026-01-01T00:01:00Z" }); + assert.equal(nativeDefaults.model, "applied-model", "absent optional selections do not erase the required model"); + assert.equal(nativeDefaults.modelProvider, "openai", "absent optional selections do not erase the required provider"); + assert.equal(nativeDefaults.reasoningEffort, "high"); + assert.equal(nativeDefaults.serviceTier, "default", "known native absence retains its omitted request tier"); + assert.equal(nativeDefaults.nativeServiceTierAbsent, true, "known native absence is recorded separately from explicit standard"); + assert.equal(nativeDefaults.reasoningSummary, undefined, "a compatibility summary is not a recorded native default"); + const incompleteDir = join(root, "incomplete"); + const incomplete = { codexPath: process.execPath, codexHome: root, serviceTier: "flex" }; + await writeSnapshot(incompleteDir, incomplete); + await writeFile(join(root, "config.toml"), 'model_provider = "observer-provider"\nmodel_reasoning_summary = "detailed"\n'); + const originalRun = { model: "stored-model", reasoningEffort: "ultra", usageOwner: originalOwner, + createdAt: "2026-01-01T00:01:00Z" }; + const raceDir = join(root, "concurrent-recovery"); + await writeSnapshot(raceDir, incomplete); + const racePath = join(raceDir, "artifacts", "deep_discovery", "execution-settings.json"); + const originalCreateReadStream = fs.createReadStream; + const readingHistory = Promise.withResolvers(); + const releaseHistory = Promise.withResolvers(); + let held = false; + let pendingRead; + fs.createReadStream = (path, options) => { + const source = originalCreateReadStream(path, options); + if (path !== join(sessionDirectory, "parent.jsonl") || held) return source; + held = true; + const delayed = new PassThrough(); + source.once("error", (error) => delayed.destroy(error)); + delayed.once("close", () => source.destroy()); + void releaseHistory.promise.then(() => source.pipe(delayed)); + readingHistory.resolve(); + return delayed; + }; + syncBuiltinESMExports(); + try { + pendingRead = loadSettings(raceDir, originalRun); + await Promise.race([readingHistory.promise, pendingRead.then(() => + assert.fail("historical recovery must reach the controlled history read"))]); + const newer = { ...settings, modelProvider: "newer-provider", reasoningSummary: "concise" }; + await writeSnapshot(raceDir, newer); + const newerBytes = await readFile(racePath, "utf8"); + releaseHistory.resolve(); + const delayedProjection = await pendingRead; + assert.equal(delayedProjection.modelProvider, "openai"); + assert.equal(delayedProjection.reasoningSummary, "none"); + assert.equal(await readFile(racePath, "utf8"), newerBytes, + "a delayed historical projection must not overwrite a newer snapshot"); + assert.deepEqual(await loadSettings(raceDir), newer); + } finally { + releaseHistory.resolve(); + await pendingRead?.catch(() => {}); + fs.createReadStream = originalCreateReadStream; + syncBuiltinESMExports(); + } + const incompletePath = join(incompleteDir, "artifacts", "deep_discovery", "execution-settings.json"); + const incompleteBytes = await readFile(incompletePath, "utf8"); + const repaired = await loadSettings(incompleteDir, originalRun); + assert.deepEqual(repaired, { ...incomplete, model: "stored-model", reasoningEffort: "ultra", + modelProvider: "openai", reasoningSummary: "none" }); + assert.equal(await readFile(incompletePath, "utf8"), incompleteBytes, + "recovering historical fields is read-only"); + await rm(sessionDirectory, { recursive: true }); + const unavailable = await loadSettings(incompleteDir, originalRun); + assert.equal(unavailable.model, "stored-model"); + assert.equal(unavailable.reasoningEffort, "ultra"); + assert.equal(unavailable.modelProvider, undefined, "unavailable history remains unknown"); + assert.equal(unavailable.reasoningSummary, undefined); + assert.equal(await readFile(incompletePath, "utf8"), incompleteBytes); + const unknownDir = join(root, "unknown"); + await writeSnapshot(unknownDir, incomplete); + const unknown = await loadSettings(unknownDir, { ...originalRun, usageOwner: null }); + assert.equal(unknown.model, "stored-model"); + assert.equal(unknown.modelProvider, undefined, "missing original ownership is not current config"); + assert.equal(unknown.reasoningSummary, undefined); + assert.equal(unknown.nativeServiceTierAbsent, undefined); + const unsupported = JSON.stringify({ version: 99, settings }); + await writeFile(savedPath, unsupported); + assert.deepEqual(await loadSettings(join(root, "one")), settings, + "artifact versions and settings cannot replace trusted run state"); + snapshots.set(join(root, "one"), { version: 99, settings }); + await assert.rejects(loadSettings(join(root, "one")), /unsupported/); + assert.equal(await readFile(savedPath, "utf8"), unsupported); + await assert.rejects(loadRecordedSettings(join(root, "one"), { + workflowVersion: "deep-security-scan/v2" + }), /no recorded original/, "an existing artifact cannot establish missing launch provenance"); + snapshots.set(join(root, "one"), { version: 1, settings }); + await rm(savedPath); + assert.deepEqual(await loadSettings(join(root, "one")), settings, + "removing the artifact does not remove the trusted launch selection"); +} finally { + await rm(root, { recursive: true, force: true }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_replay.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_replay.mjs new file mode 100644 index 000000000..f982f021f --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_replay.mjs @@ -0,0 +1,92 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { build } from "esbuild"; + +const bundle = await build({ + entryPoints: [path.resolve(import.meta.dirname, "../src/deep-scan/coordinator.ts")], + loader: { ".md": "text" }, + bundle: true, format: "esm", platform: "node", write: false, + footer: { js: "//# sourceURL=deep-scan-selected-replay.js" }, +}); +const { DeepScanCoordinator } = await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}`, +); + +for (const terminalReason of ["saturated", "capped"]) { + test(`restart publishes the selected ${terminalReason} result after an output failure`, async () => { + const scanDir = await realpath(await mkdtemp(path.join(tmpdir(), "selected-replay-"))); + try { + const scanId = "4e7b4acb-ac80-4d68-98cd-3d5ac5581cd1"; + const draft = { + scanId, complete: true, findings: [], + coverage: { completeness: "partial", surfaces: [], explicitExclusions: [], + deferred: [{ id: "review", reason: "A dependency remains unreviewed." }] }, + }; + const { coverage, ...reduction } = draft; + const bytes = JSON.stringify({ ...reduction, sourceCoverage: coverage }); + const digest = createHash("sha256").update(bytes).digest("hex"); + const resultPath = `artifacts/deep_discovery/dedup/dedup-0001/output/checkpoints/${digest}.json`; + await mkdir(path.dirname(path.join(scanDir, resultPath)), { recursive: true }); + await writeFile(path.join(scanDir, resultPath), bytes); + const selection = { version: 1, resultPath, resultSha256: digest, terminalReason, + omittedWorkerIds: [], selectedAt: "2026-01-01T00:00:00Z" }; + let run = { + scanId, scanDir, targetPath: scanDir, scope: ".", workflowVersion: "deep-security-scan/v2", + status: "running", phase: "terminal", coordinatorGeneration: 3, + finalizationInput: selection, terminalReason, createdAt: "2026-01-01T00:00:00Z", + config: { workers: 2, subagents: 0, stopAfterNoNew: 2, stopAfterConsecutiveErrors: 2, + maxDiscoveryRuns: 4, maxTimeHours: 1 }, + dispatchedCount: 4, noNewStreak: 2, consecutiveErrors: 0, + }; + const mutations = []; + const store = new Proxy({ + get: async () => structuredClone(run), + finish: async (input) => { + mutations.push("finish"); + assert.equal(input.reason, terminalReason); + assert.deepEqual(input.omittedWorkerIds, selection.omittedWorkerIds); + run = { ...run, status: "succeeded", manifestPath: input.manifestPath }; + return structuredClone(run); + }, + fail: async () => { mutations.push("fail"); run = { ...run, status: "failed" }; return run; }, + }, { get: (target, key) => key in target ? target[key] : async () => { + mutations.push(key); throw new Error(`Unexpected scheduler operation: ${String(key)}`); + } }); + let executions = 0; + let publications = 0; + const options = { + store, executor: { run: async () => { executions++; throw new Error("Unexpected model work"); } }, + pluginRoot: scanDir, threadId: "original-result-conversation", retryDelaysMs: [], + // Already expired: replay must not start a discovery deadline timer. + discoveryTimeoutMs: 1, + onComplete: async (actual, _signal, publication) => { + publications++; + assert.deepEqual(actual, draft); + assert.equal(publication.coordinatorGeneration, 3); + assert.equal(publication.resultPath, path.join(scanDir, resultPath)); + if (publications === 1) throw new Error("Synthetic publication write failure"); + }, + }; + const first = new DeepScanCoordinator({ ...options, run: structuredClone(run) }); + first.start(); + await assert.rejects(first.wait(), /Synthetic publication write failure/); + assert.equal(run.status, "running"); + assert.equal(run.terminalReason, terminalReason); + assert.deepEqual(run.finalizationInput, selection); + const restarted = new DeepScanCoordinator({ ...options, run: structuredClone(run) }); + restarted.start(); + const completed = await restarted.wait(); + assert.equal(completed.status, "succeeded"); + assert.equal(completed.terminalReason, terminalReason); + assert.equal(executions, 0); + assert.deepEqual(mutations, ["finish"]); + assert.equal(publications, 2); + } finally { + await rm(scanDir, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_selection_store.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_selection_store.mjs new file mode 100644 index 000000000..4f9300233 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_selection_store.mjs @@ -0,0 +1,42 @@ +import assert from "node:assert/strict"; +import path from "node:path"; +import { test } from "node:test"; +import { build } from "esbuild"; + +const bundle = await build({ + entryPoints: [path.resolve(import.meta.dirname, "../src/deep-scan/store.ts")], + bundle: true, format: "esm", platform: "node", write: false, + footer: { js: "//# sourceURL=deep-scan-selection-store.js" }, +}); +const { WorkbenchDeepScanStore } = await import(`data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}`); + +test("selection uses the dedicated function bridge and the store's existing replay policy", async () => { + const scanId = "ed8ff2da-01d9-4338-aeba-8bcfd4b530a9"; + const selection = { + version: 1, resultPath: "artifacts/merge/checkpoints/aggregate.json", resultSha256: "a".repeat(64), + terminalReason: "saturated", omittedWorkerIds: [], selectedAt: "2026-01-01T00:00:00Z", + }; + const response = { deepScan: { + scanId, targetPath: "/target", scope: ".", scanDir: "/scan", status: "running", + schemaVersion: 1, workflowVersion: "deep-security-scan/v2", coordinatorGeneration: 2, + config: { workers: 2, subagents: 0, stopAfterNoNew: 2, stopAfterConsecutiveErrors: 2, maxDiscoveryRuns: 4 }, + dispatchedCount: 2, noNewStreak: 2, consecutiveErrors: 0, finalizationInput: selection, + } }; + const calls = []; + const store = new WorkbenchDeepScanStore(async (...args) => { + calls.push(structuredClone(args)); + if (calls.length === 1) throw Object.assign(new Error("Synthetic lost selection response"), { code: "ETIMEDOUT" }); + return response; + }); + const result = await store.selectFinalization({ + scanId, coordinatorGeneration: 2, reason: "saturated", manifestPath: "/scan/scan-manifest.json", + resultPath: "/scan/artifacts/merge/checkpoints/aggregate.json", omittedWorkerIds: [], + }); + assert.deepEqual(result.finalizationInput, selection); + assert.equal(calls.length, 2); + assert.deepEqual(calls[1], calls[0]); + assert.deepEqual(calls[0], [[ + "finish-deep-scan", "--scan-id", scanId, "--coordinator-generation", "2", + "--terminal-reason", "saturated", "--manifest-path", "/scan/scan-manifest.json", + ], JSON.stringify({ resultPath: "/scan/artifacts/merge/checkpoints/aggregate.json" }), true]); +}); diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.mjs index e2a0fd3cd..ea2acde6e 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.mjs +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.mjs @@ -10,7 +10,10 @@ import { build } from "esbuild"; const execFileAsync = promisify(execFile); const mcpAppRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); -const pluginRoot = path.resolve(mcpAppRoot, ".."); +const installedPluginRoot = process.env.CODEX_SECURITY_TEST_PLUGIN_ROOT; +const pluginRoot = installedPluginRoot + ? path.resolve(installedPluginRoot) + : path.resolve(mcpAppRoot, ".."); const workbenchPath = path.join(pluginRoot, "scripts", "workbench_db.py"); const parentSandboxState = { permissionProfile: { @@ -55,7 +58,7 @@ async function testDeepScanStdioLifecycle() { const serverBundlePath = path.join( pluginRoot, "mcp", - `.deep-scan-stdio-test-${randomUUID()}.cjs` + installedPluginRoot ? "server.mjs" : `.deep-scan-stdio-test-${randomUUID()}.cjs` ); const threadId = "deep-scan-stdio-lifecycle-thread"; @@ -86,7 +89,7 @@ async function testDeepScanStdioLifecycle() { '' ].join('\n')); await writePythonWrapper(pythonWrapperPath); - await bundleServer(serverBundlePath); + if (!installedPluginRoot) await bundleServer(serverBundlePath); const environment = { ...process.env, @@ -429,7 +432,16 @@ async function testDeepScanStdioLifecycle() { "the MCP server must remain responsive after canceling one scan" ); - const resumedThreadId = "deep-scan-stdio-resumed-thread"; + const originalThreadId = "deep-scan-stdio-resumed-thread"; + await mkdir(path.join(codexHome, "sessions"), { recursive: true }); + await writeFile(path.join(codexHome, "sessions", "original-owner.jsonl"), [ + { type: "session_meta", timestamp: "2026-01-01T00:00:00Z", + payload: { id: originalThreadId, cli_version: "0.154.0", model_provider: "openai" } }, + { type: "event_msg", timestamp: "2026-01-01T00:00:01Z", + payload: { type: "thread_settings_applied", thread_id: originalThreadId, + thread_settings: { reasoning_summary: "none", model_provider_id: "openai" } } } + ].map(JSON.stringify).join("\n") + "\n"); + let resumedThreadId = originalThreadId; const opened = await server.request(24, "tools/call", toolCall( "open_codex_security_workspace", { targetPath, scope: ".", mode: "deep" }, @@ -439,7 +451,7 @@ async function testDeepScanStdioLifecycle() { const sessionId = opened.result.structuredContent.workspace.id; assertNoError(await server.request(25, "tools/call", toolCall( "submit_codex_security_setup", - { sessionId, targetPath, scope: ".", mode: "deep" }, + { sessionId, targetPath, scope: ".", mode: "deep", userContext: "Original discovery focus" }, resumedThreadId ))); const started = await server.request(26, "tools/call", toolCall( @@ -450,7 +462,7 @@ async function testDeepScanStdioLifecycle() { assertNoError(started); const resumedScan = started.result.structuredContent.workspace.results; const resumedScanId = resumedScan.scanId; - const handoffClaimToken = randomUUID(); + let handoffClaimToken = randomUUID(); for (const [id, name, arguments_] of [ [27, "claim_codex_security_scan_handoff_delivery", { scanId: resumedScanId, claimToken: handoffClaimToken @@ -484,8 +496,15 @@ async function testDeepScanStdioLifecycle() { const completedDraft = JSON.parse(await readFile(completedWorker.resultManifestPath, "utf8")); assert.equal(completedDraft.scanId, resumedScanId); assert.deepEqual(completedDraft.findings, []); + assert.equal(partial.workflowVersion, "deep-security-scan/v1", "the prior reader starts the legacy workflow"); + assert.equal(partial.userContext, "Original discovery focus"); + const settingsPath = path.join(resumedScan.scanDir, "artifacts", "deep_discovery", "execution-settings.json"); + await assert.rejects(readFile(settingsPath), { code: "ENOENT" }); await server.stop(); assert.throws(() => process.kill(server.pid, 0), "the original MCP server must have exited"); + // A worker can be marked running before its process appends the start log. + // Capture the original process set after shutdown has settled those launches. + const originalWorkerPids = new Set((await readJsonLines(startLogPath)).slice(restartStartIndex).map((execution) => execution.pid)); const paused = await runWorkbench(environment, ["get-scan", "--scan-id", resumedScanId]); assert.deepEqual([paused.scan.progress.status, paused.scan.progress.phase], ["running", "discovery"]); assert.deepEqual(paused.scan.progress.independentReviews, { @@ -512,6 +531,18 @@ async function testDeepScanStdioLifecycle() { path.join(stateDir, "workbench.sqlite3"), resumedScanId ]); + await runWorkbench(environment, [ + "release-handoff-delivery", "--scan-id", resumedScanId, "--claim-token", handoffClaimToken + ]); + handoffClaimToken = randomUUID(); + resumedThreadId = "deep-scan-stdio-replacement-thread"; + await runWorkbench(environment, [ + "claim-handoff-delivery", "--scan-id", resumedScanId, "--claim-token", handoffClaimToken + ]); + await runWorkbench(environment, [ + "attach-scan-continuation-thread", "--scan-id", resumedScanId, + "--claim-token", handoffClaimToken, "--thread-id", resumedThreadId + ]); await writeFile(restartControlPath, "after-restart"); const restartedServer = startServer(serverBundlePath, environment); @@ -537,8 +568,13 @@ async function testDeepScanStdioLifecycle() { environment, scanId: resumedScanId, threadId: resumedThreadId }); assert.equal(finished.status, "succeeded"); + assert.equal(finished.workflowVersion, partial.workflowVersion); assert.equal(finished.coordinatorGeneration, partial.coordinatorGeneration + 1); assert.equal(finished.dispatchedCount, 2); + assert.equal(finished.userContext, partial.userContext); + assert.equal(finished.createdAt, partial.createdAt, "recovery retains the original deadline origin"); + assert.equal(finished.config.maxTimeHours, partial.config.maxTimeHours); + await assert.rejects(readFile(settingsPath), { code: "ENOENT" }); const successfulDiscoveries = finished.workers.filter((worker) => ( worker.kind === "discovery" && worker.status === "succeeded" )); @@ -556,7 +592,12 @@ async function testDeepScanStdioLifecycle() { ); const executions = (await readJsonLines(startLogPath)).slice(restartStartIndex); for (const execution of executions) { - assert.equal(execution.argv.includes('model_reasoning_summary="none"'), true); + const summary = execution.argv.find((argument) => argument.startsWith("model_reasoning_summary=")); + // Baseline native handoff replaces its owner binding. Without a saved + // recipe or snapshot, the old summary is unavailable after that handoff. + assert.equal(summary, originalWorkerPids.has(execution.pid) ? 'model_reasoning_summary="none"' : undefined); + const context = discoveryPromptContext(execution.stdin); + if (context.workerLabel) assert.equal(context.userContext, "Original discovery focus"); } assert.equal(executions.filter((execution) => ( discoveryPromptContext(execution.stdin).workerLabel === "discovery-0001" @@ -569,7 +610,7 @@ async function testDeepScanStdioLifecycle() { throw error; } finally { await server.stop(); - await rm(serverBundlePath, { force: true }); + if (!installedPluginRoot) await rm(serverBundlePath, { force: true }); await rm(fixtureRoot, { recursive: true, force: true }); } } @@ -577,6 +618,7 @@ async function testDeepScanStdioLifecycle() { async function bundleServer(outfile) { await build({ bundle: true, + nodePaths: [fileURLToPath(new URL("../node_modules", import.meta.url))], define: { "import.meta.url": "__filename" }, entryPoints: [path.join(mcpAppRoot, "main.ts")], external: ["fsevents"], diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_store.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_store.mjs index 09eb1f884..4c14abc35 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_store.mjs +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_store.mjs @@ -21,6 +21,7 @@ await testCanonicalCommitProtocol(); await testTerminalProtocol(); testCanonicalNullAndPartialParsing(); testRunErrorParsing(); +testWorkflowVersionParsing(); testConfiguredMaximumDurationParsing(); await testWriteSerializationAndRecovery(); await testBeginUsesTheWriteQueue(); @@ -35,6 +36,15 @@ await testPersistenceRetryExhaustionPreservesDiagnostics(); await testDeterministicPersistenceFailuresAreNotRetried(); await testNonIdempotentMutationsAreNotRetried(); testInvalidPersistedConfig(); +testOriginalUsageOwnerParsing(); + +function testOriginalUsageOwnerParsing() { + const value = stateResult(randomUUID()); + const usageOwner = { threadId: "original-thread", turnId: "original-turn", startedAt: "2026-01-01T00:00:00Z" }; + assert.deepEqual(parseDeepScan({ deepScan: { ...value.deepScan, usageOwner } }).usageOwner, usageOwner); + assert.equal(parseDeepScan({ deepScan: { ...value.deepScan, usageOwner: null } }).usageOwner, null); + assert.equal(parseDeepScan(value).usageOwner, null, "old readers do not establish an original owner"); +} async function testBeginProtocolAndParsing() { const scanId = randomUUID(); @@ -71,7 +81,7 @@ async function testBeginProtocolAndParsing() { assert.equal(calls[0].input, "focus on archive parsing"); assert.equal(flagValue(calls[0].args, "--scan-root"), "/fixture/scans"); assert.equal(flagValue(calls[0].args, "--available-parallelism"), String(availableParallelism())); - assert.equal(flagValue(calls[0].args, "--workflow-version"), "deep-scan-mcp/v1"); + assert.equal(flagValue(calls[0].args, "--workflow-version"), "deep-security-scan/v1"); const claimToken = randomUUID(); let joinedArgs; @@ -424,7 +434,7 @@ async function testPersistenceRetriesRemainInsideTheWriteQueue() { if (args[0] === "claim-deep-scan-dedup" && calls.length === 1) { throw new Error("sqlite3.OperationalError: database is locked"); } - return {}; + return stateResult(scanId); }); const claim = store.claimDedup({ @@ -705,7 +715,7 @@ function idempotentPersistenceScenarios() { }) }, { operation: "claim-deep-scan-dedup", - result: {}, + result: stateResult(scanId), invoke: (store) => store.claimDedup({ id: reducerId, scanId, @@ -875,3 +885,31 @@ function deferred() { }); return { promise, resolve }; } + +function testWorkflowVersionParsing() { + const value = stateResult(randomUUID()).deepScan; + const run = parseDeepScan({ deepScan: { ...value, schemaVersion: 1, workflowVersion: "deep-scan-mcp/v1", model: "original-model", reasoningEffort: "high" } }); + assert.equal(run.model, "original-model"); + assert.equal(run.reasoningEffort, "high"); + assert.equal(run.schemaVersion, 1); + assert.equal(run.workflowVersion, "deep-scan-mcp/v1"); + const future = parseDeepScan({ deepScan: { ...value, schemaVersion: 99, workflowVersion: "future/v99" } }); + assert.equal(future.schemaVersion, 99); + assert.equal(future.workflowVersion, "future/v99", "inspection preserves unsupported versions"); +} + +for (const version of [1, 99]) { + const finalizationInput = { + version, + resultPath: null, + resultSha256: null, + terminalReason: "capped", + omittedWorkerIds: ["fixture-worker"], + selectedAt: "2026-01-01T00:00:00Z" + }; + assert.deepEqual( + parseDeepScan(stateResult(randomUUID(), { deepScan: { finalizationInput } })).finalizationInput, + finalizationInput, + "inspection preserves finalization input and version before execution compatibility checks" + ); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.mjs index a93384919..430e71600 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.mjs +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.mjs @@ -91,7 +91,10 @@ async function testRecoveredPublicationRejectsLateFailure() { paths: ["fixture.py"], }], }, - }, runWorkbench); + }, runWorkbench, undefined, { + coordinatorGeneration: claim.run.coordinatorGeneration, + resultPath: null, + }); await runWorkbench([ "cancel-scan", "--scan-id", run.scanId, "--thread-id", "publication-failure-owner", @@ -318,6 +321,7 @@ async function testReducerCommitAndFinishAgainstRealWorkbench() { CODEX_SECURITY_STATE_DIR: stateDir }; const python = process.env.PYTHON?.trim() || "python3"; + const finishCalls = []; const runWorkbench = async (args) => { const { stdout } = await execFileAsync(python, [workbenchPath, ...args], { cwd: pluginRoot, @@ -325,6 +329,12 @@ async function testReducerCommitAndFinishAgainstRealWorkbench() { maxBuffer: 4 * 1024 * 1024, timeout: 30_000 }); + if (args[0] === "finish-deep-scan") { + finishCalls.push([...args]); + if (finishCalls.length === 1) { + throw Object.assign(new Error("Synthetic lost committed finish response"), { code: "ETIMEDOUT" }); + } + } return JSON.parse(stdout); }; const store = new WorkbenchDeepScanStore(runWorkbench); @@ -505,6 +515,8 @@ async function testReducerCommitAndFinishAgainstRealWorkbench() { omittedWorkerIds: [late.id] }); assert.equal(finished.status, "succeeded"); + assert.equal(finishCalls.length, 2); + assert.deepEqual(finishCalls[1], finishCalls[0]); assert.equal(finished.terminalReason, "saturated"); assert.equal(finished.manifestPath, manifestPath); diff --git a/plugins/codex-security/mcp-app/tests/test_reader_release_settings.mjs b/plugins/codex-security/mcp-app/tests/test_reader_release_settings.mjs new file mode 100644 index 000000000..e258565ef --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_reader_release_settings.mjs @@ -0,0 +1,47 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm, stat, mkdir, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { test } from "node:test"; +import { build } from "esbuild"; + +const bundle = await build({ + bundle: true, + nodePaths: [fileURLToPath(new URL("../node_modules", import.meta.url))], + entryPoints: [fileURLToPath(new URL("../src/deep-scan/recovery-settings.ts", import.meta.url))], + platform: "node", + format: "esm", + write: false, +}); +const { loadDeepScanExecutionSettings } = await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}` +); + +for (const state of ["absent", "saved", "unsupported"]) { + test(`reader release uses ${state} settings without persisting new metadata`, async () => { + const root = await mkdtemp(join(tmpdir(), "reader-settings-")); + const path = join(root, "artifacts", "deep_discovery", "execution-settings.json"); + try { + let contextReads = 0; + const readLegacyContext = async () => { contextReads++; return { config: {} }; }; + const original = { workflowVersion: "deep-security-scan/v1", model: "original-model", reasoningEffort: "high", createdAt: "2026-01-01T00:00:00Z", usageOwner: null }; + let bytes; + if (state !== "absent") { + await mkdir(join(root, "artifacts", "deep_discovery"), { recursive: true }); + bytes = JSON.stringify({ version: state === "unsupported" ? 99 : 1, settings: { codexPath: join(root, "codex"), codexHome: root, parentSandbox: { filesystemDenies: [] } } }); + await writeFile(path, bytes); + } + const loaded = await loadDeepScanExecutionSettings(root, original, readLegacyContext); + assert.equal(loaded.model, "original-model"); + assert.equal(loaded.reasoningEffort, "high"); + assert.equal(loaded.codexPath, undefined); + assert.equal(loaded.codexHome, undefined); + assert.equal(contextReads, 1, "legacy recovery ignores planted execution artifacts"); + if (state === "absent") await assert.rejects(stat(path), { code: "ENOENT" }); + else assert.equal(await readFile(path, "utf8"), bytes); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_workbench_state_fallback.mjs b/plugins/codex-security/mcp-app/tests/test_workbench_state_fallback.mjs index e6cea4f54..960a30d0d 100644 --- a/plugins/codex-security/mcp-app/tests/test_workbench_state_fallback.mjs +++ b/plugins/codex-security/mcp-app/tests/test_workbench_state_fallback.mjs @@ -29,6 +29,7 @@ async function testWorkbenchStateFallback() { await writeFakePython(fakePythonPath); await build({ bundle: true, + nodePaths: [fileURLToPath(new URL("../node_modules", import.meta.url))], define: { "import.meta.url": "__filename" }, entryPoints: [path.join(mcpAppRoot, "main.ts")], external: ["fsevents"], diff --git a/plugins/codex-security/mcp-app/tsconfig.json b/plugins/codex-security/mcp-app/tsconfig.json index ef0922f6b..5f455476e 100644 --- a/plugins/codex-security/mcp-app/tsconfig.json +++ b/plugins/codex-security/mcp-app/tsconfig.json @@ -5,6 +5,10 @@ "module": "ESNext", "moduleResolution": "Bundler", "noEmit": true, + "paths": { + "@openai/codex-sdk": ["./node_modules/@openai/codex-sdk"], + "smol-toml": ["./node_modules/smol-toml/dist/index"] + }, "resolveJsonModule": true, "skipLibCheck": true, "strict": true, diff --git a/plugins/codex-security/native/proof-policy-windows.mts b/plugins/codex-security/native/proof-policy-windows.mts index 5ca186a39..65235e6af 100644 --- a/plugins/codex-security/native/proof-policy-windows.mts +++ b/plugins/codex-security/native/proof-policy-windows.mts @@ -8,10 +8,11 @@ import { nativeTarget } from "./platform.mjs"; const testDirectory = join(output, "policy-proof"); const helper = join(testDirectory, "helpers.cjs"); if (process.argv[2] === "build") { + const sdkModules = join(root, "../../../sdk/typescript/node_modules"); execFileSync( process.execPath, [ - join(root, "../../../sdk/typescript/node_modules/esbuild/bin/esbuild"), + join(sdkModules, "esbuild/bin/esbuild"), join(root, "../mcp-app/helpers-main.ts"), "--bundle", "--platform=node", @@ -20,7 +21,11 @@ if (process.argv[2] === "build") { "--define:import.meta.url=__filename", `--outfile=${helper}`, ], - { stdio: "inherit" }, + { + stdio: "inherit", + // Native CI installs the helper's dependencies only in the SDK. + env: { ...process.env, NODE_PATH: sdkModules }, + }, ); const nativeDirectory = join(testDirectory, "native", nativeTarget); mkdirSync(nativeDirectory, { recursive: true }); diff --git a/plugins/codex-security/scripts/deep_scan_workbench.py b/plugins/codex-security/scripts/deep_scan_workbench.py index 853094a0a..65fd39bfa 100644 --- a/plugins/codex-security/scripts/deep_scan_workbench.py +++ b/plugins/codex-security/scripts/deep_scan_workbench.py @@ -21,6 +21,7 @@ from filesystem_identity import serialize_filesystem_identity from finalize_scan_contract import _read_scan_local_json from workbench.handoff import require_current_continuation +from workbench_saved_results import _worker_checkpoint_head from workbench_target import ( directory_content_digest, directory_snapshot_regular_file_count, @@ -38,6 +39,11 @@ ) DEEP_SCAN_TERMINAL_REASONS = ("saturated", "capped") DEEP_SCAN_WORKFLOW_VERSION = "deep-security-scan/v1" +SUPPORTED_DEEP_SCAN_WORKFLOWS = { + "deep-security-scan/v2", + "deep-scan-mcp/v1", + "deep-security-scan/v1", +} DEEP_SCAN_COORDINATOR_LEASE_SECONDS = 30 DEEP_SCAN_LEGACY_COORDINATOR_GRACE_SECONDS = 120 DEEP_SCAN_MAX_ERROR_LENGTH = 2400 @@ -264,20 +270,50 @@ def require_deep_scan_run(connection: sqlite3.Connection, scan_id: str) -> sqlit return row +def require_supported_deep_scan(run: sqlite3.Row) -> None: + if run["schema_version"] != 1 or run["workflow_version"] not in SUPPORTED_DEEP_SCAN_WORKFLOWS: + raise SystemExit( + "This Deep Scan uses an unsupported workflow or schema version. " + "Resume it with a compatible Codex Security release." + ) + + finalization = deep_scan_finalization_input(run) + if finalization is not None and ( + run["workflow_version"] != "deep-security-scan/v2" + or not isinstance(finalization, dict) + or finalization.get("version") != 1 + ): + raise SystemExit("This Deep Scan uses an unsupported finalization input version.") + + +def deep_scan_finalization_input(run: sqlite3.Row) -> dict[str, Any] | None: + if "finalization_input_json" not in run.keys() or run["finalization_input_json"] is None: + return None + return json.loads(run["finalization_input_json"]) + + def deep_scan_deadline_reached(run: sqlite3.Row) -> bool: elapsed = _parse_timestamp(now()) - _parse_timestamp(str(run["created_at"])) return elapsed.total_seconds() / 3600 >= run["max_time_hours"] +def find_supported_deep_scan_run( + connection: sqlite3.Connection, scan_id: str +) -> sqlite3.Row | None: + run = connection.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) + ).fetchone() + if run is not None: + require_supported_deep_scan(run) + return run + + def require_deep_scan_ready_for_parent_completion( connection: sqlite3.Connection, scan: sqlite3.Row ) -> None: if scan["mode"] != "deep": return - run = connection.execute( - "SELECT status, manifest_path FROM deep_scan_runs WHERE scan_id = ?", - (scan["id"],), - ).fetchone() + run = find_supported_deep_scan_run(connection, scan["id"]) if run is None or run["status"] != "succeeded" or run["manifest_path"] is None: raise SystemExit( "Deep Scan discovery orchestration must finish and persist its manifest before " @@ -410,20 +446,35 @@ def canonical_discovery_artifacts(scan: sqlite3.Row) -> dict[str, str]: def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, Any]: + if connection.in_transaction: + return _deep_scan_state(connection, scan_id) + connection.execute("BEGIN") + try: + state = _deep_scan_state(connection, scan_id) + connection.commit() + return state + except BaseException: + connection.rollback() + raise + + +def _deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, Any]: run = require_deep_scan_run(connection, scan_id) scan = require_scan(connection, run["scan_id"]) worker_rows = connection.execute( """ - SELECT * - FROM deep_scan_workers - WHERE scan_id = ? - ORDER BY created_at, id + SELECT workers.*, attempts.accepted_result_path + FROM deep_scan_workers AS workers + LEFT JOIN deep_scan_attempts AS attempts + ON attempts.worker_id = workers.id AND attempts.attempt = workers.attempt + WHERE workers.scan_id = ? + ORDER BY workers.created_at, workers.id """, (run["scan_id"],), ) input_rows = connection.execute( """ - SELECT dedup_worker_id, discovery_worker_id, input_order + SELECT * FROM deep_scan_dedup_inputs WHERE scan_id = ? ORDER BY dedup_worker_id, input_order @@ -453,10 +504,22 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A "scanId": run["scan_id"], "targetPath": scan["target_path"], "scope": scan["scope"], - "userContext": scan["user_context"], + "model": scan["model"], + "reasoningEffort": scan["reasoning_effort"], + "userContext": ( + run["discovery_user_context"] + if "discovery_user_context" in run.keys() + else scan["user_context"] + ), "scanDir": scan["scan_dir"], "schemaVersion": run["schema_version"], "workflowVersion": run["workflow_version"], + "finalizationInput": deep_scan_finalization_input(run), + "usageOwner": ( + json.loads(run["usage_owner_json"]) + if "usage_owner_json" in run.keys() and run["usage_owner_json"] + else None + ), "coordinatorGeneration": run["coordinator_generation"], "status": run["status"], "phase": run["phase"], @@ -481,11 +544,55 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A "updatedAt": run["updated_at"], "completedAt": run["completed_at"], "workers": [deep_scan_worker_state(row) for row in worker_rows], + "attempts": [ + { + "workerId": row["worker_id"], + "attempt": row["attempt"], + "status": row["status"], + "startedAt": row["started_at"], + "completedAt": row["completed_at"], + "endReason": row["end_reason"], + "error": row["error_message"], + "acceptedResultPath": row["accepted_result_path"], + "acceptedResultSha256": row["accepted_result_sha256"], + } + for row in connection.execute( + "SELECT * FROM deep_scan_attempts WHERE scan_id = ? ORDER BY worker_id, attempt", + (scan_id,), + ) + ], + "attemptSessions": [ + { + "workerId": row["worker_id"], + "attempt": row["attempt"], + "sdkThreadId": row["sdk_thread_id"], + "observedAt": row["observed_at"], + } + for row in connection.execute( + "SELECT * FROM deep_scan_attempt_sessions WHERE scan_id = ? " + "ORDER BY observed_at, worker_id, attempt, sdk_thread_id", + (scan_id,), + ) + ], + "mergeClaims": [ + { + "workerId": row["worker_id"], + "previousWorkerId": row["previous_worker_id"], + "previousResultPath": row["previous_result_path"], + "previousResultSha256": row["previous_result_sha256"], + } + for row in connection.execute( + "SELECT * FROM deep_scan_merge_claims WHERE scan_id = ? ORDER BY rowid", (scan_id,) + ) + ], "dedupInputs": [ { "dedupWorkerId": row["dedup_worker_id"], "discoveryWorkerId": row["discovery_worker_id"], "inputOrder": row["input_order"], + "resultManifestPath": row["result_manifest_path"], + "resultManifestSha256": row["result_manifest_sha256"], + "attempt": row["attempt"], } for row in input_rows ], @@ -533,6 +640,9 @@ def deep_scan_worker_state(row: sqlite3.Row) -> dict[str, Any]: "promptPath": row["prompt_path"], "artifactDir": row["artifact_dir"], "resultManifestPath": row["result_manifest_path"], + "acceptedResultPath": row["accepted_result_path"] + if "accepted_result_path" in row.keys() + else None, "attempt": row["attempt"], "sdkThreadId": row["sdk_thread_id"], "completionSequence": row["completion_sequence"], @@ -561,6 +671,49 @@ def effective_deep_scan_config(args: argparse.Namespace) -> dict[str, int | floa return resolve_deep_scan_config(available_parallelism) +def require_legacy_deep_scan_creation(connection: sqlite3.Connection) -> None: + if any( + row["name"] == "discovery_user_context" + for row in connection.execute("PRAGMA table_info(deep_scan_runs)") + ): + raise SystemExit("This Deep Scan database requires a newer version to start a scan.") + + +def recorded_deep_scan_execution_settings(run: sqlite3.Row) -> dict[str, Any] | None: + saved = run["execution_settings_json"] if "execution_settings_json" in run.keys() else None + return json.loads(saved) if saved else None + + +def include_execution_settings(connection: sqlite3.Connection, result: dict[str, Any]) -> None: + if "deepScan" in result: + run = require_deep_scan_run(connection, result["deepScan"]["scanId"]) + result["deepScan"]["executionSettings"] = recorded_deep_scan_execution_settings(run) + + +def read_deep_scan_execution_settings(scan_dir: Path) -> dict[str, Any]: + relative_path = "artifacts/deep_discovery/execution-settings.json" + if not (scan_dir / relative_path).exists(): + raise SystemExit( + "This Deep Scan has no recorded original execution settings; " + "its executable and Codex home cannot be recovered." + ) + saved = _read_scan_local_json(scan_dir, relative_path, "Deep Scan execution settings") + return validate_deep_scan_execution_settings(saved) + + +def validate_deep_scan_execution_settings(saved: dict[str, Any]) -> dict[str, Any]: + if saved.get("version") != 1: + raise SystemExit("This Deep Scan uses an unsupported execution settings version.") + settings = saved.get("settings") + if not isinstance(settings, dict) or not all( + isinstance(settings.get(key), str) for key in ("codexPath", "codexHome") + ): + raise SystemExit( + "Deep Scan execution settings are missing the recorded executable or Codex home." + ) + return settings + + def ensure_deep_scan_run( connection: sqlite3.Connection, scan: sqlite3.Row, @@ -572,7 +725,11 @@ def ensure_deep_scan_run( "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan["id"],) ).fetchone() if existing is not None: + require_supported_deep_scan(existing) return existing + if workflow_version == "deep-security-scan/v2": + raise SystemExit("This Deep Scan requires a newer version to start this workflow.") + require_legacy_deep_scan_creation(connection) if scan["mode"] != "deep": raise SystemExit("Deep Scan orchestration requires a scan in deep mode.") if scan["status"] != "running": @@ -687,6 +844,15 @@ def begin_deep_scan_for_scan( ) -> dict[str, Any]: scan_id = require_uuid(scan_id, "scan-id") candidate = require_scan(connection, scan_id) + existing = connection.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) + ).fetchone() + if existing is not None: + require_legacy_deep_scan_execution(connection, existing) + if existing is None and args.workflow_version == "deep-security-scan/v2": + raise SystemExit("This Deep Scan requires a newer version to start this workflow.") + if existing is None: + require_legacy_deep_scan_creation(connection) workspace = require_workspace(connection, candidate["workspace_id"]) if ( candidate["mode"] == "deep" @@ -724,23 +890,10 @@ def begin_deep_scan_for_scan( ) if scan["mode"] != "deep": raise SystemExit("Deep Scan orchestration requires a scan in deep mode.") - model = optional_text(args.model, maximum=200) - reasoning_effort = optional_text(args.reasoning_effort, maximum=32) - if model is not None or reasoning_effort is not None: - connection.execute( - """ - UPDATE scans - SET model = COALESCE(?, model), reasoning_effort = COALESCE(?, reasoning_effort) - WHERE id = ? - """, - (model, reasoning_effort, scan_id), - ) - connection.commit() - existing = connection.execute( - "SELECT scan_id FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) - ).fetchone() if existing is not None: return deep_scan_result(connection, scan_id, start_disposition="joined") + model = optional_text(args.model, maximum=200) + reasoning_effort = optional_text(args.reasoning_effort, maximum=32) config = effective_deep_scan_config(args) workflow_version = optional_text(args.workflow_version, maximum=256) if workflow_version is None: @@ -753,6 +906,22 @@ def begin_deep_scan_for_scan( args.claim_token, error_message="Deep Scan orchestration is owned by another continuation.", ) + existing = connection.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) + ).fetchone() + if existing is not None: + require_legacy_deep_scan_execution(connection, existing) + connection.commit() + return deep_scan_result(connection, scan_id, start_disposition="joined") + if model is not None or reasoning_effort is not None: + connection.execute( + """ + UPDATE scans + SET model = COALESCE(?, model), reasoning_effort = COALESCE(?, reasoning_effort) + WHERE id = ? + """, + (model, reasoning_effort, scan_id), + ) ensure_deep_scan_run(connection, scan, config, workflow_version, now()) connection.commit() except BaseException: @@ -788,8 +957,10 @@ def begin_deep_scan_for_target( existing = existing_deep_scan_for_target(connection, thread_id, target_path, scope) if existing is not None: existing_run = connection.execute( - "SELECT 1 FROM deep_scan_runs WHERE scan_id = ?", (existing["id"],) + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (existing["id"],) ).fetchone() + if existing_run is not None: + require_legacy_deep_scan_execution(connection, existing_run) if existing_run is None: config = effective_deep_scan_config(args) workflow_version = optional_text(args.workflow_version, maximum=256) @@ -828,6 +999,7 @@ def begin_deep_scan_for_target( terminal["id"], start_disposition="joined", ) + require_legacy_deep_scan_creation(connection) config = effective_deep_scan_config(args) workflow_version = optional_text(args.workflow_version, maximum=256) if workflow_version is None: @@ -924,9 +1096,14 @@ def begin_deep_scan_for_target( def begin_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]: + workflow_version = optional_text(args.workflow_version, maximum=256) + if workflow_version not in SUPPORTED_DEEP_SCAN_WORKFLOWS: + raise SystemExit("This Deep Scan uses an unsupported workflow version.") thread_id = optional_text(args.thread_id, maximum=512) if thread_id is None: raise SystemExit("thread-id is required.") + if workflow_version == "deep-security-scan/v2" and not args.scan_id: + raise SystemExit("This Deep Scan requires a newer version to start this workflow.") if args.scan_id: if args.user_context is not None or args.user_context_stdin or args.scope != ".": raise SystemExit("scan-id cannot be combined with target setup fields.") @@ -979,6 +1156,7 @@ def coordinator_lease_is_live( def require_current_coordinator(run: sqlite3.Row, args: argparse.Namespace) -> None: + require_supported_deep_scan(run) generation = getattr(args, "coordinator_generation", None) if run["coordinator_generation"] == 1: if generation is not None: @@ -1022,10 +1200,19 @@ def claim_deep_scan_coordinator_locked( } else: adopted = run["coordinator_generation"] > 1 or run["phase"] != "setup" - if adopted: - recover_expired_coordinator(connection, run, timestamp) disposition = "adopted" if adopted else "claimed" + if deep_scan_finalization_input(run) is None: + saved = recorded_deep_scan_execution_settings(run) + if saved is not None: + validate_deep_scan_execution_settings(saved) + elif run["workflow_version"] == "deep-security-scan/v2": + raise SystemExit( + "This Deep Scan has no recorded original execution settings; " + "its executable and Codex home cannot be recovered." + ) + if disposition == "adopted": + recover_expired_coordinator(connection, run, timestamp) connection.execute( """ UPDATE deep_scan_runs @@ -1047,6 +1234,9 @@ def claim_deep_scan_coordinator_locked( def recover_expired_coordinator( connection: sqlite3.Connection, run: sqlite3.Row, timestamp: str ) -> None: + require_supported_deep_scan(run) + if deep_scan_finalization_input(run) is not None: + return scan_id = run["scan_id"] recover_candidate_ledger_publication(connection, scan_id) legacy_generation = int(run["coordinator_generation"] == 1) @@ -1169,10 +1359,25 @@ def require_deep_scan_worker(connection: sqlite3.Connection, worker_id: str) -> return row +def require_legacy_deep_scan_execution(connection: sqlite3.Connection, run: sqlite3.Row) -> None: + require_supported_deep_scan(run) + if deep_scan_finalization_input(run) is not None: + return + if ( + run["workflow_version"] == "deep-security-scan/v2" + or connection.execute( + "SELECT 1 FROM deep_scan_attempts WHERE scan_id = ? LIMIT 1", (run["scan_id"],) + ).fetchone() + is not None + ): + raise SystemExit("This Deep Scan requires a newer version to resume execution.") + + def require_running_deep_scan( connection: sqlite3.Connection, scan_id: str ) -> tuple[sqlite3.Row, sqlite3.Row]: run = require_deep_scan_run(connection, scan_id) + require_legacy_deep_scan_execution(connection, run) scan = require_scan(connection, run["scan_id"]) if run["status"] != "running" or run["cancel_requested"]: raise SystemExit("Only a running Deep Scan can update orchestration state.") @@ -1193,6 +1398,31 @@ def require_worker_transition(current: str, requested: str) -> None: raise SystemExit(f"Deep Scan worker cannot transition from {current} to {requested}.") +def validate_accepted_checkpoint(scan: sqlite3.Row, worker: sqlite3.Row) -> None: + source = deep_scan_path( + scan, worker["result_manifest_path"], "Accepted worker result", kind="file" + ) + semantic = json.loads(Path(source).read_bytes()) + if isinstance(semantic, dict): + semantic.pop("handoffClaimToken", None) + scan_dir = Path(scan["scan_dir"]) + head = ( + _worker_checkpoint_head( + scan_dir, Path(worker["artifact_dir"]).relative_to(scan_dir).as_posix(), scan["id"] + ) + if worker["kind"] == "discovery" + else None + ) + if head: + checkpoint = deep_scan_path( + scan, str(scan_dir / head), "Accepted worker checkpoint", kind="file" + ) + if json.loads(Path(checkpoint).read_bytes()) != semantic: + raise SystemExit( + "The accepted worker result does not match its current checkpoint head." + ) + + def upsert_deep_scan_worker( connection: sqlite3.Connection, args: argparse.Namespace ) -> dict[str, Any]: @@ -1236,11 +1466,14 @@ def upsert_deep_scan_worker( scan, args.artifact_dir, "Worker artifact directory", kind="directory" ) result_manifest_path = ( - deep_scan_path( - scan, - args.result_manifest_path, - "Worker result manifest path", - kind="file", + ( + deep_scan_output_path( + scan, args.result_manifest_path, "Worker result manifest path" + ) + if terminal_repeat + else deep_scan_path( + scan, args.result_manifest_path, "Worker result manifest path", kind="file" + ) ) if args.result_manifest_path else None @@ -1299,8 +1532,9 @@ def upsert_deep_scan_worker( timestamp, ), ) + result = deep_scan_result(connection, scan_id) connection.commit() - return deep_scan_result(connection, scan_id) + return result if existing["scan_id"] != scan_id or existing["kind"] != args.kind: raise SystemExit("Deep Scan worker identity does not match its persisted run and kind.") @@ -1323,8 +1557,19 @@ def upsert_deep_scan_worker( and repeated_error != existing["error_message"] ): raise SystemExit("Deep Scan worker terminal state is immutable.") + receipt = connection.execute( + "SELECT receipt_json FROM deep_scan_attempts WHERE worker_id = ? AND attempt = ?", + (worker_id, existing["attempt"]), + ).fetchone() + result = deep_scan_result(connection, scan_id) + if receipt is not None and receipt["receipt_json"]: + result["deepScan"]["workerReceipt"] = json.loads(receipt["receipt_json"]) + else: + result["deepScan"]["workerReceipt"] = next( + worker for worker in result["deepScan"]["workers"] if worker["id"] == worker_id + ) connection.commit() - return deep_scan_result(connection, scan_id) + return result attempt = args.attempt if args.attempt is not None else existing["attempt"] if attempt < existing["attempt"]: raise SystemExit("Deep Scan worker attempt cannot decrease.") @@ -1420,11 +1665,19 @@ def upsert_deep_scan_worker( worker_id, ), ) + if args.status == "succeeded" and result_manifest_path is not None: + validate_accepted_checkpoint(scan, require_deep_scan_worker(connection, worker_id)) + result = deep_scan_result(connection, scan_id) + if args.status in {"succeeded", "failed", "canceled"}: + receipt = next( + worker for worker in result["deepScan"]["workers"] if worker["id"] == worker_id + ) + result["deepScan"]["workerReceipt"] = receipt connection.commit() except BaseException: connection.rollback() raise - return deep_scan_result(connection, scan_id) + return result def claim_deep_scan_dedup( @@ -1437,7 +1690,8 @@ def claim_deep_scan_dedup( raise SystemExit("Dedup input worker IDs must be unique.") connection.execute("BEGIN IMMEDIATE") try: - run, scan = require_running_deep_scan(connection, scan_id) + run = require_deep_scan_run(connection, scan_id) + scan = require_scan(connection, scan_id) require_current_coordinator(run, args) prompt_path = deep_scan_path(scan, args.prompt_path, "Dedup prompt path", kind="file") artifact_dir = deep_scan_path( @@ -1466,9 +1720,11 @@ def claim_deep_scan_dedup( and existing["artifact_dir"] == artifact_dir and persisted_inputs == input_ids ): + result = deep_scan_result(connection, scan_id) connection.commit() - return deep_scan_result(connection, scan_id) + return result raise SystemExit("Dedup worker ID is already used by a different reducer claim.") + require_running_deep_scan(connection, scan_id) active_reducer = connection.execute( """ SELECT 1 FROM deep_scan_workers @@ -1565,11 +1821,12 @@ def claim_deep_scan_dedup( """, (timestamp, scan_id), ) + result = deep_scan_result(connection, scan_id) connection.commit() except BaseException: connection.rollback() raise - return deep_scan_result(connection, scan_id) + return result def commit_deep_scan_dedup( @@ -1595,8 +1852,14 @@ def commit_deep_scan_dedup_locked( if worker["scan_id"] != scan_id or worker["kind"] != "dedup": raise SystemExit("Dedup worker does not belong to this Deep Scan.") if worker["status"] == "succeeded": + receipt = connection.execute( + "SELECT receipt_json FROM deep_scan_merge_claims WHERE worker_id = ?", (worker_id,) + ).fetchone() + result = deep_scan_result(connection, scan_id) + if receipt is not None and receipt["receipt_json"]: + result["deepScan"]["committedMerge"] = json.loads(receipt["receipt_json"]) connection.commit() - return deep_scan_result(connection, scan_id) + return result require_running_deep_scan(connection, scan_id) if worker["status"] not in {"queued", "running"}: raise SystemExit("Only an active dedup worker can commit a result.") @@ -1642,6 +1905,23 @@ def commit_deep_scan_dedup_locked( ) if not inputs or any(row["merge_state"] != "merging" for row in inputs): raise SystemExit("Dedup inputs are not in the claimed merging state.") + claim = connection.execute( + "SELECT * FROM deep_scan_merge_claims WHERE worker_id = ?", (worker_id,) + ).fetchone() + references = [ + (row["result_manifest_path"], row["result_manifest_sha256"]) + for row in connection.execute( + "SELECT * FROM deep_scan_dedup_inputs WHERE dedup_worker_id = ? ORDER BY input_order", + (worker_id,), + ) + ] + if claim is not None and claim["previous_result_path"]: + references.append((claim["previous_result_path"], claim["previous_result_sha256"])) + for path, digest in references: + if path is not None and digest is not None: + safe_path = deep_scan_path(scan, path, "Claimed reducer input", kind="file") + if hashlib.sha256(Path(safe_path).read_bytes()).hexdigest() != digest: + raise SystemExit("A claimed Deep Scan reducer input changed after acceptance.") if candidate_ledger_path and canonical_candidate_ledger_path: canonical_path = Path(canonical_candidate_ledger_path) publication_copy = canonical_path.with_name( @@ -1685,6 +1965,9 @@ def commit_deep_scan_dedup_locked( """, (no_new_streak, timestamp, scan_id), ) + committed_worker = require_deep_scan_worker(connection, worker_id) + validate_accepted_checkpoint(scan, committed_worker) + result = deep_scan_result(connection, scan_id) connection.commit() except BaseException: connection.rollback() @@ -1697,10 +1980,20 @@ def commit_deep_scan_dedup_locked( finish_staged_file(promotion) if publication_copy is not None: publication_copy.unlink(missing_ok=True) - return deep_scan_result(connection, scan_id) + return result + +def finish_deep_scan( + connection: sqlite3.Connection, args: argparse.Namespace, select_finalization: bool = False +) -> dict[str, Any]: + if select_finalization: + import sys -def finish_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]: + args = argparse.Namespace( + **vars(args), + select_finalization=True, + finalization_result_path=json.load(sys.stdin)["resultPath"], + ) scan_id = require_uuid(args.scan_id, "scan-id") with scan_completion_lock(scan_id): return finish_deep_scan_locked(connection, args, scan_id) @@ -1714,15 +2007,28 @@ def finish_deep_scan_locked( ] if len(set(omitted_worker_ids)) != len(omitted_worker_ids): raise SystemExit("Omitted Deep Scan worker IDs must be unique.") + selecting = getattr(args, "select_finalization", False) promotion: tuple[Path, Path, Path | None] | None = None connection.execute("BEGIN IMMEDIATE") try: run = require_deep_scan_run(connection, scan_id) require_current_coordinator(run, args) scan = require_scan(connection, scan_id) + if selecting and run["workflow_version"] != "deep-security-scan/v2": + raise SystemExit("Selected finalization requires the supported v2 workflow.") + finalization = deep_scan_finalization_input(run) + if finalization is not None and ( + args.terminal_reason != finalization["terminalReason"] + or omitted_worker_ids != finalization["omittedWorkerIds"] + ): + raise SystemExit( + "Deep Scan finalization must retain its selected reason and omissions." + ) + if selecting and finalization is None: + raise SystemExit("This Deep Scan requires a newer version to select finalization.") manifest_path = ( deep_scan_output_path(scan, args.manifest_path, "Deep Scan coordinator manifest path") - if args.staged_manifest_path + if args.staged_manifest_path or selecting else deep_scan_path( scan, args.manifest_path, "Deep Scan coordinator manifest path", kind="file" ) @@ -1732,6 +2038,7 @@ def finish_deep_scan_locked( failure_capped = False if ( standard_scan_manifest + and not selecting and args.terminal_reason == "capped" and (run["status"] == "running" or omitted_worker_ids) ): @@ -1838,7 +2145,10 @@ def finish_deep_scan_locked( "Deep Scan cannot finish capped before reaching its configured maximum." ) canonical_artifacts = None - if standard_scan_manifest: + if selecting: + if not standard_scan_manifest: + raise SystemExit("Selected Deep Scan finalization requires the parent manifest.") + elif standard_scan_manifest: for artifact_name in ("scan-manifest.json", "findings.json", "coverage.json"): deep_scan_path( scan, @@ -1942,6 +2252,9 @@ def finish_deep_scan_locked( f"Deep Scan {args.terminal_reason} completion must exactly identify all buffered discovery " "workers with --omitted-worker-id." ) + if selecting: + connection.commit() + return deep_scan_result(connection, scan_id) if args.staged_manifest_path: staged_manifest_path = deep_scan_path( scan, diff --git a/plugins/codex-security/scripts/report_projection.py b/plugins/codex-security/scripts/report_projection.py index 98b4f00d1..a4e363857 100644 --- a/plugins/codex-security/scripts/report_projection.py +++ b/plugins/codex-security/scripts/report_projection.py @@ -513,7 +513,14 @@ def _target_scope_lines(target: dict[str, Any]) -> list[str]: def _surface_notes(surface: dict[str, Any]) -> str: - notes = surface.get("notes", "No additional canonical notes were recorded.") + notes = surface.get( + "notes", surface.get("reason", "No additional canonical notes were recorded.") + ) + if "notes" in surface and "reason" in surface and surface["reason"] != notes: + notes += f" {surface['reason']}" + source = _coverage_source(surface) + if source: + notes = f"{source}. {notes}" receipt_refs = surface.get("receiptRefs", []) if not isinstance(receipt_refs, list) or not receipt_refs: return _cell(notes) @@ -523,6 +530,51 @@ def _surface_notes(surface: dict[str, Any]) -> str: return _cell(f"{notes} Evidence: {evidence}") +def _coverage_source(item: dict[str, Any]) -> str: + provenance = item.get("provenance", {}) + if not isinstance(provenance, dict) or not provenance.get("workerId"): + return "" + source = f"Review {provenance['workerId']}" + if provenance.get("attempt") is not None: + source += f", attempt {provenance['attempt']}" + return source + + +def _remediation_section(finding: dict[str, Any]) -> list[str]: + remediation = _text(finding.get("remediation"), "No canonical remediation was recorded.") + lines = ["", "#### Remediation", "", remediation] + seen = {remediation} + sources = finding.get("provenance", {}).get("sourceFindings", []) + originals = ( + [ + source + for source in sources + if isinstance(source, dict) and isinstance(source.get("finding"), dict) + ] + if isinstance(sources, list) + else [] + ) + for source in originals: + text = _text(source["finding"].get("remediation"), "") + if text and text not in seen: + seen.add(text) + lines.extend(["", f"Source {_text(source.get('id'), 'finding')}: {text}"]) + for field, label in ( + ("remediationTests", "Tests"), + ("preventiveControls", "Preventive controls"), + ): + values = list( + dict.fromkeys( + value + for original in [finding, *(source["finding"] for source in originals)] + for value in _strings(original.get(field)) + ) + ) + if values: + lines.extend(["", f"{label}:", *_bullets(values, "None recorded.")]) + return lines + + def _finding_section(number: int, finding: dict[str, Any]) -> list[str]: validation = finding.get("validation") if isinstance(finding.get("validation"), dict) else {} _, raw_root_cause = merged_root_cause(finding) @@ -616,8 +668,6 @@ def _finding_section(number: int, finding: dict[str, Any]) -> list[str]: severity.get("changeConditions"), "Additional runtime or deployment evidence could raise or lower this severity.", ) - remediation_tests = _strings(finding.get("remediationTests")) - preventive_controls = _strings(finding.get("preventiveControls")) attack_steps = _strings(attack_path.get("steps")) cwes = ", ".join(finding["taxonomy"]["cwe"]) or "none" title = _text(finding["title"], "Untitled finding") @@ -736,18 +786,7 @@ def _finding_section(number: int, finding: dict[str, Any]) -> list[str]: lines.extend( ["", f"{label} assessment:", *(f"- **{name}:** {value}" for name, value in details)] ) - lines.extend( - [ - "", - "#### Remediation", - "", - _text(finding["remediation"], "No canonical remediation was recorded."), - ] - ) - if remediation_tests: - lines.extend(["", "Tests:", *_bullets(remediation_tests, "No tests recorded.")]) - if preventive_controls: - lines.extend(["", "Preventive controls:", *_bullets(preventive_controls, "None recorded.")]) + lines.extend(_remediation_section(finding)) return lines @@ -769,8 +808,12 @@ def _linked_finding_section(number: int, finding: dict[str, Any], report_path: s f"| CWE | {_cell(cwes)} |", f"| Affected lines | {_cell(_locations(finding))} |", ] - for heading in ("Summary", "Validation", "Dataflow", "Reachability", "Severity", "Remediation"): + for heading in ("Summary", "Validation", "Dataflow", "Reachability", "Severity"): lines.extend(["", f"#### {heading}", "", f"See the {link}."]) + if finding.get("provenance", {}).get("sourceFindings"): + lines.extend(_remediation_section(finding)) + else: + lines.extend(["", "#### Remediation", "", f"See the {link}."]) return lines @@ -1007,6 +1050,22 @@ def build_report_markdown( f"[Open the structural hardening portfolio]({hardening_portfolio_path})", ] ) + reviews = coverage.get("reviews", []) + if reviews: + lines.extend( + [ + "", + "## Source Review Coverage", + "", + "| Review | Attempt | Coverage |", + "| --- | --- | --- |", + ] + ) + for review in reviews: + if isinstance(review, dict): + lines.append( + f"| {_cell(review.get('workerId'))} | {_cell(str(review.get('attempt', 'unknown')))} | {_cell(review.get('completeness'))} |" + ) surfaces = coverage.get("surfaces", []) if surfaces: lines.extend( @@ -1044,6 +1103,7 @@ def build_report_markdown( questions.extend( { "question": item.get("reason", "Deferred review requires follow-up."), + "provenance": item.get("provenance", {}), "followUpPrompt": " ".join( ( f"Review deferred unit {item.get('id', 'unknown')} and close its stated proof gap.", @@ -1065,6 +1125,9 @@ def build_report_markdown( if not isinstance(question, dict): continue lines.append(f"- {_text(question.get('question'), 'Unspecified open question.')}") + source = _coverage_source(question) + if source: + lines.append(f" - {_text(source, '')}.") prompt = _text(question.get("followUpPrompt"), "") if prompt: lines.append(f" - Follow-up prompt: {prompt}") diff --git a/plugins/codex-security/scripts/workbench_db.py b/plugins/codex-security/scripts/workbench_db.py index 282cd0c76..cc58bf6a5 100644 --- a/plugins/codex-security/scripts/workbench_db.py +++ b/plugins/codex-security/scripts/workbench_db.py @@ -56,7 +56,6 @@ finalize_scan, finding_candidate_id, open_scan_local_file_descriptor, - write_scan_local_bytes, ) from finding_preview import bounded_finding_details from workbench import handoff @@ -137,6 +136,7 @@ from workbench_validation import ( bounded_output_text, optional_text, + parse_budget_scan_cost, parse_scan_cost, path_within_scope, reject_non_finite_json, @@ -1155,18 +1155,17 @@ def complete_budget_exhausted_scan( connection: sqlite3.Connection, args: argparse.Namespace ) -> dict[str, Any]: scan_id = require_uuid(args.scan_id, "scan-id") - cost_json = parse_scan_cost(args.cost_json) - if cost_json is None: - raise SystemExit("Budget-exhausted scan completion requires the measured scan cost.") + cost_json, measured = parse_budget_scan_cost(args.cost_json) with scan_completion_lock(scan_id): scan = require_scan(connection, scan_id) if scan["status"] != "running" or scan["mode"] != "deep" or scan["recipe_json"] is None: raise SystemExit("Only a running CLI Deep Scan can complete after its cost limit.") + handoff.require_current_continuation( + scan, None, error_message="Scan completion is owned by another continuation." + ) recipe = json.loads(scan["recipe_json"], parse_constant=reject_non_finite_json) if not isinstance(recipe, dict) or recipe.get("mode") != "deep": raise SystemExit("Budget-exhausted scan completion requires a Deep Scan launch recipe.") - cost = json.loads(cost_json) - measured = cost.get("cost", cost) limit = recipe.get("maxCostUsd") if ( not isinstance(limit, (int, float)) @@ -1175,10 +1174,7 @@ def complete_budget_exhausted_scan( or measured.get("estimatedUsd", 0) <= limit ): raise SystemExit("Deep Scan has not exceeded its configured cost limit.") - run = connection.execute( - "SELECT status, terminal_reason, manifest_path FROM deep_scan_runs WHERE scan_id = ?", - (scan_id,), - ).fetchone() + run = deep_scan.find_supported_deep_scan_run(connection, scan_id) if ( run is None or run["status"] != "succeeded" @@ -1189,19 +1185,13 @@ def complete_budget_exhausted_scan( "Budget-exhausted scan completion requires successfully completed Deep Scan " "discovery." ) - scan_dir = require_canonical_scan_directory(Path(scan["scan_dir"])) - candidates = ( - [] - if run["manifest_path"] == str(scan_dir / "scan-manifest.json") - else budget_exhausted_candidates(scan, scan_dir) - ) warning = optional_text(args.message, maximum=2400) if warning is None: warning = ( f"Deep Scan reached its cost limit after an estimated " f"${measured['estimatedUsd']:.6g}; completed discovery was preserved." ) - budget_exhausted_draft(scan, scan_dir, candidates, warning) + saved_results.prepare_budget_draft(_WORKBENCH_DB_CONTEXT, connection, scan, warning) warnings = json.loads(scan["completion_warnings_json"]) if warning not in warnings: connection.execute( @@ -1289,7 +1279,7 @@ def budget_exhausted_draft( scan_dir: Path, candidates: list[dict[str, Any]], warning: str, -) -> None: +) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]] | None: documents: dict[str, dict[str, Any]] = {} for name in ("scan-manifest.json", "findings.json", "coverage.json"): path = artifact_path(scan_dir, name, required=False) @@ -1310,7 +1300,10 @@ def budget_exhausted_draft( if not isinstance(coverage.get(key), list): raise SystemExit("Budget-exhausted scan contains invalid canonical coverage.") if manifest["scan"].get("sealedAt") is not None or manifest["scan"].get("artifacts"): - raise SystemExit("Budget-exhausted scan cannot replace an already sealed scan draft.") + saved_results.validate_sealed_budget_draft( + _WORKBENCH_DB_CONTEXT, scan, scan_dir, manifest + ) + return None else: contract = scan_contract(scan) target_contract = contract["target"] @@ -1418,19 +1411,7 @@ def budget_exhausted_draft( } ) coverage["completeness"] = "partial" - for name, payload in ( - ("findings.json", findings), - ("coverage.json", coverage), - ("scan-manifest.json", manifest), - ): - try: - write_scan_local_bytes( - scan_dir, - name, - (json.dumps(payload, allow_nan=False, indent=2, sort_keys=True) + "\n").encode(), - ) - except (ContractError, OSError, TypeError, ValueError) as exc: - raise SystemExit(f"Budget-exhausted scan draft could not be saved: {exc}") from exc + return manifest, findings, coverage def complete_scan_locked( @@ -1444,6 +1425,7 @@ def complete_scan_locked( ) -> dict[str, Any]: scan = require_scan(connection, scan_id) if scan["status"] == "complete": + deep_scan.find_supported_deep_scan_run(connection, scan_id) scan_dir = require_canonical_scan_directory(Path(scan["scan_dir"])) require_recorded_manifest_digest(scan, scan_dir) verify_manifest_binding(scan, read_json_object(scan_dir / ARTIFACTS["manifest"])) @@ -1525,8 +1507,7 @@ def add_warning() -> None: scan_dir, expected_coverage_mode=expected_coverage_mode(scan), completion_binding=completion_binding, - # Save the finished Deep result as submitted. Worker drafts and - # recovery repairs belong to the stopped-scan path. + # Preserve the finished Deep output. completion_warnings=warnings if scan["mode"] != "deep" else None, draft_documents=saved_results.merge_saved_results( scan_dir, @@ -1543,12 +1524,17 @@ def add_warning() -> None: if scan["mode"] != "deep" and current_manifest_path is not None and not already_sealed else None, ) + saved_results.require_selected_publication( + _WORKBENCH_DB_CONTEXT, connection, scan, prepared + ) add_warning() wrote = True manifest, findings, _ = _write_prepared_scan_finalization(prepared) except ContractError as exc: - if wrote or ( + # Replay a validated Deep aggregate after an output write fails. + if (wrote and scan["mode"] != "deep") or ( scan["mode"] == "deep" + and not wrote and not already_sealed and not isinstance(exc, RecoverableContractError) ): @@ -2839,8 +2825,7 @@ def scan_result( **scan_usage.stored_scan_cost_fields(scan["cost_json"]), "contract": scan_contract(scan), "continuationThreadId": scan["continuation_thread_id"], - "threadIds": scan_usage._scan_root_thread_ids(connection, scan, None), - "executionThreadIds": scan_usage._scan_execution_thread_ids(connection, scan), + **scan_usage.scan_execution_fields(connection, scan), "failureMessage": scan["failure_message"], "findings": [ finding_result(connection, scan, row, related=relations.get(row["id"], [])) @@ -3380,6 +3365,8 @@ def read_json_object(path: Path) -> dict[str, Any]: _WORKBENCH_DB_CONTEXT = saved_results.WorkbenchDbContext( ARTIFACTS=ARTIFACTS, artifact_path=artifact_path, + budget_exhausted_candidates=budget_exhausted_candidates, + budget_exhausted_draft=budget_exhausted_draft, deep_scan=deep_scan, expected_coverage_mode=expected_coverage_mode, handoff=handoff, @@ -3402,8 +3389,7 @@ def read_json_object(path: Path) -> dict[str, Any]: ) -def main() -> None: - # Workbench callers send UTF-8 even when Windows uses a legacy code page. +def main(*, select_finalization: bool = False, with_execution_settings: bool = False) -> None: sys.stdin.reconfigure(encoding="utf-8") args = parse_args(__doc__) deep_scan.configure( @@ -3477,7 +3463,7 @@ def main() -> None: elif args.command == "commit-deep-scan-dedup": result = deep_scan.commit_deep_scan_dedup(connection, args) elif args.command == "finish-deep-scan": - result = deep_scan.finish_deep_scan(connection, args) + result = deep_scan.finish_deep_scan(connection, args, select_finalization) elif args.command == "fail-deep-scan": result = deep_scan.fail_deep_scan(connection, args) elif args.command == "record-deep-scan-publication-failure": @@ -3653,6 +3639,8 @@ def main() -> None: result = list_stored_findings(connection, limit=args.limit, offset=args.offset) else: raise SystemExit(f"Unknown command: {args.command}") + if with_execution_settings: + deep_scan.include_execution_settings(connection, result) print(json.dumps(result, allow_nan=False, sort_keys=True)) diff --git a/plugins/codex-security/scripts/workbench_saved_results.py b/plugins/codex-security/scripts/workbench_saved_results.py index 10519b8e9..2d6fe4d7c 100644 --- a/plugins/codex-security/scripts/workbench_saved_results.py +++ b/plugins/codex-security/scripts/workbench_saved_results.py @@ -21,6 +21,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from finalize_scan_contract import ( ContractError, + RecoverableContractError, _finding_strength, _populate_unsealed_artifact_envelope, _populate_unsealed_manifest_envelope, @@ -59,6 +60,8 @@ class WorkbenchDbContext: ARTIFACTS: dict[str, str] artifact_path: Callable[..., Path | None] + budget_exhausted_candidates: Callable[..., list[dict[str, Any]]] + budget_exhausted_draft: Callable[..., Any] deep_scan: ModuleType expected_coverage_mode: Callable[..., str] handoff: ModuleType @@ -80,6 +83,21 @@ class WorkbenchDbContext: workspace_state: Callable[..., dict[str, Any]] +def validate_sealed_budget_draft( + db: WorkbenchDbContext, scan: sqlite3.Row, scan_dir: Path, manifest: dict[str, Any] +) -> None: + # The seal can reach disk before parent completion commits. Validate + # it without changing bytes; the existing finalizer commits replay. + try: + _prepare_scan_finalization( + scan_dir, + expected_coverage_mode=db.expected_coverage_mode(scan), + completion_binding=db.workbench_completion_binding(scan, scan["started_at"], manifest), + ) + except ContractError as exc: + raise SystemExit(str(exc)) from exc + + def _encoded(value: Any) -> bytes: return json.dumps( value, ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":") @@ -104,6 +122,35 @@ def _children(scan_dir: Path, relative: str) -> list[str]: return sorted(child.name for child in cursor.iterdir()) +def _saved_workers(connection: Any, scan_id: str) -> list[dict[str, Any]]: + rows = connection.execute( + "SELECT worker.*, attempt.accepted_result_path FROM deep_scan_workers AS worker " + "LEFT JOIN deep_scan_attempts AS attempt " + "ON attempt.worker_id = worker.id AND attempt.attempt = worker.attempt " + "WHERE worker.scan_id = ? ORDER BY worker.created_at, worker.id", + (scan_id,), + ).fetchall() + return [ + { + **dict(row), + "result_manifest_path": row["accepted_result_path"] or row["result_manifest_path"], + } + for row in rows + ] + + +def _accepted_source_digests(connection: Any, scan_id: str) -> dict[str, str]: + return { + row["accepted_result_path"]: row["accepted_result_sha256"] + for row in connection.execute( + "SELECT accepted_result_path, accepted_result_sha256 FROM deep_scan_attempts " + "WHERE scan_id = ? AND accepted_result_path IS NOT NULL " + "AND accepted_result_sha256 IS NOT NULL", + (scan_id,), + ) + } + + def _latest_successful_reducer(workers: list[Any]) -> Any | None: return max( ( @@ -161,18 +208,81 @@ def checkpoints(directory: str, kind: str | None = None) -> Iterator[tuple[str, def _read_saved_result( - scan_dir: Path, relative: str, scan_id: str, *, kind: str | None = None + scan_dir: Path, + relative: str, + scan_id: str, + *, + kind: str | None = None, + accepted_source_digests: dict[str, str] | None = None, ) -> tuple[dict[str, Any], str]: - draft = _read_scan_local_json(scan_dir, relative, "Saved scan checkpoint") + draft, contents = _read_scan_local_json_bytes(scan_dir, relative, "Saved scan checkpoint") + expected = (accepted_source_digests or {}).get(str(scan_dir / relative)) + if expected is not None and hashlib.sha256(contents).hexdigest() != expected: + raise ContractError("checkpoint changed after acceptance") if draft.get("scanId") != scan_id: raise ContractError("checkpoint belongs to a different scan") - if not isinstance(draft.get("findings"), list) or not isinstance( - draft.get("coverage", {} if kind == "dedup" else None), dict - ): + coverage = ( + draft.get("sourceCoverage", draft.get("coverage", {})) + if kind == "dedup" + else draft.get("coverage") + ) + if not isinstance(draft.get("findings"), list) or not isinstance(coverage, dict): raise ContractError("checkpoint has no semantic findings or coverage") return draft, _digest(draft) +def _worker_checkpoint_head( + scan_dir: Path, + directory: str, + scan_id: str, + accepted_source_digests: dict[str, str] | None = None, +) -> str | None: + relative = f"{directory}/checkpoint-head.json" + try: + (scan_dir / relative).lstat() + except FileNotFoundError: + return None + head = _read_scan_local_json(scan_dir, relative, "Saved worker checkpoint head") + name = head.get("checkpoint") + if not isinstance(name, str) or not re.fullmatch(r"[0-9a-f]{64}\.json", name): + raise ContractError("Saved worker checkpoint head is invalid.") + checkpoint = f"{directory}/checkpoints/{name}" + # A committed head precedes replacement of result.json. Do not fall back to + # that older result if the selected checkpoint cannot be read. + _read_saved_result( + scan_dir, checkpoint, scan_id, accepted_source_digests=accepted_source_digests + ) + return checkpoint + + +def _worker_checkpoint_heads( + scan_dir: Path, + workers: list[Any], + scan_id: str, + accepted_source_digests: dict[str, str] | None = None, +) -> dict[str, str]: + heads: dict[str, str] = {} + for worker in workers: + if worker["kind"] != "discovery": + continue + try: + output = Path(worker["artifact_dir"]).relative_to(scan_dir) + except (TypeError, ValueError): + continue + attempts = (output.parent if output.name == "output" else output) / "attempts" + directories = [output] + [ + attempts / name + for name in _children(scan_dir, attempts.as_posix()) + if re.fullmatch(r"attempt-\d+", name) + ] + for directory in directories: + relative = directory.as_posix() + head = _worker_checkpoint_head(scan_dir, relative, scan_id, accepted_source_digests) + if head is not None: + heads[relative] = head + return heads + + def _read_saved_parent_result( scan_dir: Path, scan_id: str ) -> tuple[dict[str, Any], dict[str, Any]]: @@ -215,18 +325,21 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool: try: scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) manifest_path = db.artifact_path(scan_dir, db.ARTIFACTS["manifest"], required=False) - workers = connection.execute( - "SELECT id, kind, status, completed_at, artifact_dir, result_manifest_path " - "FROM deep_scan_workers WHERE scan_id = ?", - (scan["id"],), - ).fetchall() + workers = _saved_workers(connection, scan["id"]) + accepted_digests = _accepted_source_digests(connection, scan["id"]) paths = dict(_saved_result_paths(scan_dir, workers)) frozen_sources = scan["retained_source_digests_json"] def has_saved_source() -> bool: for path in paths: try: - _read_saved_result(scan_dir, path, scan["id"], kind=paths[path]) + _read_saved_result( + scan_dir, + path, + scan["id"], + kind=paths[path], + accepted_source_digests=accepted_digests, + ) return True except (ContractError, OSError, ValueError): continue @@ -254,11 +367,21 @@ def has_saved_source() -> bool: published_sources = _source_digests( manifest_scan.get("preservedSources", {}), "Published scan" ) + if ( + "preservedCheckpointHeads" in manifest_scan + and _worker_checkpoint_heads(scan_dir, workers, scan["id"], accepted_digests) + != manifest_scan["preservedCheckpointHeads"] + ): + return True current_sources = dict(published_sources) for path in paths: try: _, current_sources[path] = _read_saved_result( - scan_dir, path, scan["id"], kind=paths[path] + scan_dir, + path, + scan["id"], + kind=paths[path], + accepted_source_digests=accepted_digests, ) except (ContractError, OSError, ValueError): continue @@ -267,11 +390,17 @@ def has_saved_source() -> bool: return False -def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[str, str], bool]: +def _recovery_source_digests( + db: Any, connection: Any, scan: Any +) -> tuple[dict[str, str], bool, dict[str, str] | None]: scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) frozen_sources: dict[str, str] | None = None include_parent = True raw_frozen_sources = scan["retained_source_digests_json"] + raw_checkpoint_heads = scan["retained_checkpoint_heads_json"] + checkpoint_heads = ( + json.loads(raw_checkpoint_heads) if raw_checkpoint_heads is not None else None + ) if raw_frozen_sources is not None: frozen_sources = _source_digests(json.loads(raw_frozen_sources), "Saved stopped-scan") include_parent = False @@ -289,6 +418,8 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[ if scan["seal_manifest_digest"] is not None or ( manifest_scan.get("sealedAt") is not None or manifest_scan.get("artifacts") is not None ): + if checkpoint_heads is None: + checkpoint_heads = manifest_scan.get("preservedCheckpointHeads") if "preservedSources" in manifest_scan: published_sources = _source_digests( manifest_scan["preservedSources"], "Published scan" @@ -305,16 +436,19 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[ else: include_parent = True - workers = connection.execute( - "SELECT id, kind, status, completed_at, artifact_dir, result_manifest_path " - "FROM deep_scan_workers WHERE scan_id = ?", - (scan["id"],), - ).fetchall() + workers = _saved_workers(connection, scan["id"]) + accepted_digests = _accepted_source_digests(connection, scan["id"]) paths = dict(_saved_result_paths(scan_dir, workers)) recovery_sources = dict(frozen_sources or {}) for relative, expected_digest in recovery_sources.items(): try: - _, digest = _read_saved_result(scan_dir, relative, scan["id"], kind=paths.get(relative)) + _, digest = _read_saved_result( + scan_dir, + relative, + scan["id"], + kind=paths.get(relative), + accepted_source_digests=accepted_digests, + ) except (ContractError, OSError, ValueError) as exc: raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc if digest != expected_digest: @@ -323,11 +457,23 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[ for relative in paths.keys() - recovery_sources.keys(): try: _, recovery_sources[relative] = _read_saved_result( - scan_dir, relative, scan["id"], kind=paths[relative] + scan_dir, + relative, + scan["id"], + kind=paths[relative], + accepted_source_digests=accepted_digests, ) except (ContractError, OSError, ValueError): continue - return recovery_sources, include_parent + if checkpoint_heads is not None and ( + recovery_sources != frozen_sources + or _worker_checkpoint_heads(scan_dir, workers, scan["id"], accepted_digests) + != checkpoint_heads + ): + raise SystemExit( + "This stopped scan requires a newer version to select recovery checkpoints." + ) + return recovery_sources, include_parent, checkpoint_heads def scan_results_recovery_needed(db: Any, connection: Any, scan: Any) -> bool: @@ -469,7 +615,9 @@ def merge_saved_results( stopped: bool, reason: str, frozen_source_digests: dict[str, str] | None = None, + checkpoint_heads: dict[str, str] | None = None, allow_frozen_legacy_parent: bool = False, + accepted_source_digests: dict[str, str] | None = None, ) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]] | None: """Read only bound parent/worker files; return an unsealed loss-preserving union.""" initial_warnings = set(warnings) @@ -519,6 +667,7 @@ def merge_saved_results( reducer_paths.add(latest_reducer) except ValueError: warnings.append("Skipped a reducer result outside the scan directory.") + accepted_reducer = latest_reducer def checkpoints(directory: str, worker_id: str | None) -> None: for name in _children(scan_dir, directory): @@ -560,22 +709,40 @@ def reducer_output(directory: str, attempt: int, reducer_worker: Any) -> None: continue if worker["kind"] != "discovery": continue + head = (checkpoint_heads or {}).get(output) + if head is not None: + paths[head] = worker["id"] + current_results.add(head) paths[f"{output}/result.json"] = worker["id"] - current_results.add(f"{output}/result.json") + if head is None: + current_results.add(f"{output}/result.json") checkpoints(f"{output}/checkpoints", worker["id"]) attempts = ( Path(output).parent if Path(output).name == "output" else Path(output) ) / "attempts" - for name in _children(scan_dir, attempts.as_posix()): - if re.fullmatch(r"attempt-\d+", name): - archived = (attempts / name).as_posix() - paths[f"{archived}/result.json"] = worker["id"] - checkpoints(f"{archived}/checkpoints", worker["id"]) + archived_attempts = sorted( + ( + name + for name in _children(scan_dir, attempts.as_posix()) + if re.fullmatch(r"attempt-\d+", name) + ), + key=lambda name: int(name.removeprefix("attempt-")), + reverse=True, + ) + for name in archived_attempts: + archived = (attempts / name).as_posix() + archived_head = (checkpoint_heads or {}).get(archived) + if archived_head is not None: + paths[archived_head] = worker["id"] + current_results.add(archived_head) + paths[f"{archived}/result.json"] = worker["id"] + checkpoints(f"{archived}/checkpoints", worker["id"]) if worker["result_manifest_path"]: try: current_path = Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix() paths[current_path] = worker["id"] - current_results.add(current_path) + if head is None: + current_results.add(current_path) except ValueError: warnings.append("Skipped a worker result outside the scan directory.") @@ -592,14 +759,23 @@ def reducer_output(directory: str, attempt: int, reducer_worker: Any) -> None: for relative, worker_id in paths.items(): try: draft, digest = _read_saved_result( - scan_dir, relative, scan_id, kind="dedup" if relative in reducer_paths else None + scan_dir, + relative, + scan_id, + kind="dedup" if relative in reducer_paths else None, + accepted_source_digests=accepted_source_digests, ) if frozen_source_digests is not None and frozen_source_digests[relative] != digest: raise ContractError("checkpoint changed after the scan stopped") source_digests[relative] = digest - # Recovery expects coverage, but reducer results only contain findings - # and context. Add an empty value after hashing the original result. - sources.append((relative, {"coverage": {}, **draft}, worker_id)) + # The host supplies reducer coverage separately from model output. + # Preserve the digest of the original accepted document. + if relative in reducer_paths: + draft = { + **draft, + "coverage": draft.get("sourceCoverage", draft.get("coverage", {})), + } + sources.append((relative, draft, worker_id)) except (ContractError, OSError, ValueError) as exc: if (scan_dir / relative).exists(): warnings.append(f"Preserved unreadable checkpoint {relative}: {exc}") @@ -608,6 +784,15 @@ def reducer_output(directory: str, attempt: int, reducer_worker: Any) -> None: raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") drafts_by_path = {relative: draft for relative, draft, _ in sources} + if "sourceCoverage" not in drafts_by_path.get(accepted_reducer, {}): + accepted_reducer = None + accepted_coverage = drafts_by_path.get(accepted_reducer, {}).get("coverage", {}) + reviewed_attempts = { + (review.get("workerId"), review.get("attempt")) + for review in accepted_coverage.get("reviews", []) + if isinstance(review, dict) + } + workers_by_id = {worker["id"]: worker for worker in workers} latest_reducer_key = ( (reducer["completed_at"] or "", reducer["id"], int(reducer["attempt"] or 0)) if reducer is not None and latest_reducer in drafts_by_path @@ -637,6 +822,7 @@ def reducer_output(directory: str, attempt: int, reducer_worker: Any) -> None: and parent_manifest["scan"].get("sealedAt") and parent_manifest["scan"].get("status") == binding["status"] and parent_manifest["scan"].get("preservedSources") == source_digests + and parent_manifest["scan"].get("preservedCheckpointHeads") == checkpoint_heads and all(warning in initial_warnings for warning in warnings) ): return None @@ -668,6 +854,8 @@ def reducer_output(directory: str, attempt: int, reducer_worker: Any) -> None: for key in ("sealedAt", "artifacts"): manifest["scan"].pop(key, None) manifest["scan"]["preservedSources"] = source_digests + if checkpoint_heads is not None: + manifest["scan"]["preservedCheckpointHeads"] = checkpoint_heads coverage = ( copy.deepcopy(parent["coverage"]) if parent and parent["coverage"] @@ -722,8 +910,19 @@ def valid_finding(value: Any) -> bool: all_sources = ([("parent", parent, None)] if parent else []) + sources current_drafts = ([(None, parent)] if parent else []) + [ - (worker_id, draft) for relative, draft, worker_id in sources if relative in current_results + (worker_id, draft) + for relative, draft, worker_id in sources + if relative in current_results or relative == accepted_reducer ] + + def coverage_candidate(owner: str | None, item: dict[str, Any]) -> tuple[str | None, Any]: + provenance = item.get("provenance") + if owner is None and isinstance(provenance, dict): + return provenance.get("workerId"), provenance.get( + "candidateId", item.get("candidateId") + ) + return owner, item.get("candidateId") + resolved: dict[tuple[str | None, str], str] = {} for owner, draft in current_drafts: for finding in draft["findings"]: @@ -741,7 +940,7 @@ def valid_finding(value: Any) -> bool: and isinstance(item.get("candidateId"), str) and item.get("disposition") in {"reported", "rejected", "not_applicable"} ): - resolved.setdefault((owner, item["candidateId"]), item["disposition"]) + resolved.setdefault(coverage_candidate(owner, item), item["disposition"]) # Only the current parent may claim that another worker finding was absorbed. # A superseded checkpoint must not suppress a newer independent result. for draft in [parent] if parent else []: @@ -955,11 +1154,18 @@ def valid_finding(value: Any) -> bool: continue finding_positions[key] = len(findings) findings.append(finding) - if superseded: + worker = workers_by_id.get(worker_id) + reviewed = ( + worker is not None + and worker["status"] == "succeeded" + and worker["merge_state"] == "merged" + and (worker_id, worker["attempt"]) in reviewed_attempts + ) + if reviewed or (superseded and relative != accepted_reducer): continue - for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"): + for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions", "reviews"): items = draft["coverage"].get(field, []) - if not isinstance(items, list): + if not isinstance(items, list) or (field == "reviews" and not items): continue output = coverage.setdefault(field, []) if not isinstance(output, list): @@ -990,7 +1196,7 @@ def valid_finding(value: Any) -> bool: history.append(copy.deepcopy(finding)) if ( isinstance(item, dict) - and (worker_id, item.get("candidateId")) in resolved + and coverage_candidate(worker_id, item) in resolved and (field == "deferred" or item.get("disposition") == "needs_follow_up") ): continue @@ -1100,6 +1306,7 @@ def preserve_scan_results_locked( scan_id: str, *, recovery_source_digests: dict[str, str] | None = None, + recovery_checkpoint_heads: dict[str, str] | None = None, include_parent_with_recovery: bool = False, ) -> bool: """Publish or verify retained terminal results through the workbench host.""" @@ -1107,6 +1314,7 @@ def preserve_scan_results_locked( if scan["status"] != "failed": return False frozen_source_digests: dict[str, str] | None = None + checkpoint_heads = recovery_checkpoint_heads raw_frozen_sources = scan["retained_source_digests_json"] if recovery_source_digests is not None: frozen_source_digests = recovery_source_digests @@ -1114,10 +1322,16 @@ def preserve_scan_results_locked( frozen_source_digests = _source_digests( json.loads(raw_frozen_sources), "Saved stopped-scan" ) + raw_checkpoint_heads = scan["retained_checkpoint_heads_json"] + checkpoint_heads = ( + json.loads(raw_checkpoint_heads) if raw_checkpoint_heads is not None else None + ) scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) deep_run = connection.execute( - "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) ).fetchone() + if deep_run is not None: + db.deep_scan.require_supported_deep_scan(deep_run) outcome = ( "canceled" if scan["canceled_at"] @@ -1198,6 +1412,9 @@ def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> No db.verify_manifest_binding(scan, existing) if existing_scan.get("status") == outcome: existing_sources = existing_scan.get("preservedSources") + existing_heads = existing_scan.get("preservedCheckpointHeads") + if checkpoint_heads is None: + checkpoint_heads = existing_heads if frozen_source_digests is None: if not isinstance(existing_sources, dict) or not all( isinstance(relative, str) and isinstance(digest, str) @@ -1205,7 +1422,8 @@ def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> No ): raise ContractError("Stopped scan source digests could not be frozen.") frozen_source_digests = existing_sources - if existing_sources == frozen_source_digests: + checkpoint_heads = existing_heads + if existing_sources == frozen_source_digests and existing_heads == checkpoint_heads: if ( raw_frozen_sources is not None and scan["seal_manifest_digest"] is not None @@ -1221,10 +1439,7 @@ def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> No scan_dir, scan_id, binding, - connection.execute( - "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id", - (scan_id,), - ).fetchall(), + _saved_workers(connection, scan_id), warnings, stopped=True, reason=( @@ -1232,6 +1447,8 @@ def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> No f"{scan['failure_message'] or ''}" ).strip(), frozen_source_digests=frozen_source_digests, + checkpoint_heads=checkpoint_heads, + accepted_source_digests=_accepted_source_digests(connection, scan_id), allow_frozen_legacy_parent=( include_parent_with_recovery or ( @@ -1264,7 +1481,10 @@ def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> No connection.execute( "UPDATE scans SET retained_source_digests_json = ? " "WHERE id = ? AND retained_source_digests_json IS NULL", - (json.dumps(retained_sources, sort_keys=True), scan_id), + ( + json.dumps(retained_sources, sort_keys=True), + scan_id, + ), ) prepared = _prepare_scan_finalization( scan_dir, @@ -1292,12 +1512,15 @@ def recover_scan_results(db: Any, connection: Any, args: Any) -> dict[str, Any]: raise SystemExit("Only a stopped scan can recover terminal results.") if scan["canceled_at"] is not None: raise SystemExit("Canceled scans cannot recover terminal results.") - recovery_source_digests, include_parent = _recovery_source_digests(db, connection, scan) + recovery_source_digests, include_parent, checkpoint_heads = _recovery_source_digests( + db, connection, scan + ) if not preserve_scan_results_locked( db, connection, scan_id, recovery_source_digests=recovery_source_digests, + recovery_checkpoint_heads=checkpoint_heads, include_parent_with_recovery=include_parent, ): raise SystemExit("No saved stopped-scan results were available to recover.") @@ -1382,6 +1605,310 @@ def save_scan_artifact(db: Any, connection: Any, args: Any) -> dict[str, Any]: return {"scanId": scan_id, "path": str(scan_dir / output)} +def _read_staged_scan_draft(scan_dir: Path, draft_path: str) -> dict[str, Any]: + try: + relative = Path(draft_path).relative_to(scan_dir).as_posix() + except ValueError as exc: + raise SystemExit("Scan draft must be inside the registered scan drafts directory.") from exc + if not re.fullmatch(r"drafts/[0-9a-fA-F-]+\.json", relative): + raise SystemExit("Scan draft must be inside the registered scan drafts directory.") + return _read_scan_local_json(scan_dir, relative, "Staged scan draft") + + +def _selected_publication_digest(prepared: Any) -> str: + # Completion time is chosen at sealing, after publication. Everything else + # must remain the host projection of the same accepted aggregate. + manifest = copy.deepcopy(prepared[2]) + for field in ("completedAt", "sealedAt"): + manifest["scan"].pop(field, None) + return _digest([manifest, prepared[3], prepared[4]]) + + +def _require_selected_result(scan: Any, selection: dict[str, Any]) -> None: + relative = selection["resultPath"] + if relative is not None: + _read_saved_result( + Path(scan["scan_dir"]), + relative, + scan["id"], + kind="dedup", + accepted_source_digests={ + str(Path(scan["scan_dir"]) / relative): selection["resultSha256"] + }, + ) + + +def record_selected_publication(db: Any, connection: Any, scan: Any, documents: Any) -> None: + run = db.deep_scan.require_deep_scan_run(connection, scan["id"]) + selection = db.deep_scan.deep_scan_finalization_input(run) + if selection is None: + return + _require_selected_result(scan, selection) + prepared = _prepare_scan_finalization( + Path(scan["scan_dir"]), + expected_coverage_mode=db.expected_coverage_mode(scan), + completion_binding=db.workbench_completion_binding(scan, db.now(), documents[0]), + draft_documents=documents, + ) + selection["publicationSha256"] = _selected_publication_digest(prepared) + with connection: + connection.execute( + "UPDATE deep_scan_runs SET finalization_input_json = ? WHERE scan_id = ?", + (json.dumps(selection), scan["id"]), + ) + + +def retain_unmerged_budget_coverage( + scan: Any, scan_dir: Path, coverage: dict[str, Any], worker: Any +) -> None: + """Keep each unmerged review's obligations; its findings remain evidence only.""" + accepted_digests = _source_digests( + {worker["accepted_result_path"]: worker["accepted_result_sha256"]}, "Accepted budget" + ) + relative = Path(worker["accepted_result_path"]).relative_to(scan_dir).as_posix() + draft, _ = _read_saved_result( + scan_dir, + relative, + scan["id"], + accepted_source_digests=accepted_digests, + ) + head = _worker_checkpoint_head( + scan_dir, Path(worker["artifact_dir"]).relative_to(scan_dir).as_posix(), scan["id"] + ) + if head is not None and _read_saved_result(scan_dir, head, scan["id"])[0] != draft: + raise ContractError("The accepted discovery differs from its current checkpoint head.") + source = draft["coverage"] + provenance = {"workerId": worker["id"], "attempt": worker["attempt"]} + prefix = f"{worker['id']}-attempt-{worker['attempt']}" + artifact_prefix = Path(worker["artifact_dir"]).relative_to(scan_dir).as_posix() + surfaces = { + item.get("id"): f"{prefix}-surface-{index + 1}" + for index, item in enumerate(source.get("surfaces", [])) + } + + def retain(field: str, item: dict[str, Any]) -> None: + # A committed budget draft can be replayed before the scan is sealed. + # These IDs and provenance identify the same immutable accepted review. + items = coverage.setdefault(field, []) + if "id" in item: + matches = [] + for index, existing in enumerate(items): + existing_provenance = ( + existing.get("provenance") if isinstance(existing, dict) else None + ) + if ( + isinstance(existing_provenance, dict) + and existing.get("id") == item["id"] + and all( + existing_provenance.get(key) == value for key, value in provenance.items() + ) + ): + previous = copy.deepcopy(existing) + previous["provenance"] = {**item["provenance"], **existing_provenance} + if previous != item: + raise ContractError( + "Legacy budget coverage changed from its accepted review." + ) + matches.append(index) + if matches: + # Refresh older projections from the same accepted bytes, even + # if an interrupted writer saved more than one copy. + items[matches[0]] = item + for index in reversed(matches[1:]): + del items[index] + return + if item not in items: + items.append(item) + + for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"): + for index, original in enumerate(source.get(field, [])): + item = copy.deepcopy(original if isinstance(original, dict) else {"question": original}) + source_provenance = item.get("provenance") + if not isinstance(source_provenance, dict): + source_provenance = {} + # Keep source descriptions; the accepted owner supplies identity. + for key in ("workerId", "attempt", "sourceId", "candidateId"): + source_provenance.pop(key, None) + item["provenance"] = { + **source_provenance, + **provenance, + **({"sourceId": item["id"]} if "id" in item else {}), + **({"candidateId": item["candidateId"]} if "candidateId" in item else {}), + } + item["id"] = f"{prefix}-{field}-{index + 1}" + if field == "surfaces": + item["id"] = f"{prefix}-surface-{index + 1}" + item["receiptRefs"] = [ + f"{artifact_prefix}/{ref}" for ref in item.get("receiptRefs", []) + ] + if field == "deferred" and "candidateId" in item: + item["candidateId"] = f"{prefix}-candidate-{index + 1}" + if "surfaceIds" in item: + item["surfaceIds"] = [surfaces.get(value, value) for value in item["surfaceIds"]] + retain(field, item) + retain("reviews", {**provenance, "completeness": source["completeness"]}) + for index, limitation in enumerate(draft.get("scope", {}).get("limitations", [])): + retain( + "deferred", + { + "id": f"{prefix}-scope-{index + 1}", + "reason": limitation, + "provenance": provenance, + }, + ) + retain( + "deferred", + { + "id": f"{prefix}-unmerged", + "provenance": provenance, + "reason": "This accepted discovery was not merged before the scan reached its cost limit.", + }, + ) + + +def prepare_budget_draft(db: Any, connection: Any, scan: Any, warning: str) -> None: + scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) + run = db.deep_scan.require_deep_scan_run(connection, scan["id"]) + selection = db.deep_scan.deep_scan_finalization_input(run) + has_publication = selection is not None and "publicationSha256" in selection + candidates = ( + [] + if run["manifest_path"] == str(scan_dir / "scan-manifest.json") + else db.budget_exhausted_candidates(scan, scan_dir) + ) + try: + if has_publication: + documents = tuple( + _read_scan_local_json(scan_dir, name, name) + for name in ("scan-manifest.json", "findings.json", "coverage.json") + ) + prepared = _prepare_scan_finalization( + scan_dir, + expected_coverage_mode=db.expected_coverage_mode(scan), + completion_binding=db.workbench_completion_binding(scan, db.now(), documents[0]), + draft_documents=documents, + ) + manifest = copy.deepcopy(documents[0]) + manifest["scan"].setdefault("id", scan["id"]) + # Budget drafts can omit host-owned scope fields until finalization. + manifest["scan"]["scope"] = { + **prepared[2]["scan"]["scope"], + **manifest["scan"]["scope"], + } + db.verify_manifest_binding(scan, manifest) + require_selected_publication(db, connection, scan, prepared) + documents = db.budget_exhausted_draft(scan, scan_dir, candidates, warning) + if documents is None: + return + if selection is not None and not has_publication: + _require_selected_result(scan, selection) + unmerged = connection.execute( + "SELECT workers.*, attempts.accepted_result_path, attempts.accepted_result_sha256 " + "FROM deep_scan_workers AS workers JOIN deep_scan_attempts AS attempts " + "ON attempts.worker_id = workers.id AND attempts.attempt = workers.attempt " + "WHERE workers.scan_id = ? AND workers.kind = 'discovery' " + "AND workers.status = 'succeeded' AND workers.merge_state IN ('buffered', 'merging') " + "ORDER BY workers.completion_sequence, workers.id", + (scan["id"],), + ).fetchall() + for worker in unmerged: + retain_unmerged_budget_coverage(scan, scan_dir, documents[2], worker) + prepared = _prepare_scan_finalization( + scan_dir, + expected_coverage_mode=db.expected_coverage_mode(scan), + completion_binding=db.workbench_completion_binding(scan, db.now(), documents[0]), + draft_documents=documents, + ) + manifest = copy.deepcopy(documents[0]) + manifest["scan"].setdefault("id", scan["id"]) + manifest["scan"]["scope"] = { + **prepared[2]["scan"]["scope"], + **manifest["scan"]["scope"], + } + db.verify_manifest_binding(scan, manifest) + for name, payload in ( + ("findings.json", documents[1]), + ("coverage.json", documents[2]), + ("scan-manifest.json", documents[0]), + ): + try: + write_scan_local_bytes( + scan_dir, + name, + ( + json.dumps(payload, allow_nan=False, indent=2, sort_keys=True) + "\n" + ).encode(), + ) + except (ContractError, OSError, TypeError, ValueError) as exc: + raise SystemExit(f"Budget-exhausted scan draft could not be saved: {exc}") from exc + if has_publication: + documents = tuple( + _read_scan_local_json(scan_dir, name, name) + for name in ("scan-manifest.json", "findings.json", "coverage.json") + ) + record_selected_publication(db, connection, scan, documents) + except ContractError as exc: + raise SystemExit(str(exc)) from exc + + +def require_selected_publication(db: Any, connection: Any, scan: Any, prepared: Any) -> None: + if scan["mode"] != "deep": + return + run = db.deep_scan.require_deep_scan_run(connection, scan["id"]) + selection = db.deep_scan.deep_scan_finalization_input(run) + if selection is None or "publicationSha256" not in selection: + return + try: + _require_selected_result(scan, selection) + except ContractError as exc: + raise RecoverableContractError(str(exc)) from exc + if selection.get("publicationSha256") != _selected_publication_digest(prepared): + raise RecoverableContractError( + "The selected Deep Scan publication changed or is missing; " + "republish its accepted result before completing the scan." + ) + + +def _require_current_deep_publication( + db: Any, connection: Any, scan_id: str, draft: dict[str, Any] +) -> None: + publication = draft.get("deepScanPublication") + run = db.deep_scan.require_deep_scan_run(connection, scan_id) + db.deep_scan.require_current_coordinator( + run, + argparse.Namespace( + coordinator_generation=publication.get("coordinatorGeneration") if publication else None + ), + ) + selection = db.deep_scan.deep_scan_finalization_input(run) + if selection is not None: + if publication is None: + raise SystemExit("Deep Scan publication requires its committed selection.") + scan = db.require_scan(connection, scan_id) + if "publicationSha256" in selection: + prepared = _prepare_scan_finalization( + Path(scan["scan_dir"]), + expected_coverage_mode=db.expected_coverage_mode(scan), + completion_binding=db.workbench_completion_binding(scan, db.now()), + draft_documents=(draft["manifest"], draft["findings"], draft["coverage"]), + ) + require_selected_publication(db, connection, scan, prepared) + selected_result = ( + str(Path(scan["scan_dir"]) / selection["resultPath"]) + if selection["resultPath"] is not None + else None + ) + else: + # Generation-one runs predate host publication metadata. Keep their existing + # draft path; adopted coordinators must carry their generation and selection. + if publication is None: + return + reducer = _latest_successful_reducer(_saved_workers(connection, scan_id)) + selected_result = reducer["result_manifest_path"] if reducer is not None else None + if publication["resultPath"] != selected_result: + raise SystemExit("Deep Scan aggregate belongs to a superseded publication selection.") + + def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]: scan_id = db.require_uuid(args.scan_id, "scan-id") with db.scan_completion_lock(scan_id): @@ -1394,6 +1921,10 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]: "The scan stopped; its saved checkpoint was retained without replacing sealed results." ) scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) + draft = None + if scan["mode"] == "deep": + draft = _read_staged_scan_draft(scan_dir, args.draft_path) + _require_current_deep_publication(db, connection, scan_id, draft) if args.checkpoint_path is not None: try: checkpoint_relative = Path(args.checkpoint_path).relative_to(scan_dir).as_posix() @@ -1423,15 +1954,8 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]: raise SystemExit( "scan_draft_conflict: canonical scan results changed; reconcile the saved checkpoint again." ) - try: - relative = Path(args.draft_path).relative_to(scan_dir).as_posix() - except ValueError as exc: - raise SystemExit( - "Scan draft must be inside the registered scan drafts directory." - ) from exc - if not re.fullmatch(r"drafts/[0-9a-fA-F-]+\.json", relative): - raise SystemExit("Scan draft must be inside the registered scan drafts directory.") - draft = _read_scan_local_json(scan_dir, relative, "Staged scan draft") + if draft is None: + draft = _read_staged_scan_draft(scan_dir, args.draft_path) manifest, findings, coverage = draft["manifest"], draft["findings"], draft["coverage"] binding = db.workbench_completion_binding(scan, db.now()) # Validate on copies: saved canonical documents remain ordinary unsealed drafts. diff --git a/plugins/codex-security/scripts/workbench_scan_history.py b/plugins/codex-security/scripts/workbench_scan_history.py index fc145bc04..e814adda9 100644 --- a/plugins/codex-security/scripts/workbench_scan_history.py +++ b/plugins/codex-security/scripts/workbench_scan_history.py @@ -75,10 +75,16 @@ def cli_scan_resume( ): raise SystemExit("Resume requires the original owning CLI session.") run = connection.execute( - "SELECT status, cancel_requested FROM deep_scan_runs WHERE scan_id = ?", (scan["id"],) + "SELECT status, cancel_requested, finalization_input_json FROM deep_scan_runs " + "WHERE scan_id = ?", + (scan["id"],), ).fetchone() if run is not None and ( - run["status"] not in {"running", "succeeded"} or run["cancel_requested"] + run["status"] not in {"running", "succeeded"} + or ( + run["cancel_requested"] + and not (run["status"] == "succeeded" and run["finalization_input_json"] is not None) + ) ): raise SystemExit("This Deep Scan has stopped and cannot resume.") try: diff --git a/plugins/codex-security/scripts/workbench_scan_usage.py b/plugins/codex-security/scripts/workbench_scan_usage.py index 648924d92..b00787117 100644 --- a/plugins/codex-security/scripts/workbench_scan_usage.py +++ b/plugins/codex-security/scripts/workbench_scan_usage.py @@ -9,6 +9,7 @@ import sqlite3 import sys import uuid +from collections import deque from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path @@ -90,12 +91,54 @@ def collect_scan_usage( ) -> dict[str, Any]: """Count only complete, attributable rollout events inside this scan's window.""" - roots = _scan_root_thread_ids(connection, scan, thread_id) + attribution = scan_execution_attribution(connection, scan) + if attribution and attribution.get("legacy"): + attribution = None + roots = ( + list( + dict.fromkeys( + [ + *( + [attribution["owner"]["threadId"]] + if attribution["owner"].get("threadId") + else [] + ), + *attribution["executionThreadIds"], + ] + ) + ) + if attribution + else _scan_root_thread_ids(connection, scan, thread_id) + ) if not roots: return _unavailable_usage("scan_thread_unavailable") - state_database = _codex_state_database() - if state_database is None: + warnings: set[str] = set() + current_database = _codex_state_database() + worker_codex_home = None + if scan["mode"] == "deep": + # Deep orchestration imports the owner-capture helper from this module; + # its settings reader is available once completion starts. + from deep_scan_workbench import read_deep_scan_execution_settings + + try: + settings = read_deep_scan_execution_settings(Path(scan["scan_dir"])) + worker_codex_home = Path(settings["codexHome"]) + except SystemExit: + # Legacy scans may have no recorded home. Keep usage best effort. + pass + groups = [(current_database, roots)] + worker_roots: set[str] = set() + if worker_codex_home is not None: + worker_roots = set( + _scan_root_thread_ids(connection, scan, None, include_owner_threads=False) + ) + # Workers retain their Codex home, but inherit an explicit current + # SQLite home. Their earlier and resumed sessions can be in either index. + worker_database = _codex_state_database(worker_codex_home) + if worker_database != current_database: + groups.append((worker_database, [root for root in roots if root in worker_roots])) + if not any(database is not None for database, _ in groups) and not worker_roots: return _unavailable_usage("codex_state_unavailable") started_at = _timestamp(scan["started_at"]) @@ -103,24 +146,75 @@ def collect_scan_usage( if started_at is None: return _unavailable_usage("scan_window_unavailable") - warnings: set[str] = set() - try: - sessions, missing_thread_ids = _discover_rollout_sessions( - state_database, - roots, - warnings, + sessions: dict[str, list[RolloutSession]] = {} + missing_thread_ids: set[str] = set() + seen_thread_ids: set[str] = set() + for state_database, group_roots in groups: + if not group_roots: + continue + try: + if state_database is None: + raise FileNotFoundError("Codex state is unavailable") + discovered, missing = _discover_rollout_sessions( + state_database, + group_roots, + warnings, + descendant_roots=set(attribution["executionThreadIds"]) if attribution else None, + ) + except (OSError, sqlite3.Error, ValueError): + warnings.add("codex_state_unavailable") + missing_thread_ids.update(group_roots) + continue + missing_thread_ids.update(missing) + for session in discovered: + copies = sessions.setdefault(session.thread_id, []) + if session not in copies: + copies.append(session) + seen_thread_ids.add(session.thread_id) + + if worker_codex_home is not None and worker_roots: + # An external SQLite location can change on recovery. Native rollouts + # still live in the recorded worker home; they retain their lineage. + for session in _discover_recorded_worker_sessions(worker_codex_home, worker_roots): + copies = sessions.setdefault(session.thread_id, []) + if session not in copies: + copies.append(session) + seen_thread_ids.add(session.thread_id) + + # Absence from one known index is not missing usage when another has it. + missing_thread_ids.difference_update(seen_thread_ids) + if not missing_thread_ids: + warnings.difference_update( + {"scan_root_unavailable", "codex_state_unavailable", "rollout_unavailable"} ) - except (OSError, sqlite3.Error, ValueError): - return _unavailable_usage("codex_state_unavailable") if not sessions: - return _unavailable_usage("scan_thread_unavailable", warnings=warnings) + return _unavailable_usage( + "codex_state_unavailable" + if "codex_state_unavailable" in warnings + else "scan_thread_unavailable", + warnings=warnings, + ) total = _empty_token_usage() observed_thread_count = 0 accepted_thread_ids: set[str] = set() excluded_thread_ids: set[str] = set() - for session in sessions: + model_usage: dict[str | None, dict[str, int]] = {} + for copies in sessions.values(): + session = copies[0] + owner_turn_id = None + if ( + attribution + and session.thread_id not in attribution["executionThreadIds"] + and session.parent_thread_id is None + ): + owner = attribution["owner"] + if session.thread_id != owner.get("threadId") or not owner.get("turnId"): + missing_thread_ids.add(session.thread_id) + warnings.add("scan_owner_turn_unavailable") + continue + owner_turn_id = owner["turnId"] if session.parent_thread_id in excluded_thread_ids: excluded_thread_ids.add(session.thread_id) continue @@ -132,10 +226,12 @@ def collect_scan_usage( warnings.add("thread_lineage_incomplete") continue try: - session_usage, session_warnings = _read_rollout_usage( - session, + session_usage, session_warnings = _read_rollout_copies_usage( + copies, started_at=started_at, completed_at=stopped_at, + owner_turn_id=owner_turn_id, + model_usage=model_usage, ) except (OSError, UnicodeError, ValueError): missing_thread_ids.add(session.thread_id) @@ -151,6 +247,9 @@ def collect_scan_usage( missing_thread_ids.add(session.thread_id) continue accepted_thread_ids.add(session.thread_id) + if "token_usage_unavailable" in session_warnings: + missing_thread_ids.add(session.thread_id) + continue observed_thread_count += 1 _add_token_usage(total, session_usage) @@ -167,9 +266,57 @@ def collect_scan_usage( result["missingThreadCount"] = len(missing_thread_ids) if warnings: result["warnings"] = sorted(warnings) + if attribution or any(model is not None for model in model_usage): + result["modelUsage"] = [{"model": model, **usage} for model, usage in model_usage.items()] return result +def _read_rollout_copies_usage( + copies: list[RolloutSession], + *, + started_at: datetime, + completed_at: datetime | None, + owner_turn_id: str | None, + model_usage: dict[str | None, dict[str, int]], +) -> tuple[dict[str, int], set[str]]: + readings = [] + for session in copies: + local_models: dict[str | None, dict[str, int]] = {} + try: + usage, warnings = _read_rollout_usage( + session, + started_at=started_at, + completed_at=completed_at, + owner_turn_id=owner_turn_id, + model_usage=local_models, + ) + except (OSError, UnicodeError, ValueError): + continue + readings.append((usage, warnings, local_models)) + if not readings: + raise ValueError("No readable rollout copy.") + attributable = [ + reading + for reading in readings + if not reading[1].intersection( + { + "thread_identity_mismatch", + "thread_ownership_unavailable", + "thread_outside_scan_window", + "token_usage_unavailable", + } + ) + ] + # Restored indexes can reference a prefix and its complete continuation. + # Keep totals and model attribution from the same copy, counting it once. + usage, warnings, selected_models = max( + attributable or readings, key=lambda reading: reading[0]["totalTokens"] + ) + for model, tokens in selected_models.items(): + _add_token_usage(model_usage.setdefault(model, _empty_token_usage()), tokens) + return usage, warnings + + def _scan_root_thread_ids( connection: sqlite3.Connection, scan: sqlite3.Row, @@ -194,12 +341,13 @@ def _scan_root_thread_ids( row["sdk_thread_id"] for row in connection.execute( """ - SELECT DISTINCT sdk_thread_id - FROM deep_scan_workers + SELECT sdk_thread_id FROM deep_scan_attempt_sessions WHERE scan_id = ? + UNION + SELECT sdk_thread_id FROM deep_scan_workers WHERE scan_id = ? AND sdk_thread_id IS NOT NULL ORDER BY sdk_thread_id """, - (scan["id"],), + (scan["id"], scan["id"]), ) ) roots: list[str] = [] @@ -221,15 +369,105 @@ def _scan_execution_thread_ids(connection: sqlite3.Connection, scan: sqlite3.Row ) -def _codex_state_database() -> Path | None: - configured_database = os.environ.get("CODEX_STATE_DB", "").strip() +def capture_scan_usage_owner(connection: sqlite3.Connection, scan: sqlite3.Row) -> dict[str, Any]: + """Bind the active native turn once; joining a scan does not bind later conversation work.""" + roots = _scan_root_thread_ids(connection, scan, None) + owner = roots[0] if roots else None + result = { + "threadId": owner, + "turnId": None, + "startedAt": scan["started_at"], + "dedicated": scan["recipe_json"] is not None, + } + database = _codex_state_database() + if owner is None or database is None: + return result + try: + sessions, _ = _discover_rollout_sessions(database, [owner], set(), descendant_roots=set()) + if not sessions: + return result + with sessions[0].path.open("rb") as source: + for line in source: + if not line.endswith(b"\n"): + continue + event = json.loads(line) + payload = event.get("payload") + if not isinstance(payload, dict): + continue + if event.get("type") == "turn_context" or ( + event.get("type") == "event_msg" and payload.get("type") == "task_started" + ): + turn_id = payload.get("turn_id") + if isinstance(turn_id, str): + result["turnId"] = turn_id + elif event.get("type") == "event_msg" and payload.get("type") == "task_complete": + result["turnId"] = None + except (OSError, ValueError, sqlite3.Error): + # Accounting availability must not prevent a scan from starting. + pass + return result + + +def scan_execution_attribution( + connection: sqlite3.Connection, scan: sqlite3.Row +) -> dict[str, Any] | None: + if scan["mode"] != "deep": + return None + run = connection.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan["id"],) + ).fetchone() + if run is None: + return None + owner_json = run["usage_owner_json"] if "usage_owner_json" in run.keys() else None + legacy = False + if owner_json is None: + legacy = ( + connection.execute( + "SELECT 1 FROM deep_scan_attempts WHERE scan_id = ? LIMIT 1", (scan["id"],) + ).fetchone() + is None + ) + roots = _scan_root_thread_ids(connection, scan, None) + owner = { + "threadId": roots[0] if roots else None, + "turnId": None, + "startedAt": scan["started_at"], + "dedicated": scan["recipe_json"] is not None, + } + else: + owner = json.loads(owner_json) + executions = _scan_execution_thread_ids(connection, scan) + if owner.get("dedicated") and owner.get("threadId") not in executions: + executions.append(owner["threadId"]) + return { + "formatVersion": 1, + **({"legacy": True} if legacy else {}), + "executionThreadIds": executions, + "owner": owner, + "startedAt": scan["started_at"], + "completedAt": scan["completed_at"], + } + + +def scan_execution_fields(connection: sqlite3.Connection, scan: sqlite3.Row) -> dict[str, Any]: + return { + "threadIds": _scan_root_thread_ids(connection, scan, None), + "executionThreadIds": _scan_execution_thread_ids(connection, scan), + "executionAttribution": scan_execution_attribution(connection, scan), + } + + +def _codex_state_database(worker_codex_home: Path | None = None) -> Path | None: + configured_home = os.environ.get("CODEX_HOME", "").strip() + current_home = Path(configured_home).expanduser() if configured_home else Path.home() / ".codex" + codex_home = worker_codex_home if worker_codex_home is not None else current_home + same_home = worker_codex_home is None or codex_home.resolve() == current_home.resolve() + configured_database = os.environ.get("CODEX_STATE_DB", "").strip() if same_home else "" if configured_database: path = Path(configured_database).expanduser() return path.resolve() if path.is_file() and os.access(path, os.R_OK) else None - configured_home = os.environ.get("CODEX_HOME", "").strip() - codex_home = Path(configured_home).expanduser() if configured_home else Path.home() / ".codex" - configured_sqlite_home = os.environ.get("CODEX_SQLITE_HOME", "").strip() + configured_sqlite_home = os.environ.get("CODEX_SQLITE_HOME", "").strip() if same_home else "" search_roots = [ *([Path(configured_sqlite_home).expanduser()] if configured_sqlite_home else []), codex_home, @@ -260,6 +498,8 @@ def _discover_rollout_sessions( state_database: Path, roots: list[str], warnings: set[str], + *, + descendant_roots: set[str] | None = None, ) -> tuple[list[RolloutSession], set[str]]: database = sqlite3.connect( state_database.as_uri() + "?mode=ro", @@ -295,6 +535,8 @@ def _discover_rollout_sessions( continue sessions.append(RolloutSession(root, None, path)) seen_thread_ids.add(root) + if descendant_roots is not None and root not in descendant_roots: + continue descendants = database.execute( """ WITH RECURSIVE descendants( @@ -357,6 +599,50 @@ def _discover_rollout_sessions( database.close() +def _discover_recorded_worker_sessions(codex_home: Path, roots: set[str]) -> list[RolloutSession]: + recorded: dict[str, list[RolloutSession]] = {} + children: dict[str, set[str]] = {} + for candidate in sorted((codex_home / "sessions").rglob("*.jsonl")): + path = _rollout_path(str(candidate)) + if path is None: + continue + try: + with path.open("rb") as stream: + metadata = json.loads(stream.readline()) + except (OSError, UnicodeError, ValueError): + continue + if not isinstance(metadata, dict) or metadata.get("type") != "session_meta": + continue + payload = metadata.get("payload") + if not isinstance(payload, dict): + continue + thread_id = payload.get("id") or payload.get("session_id") + if not isinstance(thread_id, str): + continue + parent_id = _session_parent_thread_id(payload) + recorded.setdefault(thread_id, []).append(RolloutSession(thread_id, parent_id, path)) + if parent_id is not None: + children.setdefault(parent_id, set()).add(thread_id) + + sessions: list[RolloutSession] = [] + included = set(roots) + pending = deque(sorted(roots)) + while pending: + thread_id = pending.popleft() + for session in recorded.get(thread_id, []): + sessions.append( + RolloutSession( + thread_id, + None if thread_id in roots else session.parent_thread_id, + session.path, + ) + ) + for child_id in sorted(children.get(thread_id, set()) - included): + included.add(child_id) + pending.append(child_id) + return sessions + + def _require_state_columns( connection: sqlite3.Connection, table: str, @@ -399,11 +685,22 @@ def _read_rollout_usage( *, started_at: datetime, completed_at: datetime | None, + owner_turn_id: str | None = None, + model_usage: dict[str | None, dict[str, int]] | None = None, ) -> tuple[dict[str, int], set[str]]: total = _empty_token_usage() + counter_total = _empty_token_usage() warnings: set[str] = set() previous = _empty_token_usage() boundary_reached = False + usage_observed = False + current_turn_id: str | None = None + current_model: str | None = None + response_ids: set[str] = set() + response_usage_observed = False + response_tokens = 0 + expected_response_tokens = 0 + local_models: dict[str | None, dict[str, int]] = {} with session.path.open("rb") as source: for line_number, raw_line in enumerate(source, start=1): @@ -423,6 +720,10 @@ def _read_rollout_usage( warnings.add("rollout_record_invalid") continue payload = event.get("payload") + if event.get("type") in {"session_meta", "turn_context"} and isinstance(payload, dict): + model = payload.get("model") + if isinstance(model, str) and model: + current_model = model if line_number == 1: if event.get("type") != "session_meta" or not isinstance(payload, dict): warnings.add("thread_identity_mismatch") @@ -445,6 +746,10 @@ def _read_rollout_usage( if not isinstance(payload, dict): continue + if event.get("type") == "turn_context" or ( + event.get("type") == "event_msg" and payload.get("type") == "task_started" + ): + current_turn_id = payload.get("turn_id") if not boundary_reached: if _is_owned_task_start(session.thread_id, event, payload): task_started_at = _timestamp(event.get("timestamp")) @@ -462,28 +767,97 @@ def _read_rollout_usage( if inherited_usage is not None: previous = inherited_usage continue + if event.get("type") == "token_usage_record": + response_id = payload.get("response_id") + usage = _token_snapshot({"info": {"total_token_usage": payload.get("usage")}}) + if ( + not isinstance(response_id, str) + or usage is None + or payload.get("thread_id", session.thread_id) != session.thread_id + or response_id in response_ids + ): + continue + response_ids.add(response_id) + cumulative = _token_snapshot( + {"info": {"total_token_usage": payload.get("thread_token_usage")}} + ) + if cumulative is not None: + expected_response_tokens = max( + expected_response_tokens, cumulative["totalTokens"] + ) + if not response_usage_observed: + response_usage_observed = True + total = _empty_token_usage() + local_models = {} + response_tokens += usage["totalTokens"] + timestamp = _timestamp(event.get("timestamp")) + if timestamp is None: + warnings.add("token_record_invalid") + continue + if timestamp < started_at or ( + completed_at is not None and timestamp > completed_at + ): + continue + if ( + owner_turn_id is not None + and payload.get("turn_id", current_turn_id) != owner_turn_id + ): + continue + usage_observed = True + model = payload.get("model", current_model) + if not isinstance(model, str): + model = None + _add_token_usage(total, usage) + _add_token_usage(local_models.setdefault(model, _empty_token_usage()), usage) + continue if event.get("type") != "event_msg" or payload.get("type") != "token_count": continue + # Native rate-limit updates can carry no token usage. + if "info" in payload and payload["info"] is None: + continue timestamp = _timestamp(event.get("timestamp")) snapshot = _token_snapshot(payload) if timestamp is None or snapshot is None: warnings.add("token_record_invalid") continue - delta = { - key: value - previous[key] if value >= previous[key] else value - for key, value in snapshot.items() - } + if snapshot["totalTokens"] < previous["totalTokens"]: + warnings.add("token_counter_regressed") + continue + delta = {key: max(0, value - previous[key]) for key, value in snapshot.items()} previous = snapshot if timestamp < started_at: continue if completed_at is not None and timestamp > completed_at: continue + if owner_turn_id is not None and current_turn_id != owner_turn_id: + continue + usage_observed = True + if not response_usage_observed: + local_models.setdefault(current_model, _empty_token_usage()) if delta["totalTokens"] <= 0: continue - _add_token_usage(total, delta) + _add_token_usage(counter_total, delta) + if not response_usage_observed: + _add_token_usage(total, delta) + _add_token_usage( + local_models.setdefault(current_model, _empty_token_usage()), delta + ) + if counter_total["totalTokens"] > total["totalTokens"]: + remainder = {key: max(0, value - total[key]) for key, value in counter_total.items()} + total = dict(counter_total) + _add_token_usage(local_models.setdefault(None, _empty_token_usage()), remainder) + if response_usage_observed: + warnings.discard("token_counter_regressed") + if expected_response_tokens > response_tokens: + warnings.add("token_receipts_incomplete") + if model_usage is not None: + for model, usage in local_models.items(): + _add_token_usage(model_usage.setdefault(model, _empty_token_usage()), usage) if not boundary_reached: warnings.add("thread_ownership_unavailable") + elif not usage_observed: + warnings.add("token_usage_unavailable") return total, warnings diff --git a/plugins/codex-security/scripts/workbench_schema.py b/plugins/codex-security/scripts/workbench_schema.py index 66baa2b4f..ef1d20d6c 100644 --- a/plugins/codex-security/scripts/workbench_schema.py +++ b/plugins/codex-security/scripts/workbench_schema.py @@ -867,6 +867,78 @@ ); """, ), + ( + 45, + "retain deep scan attempts and exact merge inputs", + """ + CREATE TABLE deep_scan_attempts ( + scan_id TEXT NOT NULL REFERENCES deep_scan_runs(scan_id) ON DELETE CASCADE, + worker_id TEXT NOT NULL REFERENCES deep_scan_workers(id) ON DELETE CASCADE, + attempt INTEGER NOT NULL, + status TEXT NOT NULL, + started_at TEXT NOT NULL, + completed_at TEXT, + end_reason TEXT, + error_message TEXT, + accepted_result_path TEXT, + accepted_result_sha256 TEXT, + receipt_json TEXT, + PRIMARY KEY (worker_id, attempt) + ); + + CREATE TABLE deep_scan_attempt_sessions ( + scan_id TEXT NOT NULL REFERENCES deep_scan_runs(scan_id) ON DELETE CASCADE, + worker_id TEXT NOT NULL, + attempt INTEGER NOT NULL, + sdk_thread_id TEXT NOT NULL, + observed_at TEXT NOT NULL, + PRIMARY KEY (worker_id, attempt, sdk_thread_id), + FOREIGN KEY (worker_id, attempt) + REFERENCES deep_scan_attempts(worker_id, attempt) ON DELETE CASCADE + ); + + CREATE TABLE deep_scan_merge_claims ( + worker_id TEXT PRIMARY KEY REFERENCES deep_scan_workers(id) ON DELETE CASCADE, + scan_id TEXT NOT NULL REFERENCES deep_scan_runs(scan_id) ON DELETE CASCADE, + previous_worker_id TEXT, + previous_result_path TEXT, + previous_result_sha256 TEXT, + receipt_json TEXT + ); + + ALTER TABLE deep_scan_dedup_inputs ADD COLUMN result_manifest_path TEXT; + ALTER TABLE deep_scan_dedup_inputs ADD COLUMN result_manifest_sha256 TEXT; + ALTER TABLE deep_scan_dedup_inputs ADD COLUMN attempt INTEGER; + """, + ), + ( + 46, + "persist selected deep scan finalization input", + """ + ALTER TABLE deep_scan_runs ADD COLUMN finalization_input_json TEXT; + """, + ), + ( + 47, + "freeze stopped scan checkpoint selections", + """ + ALTER TABLE scans ADD COLUMN retained_checkpoint_heads_json TEXT; + """, + ), + ( + 48, + "bind original deep scan parent usage turn", + """ + ALTER TABLE deep_scan_runs ADD COLUMN usage_owner_json TEXT; + """, + ), + ( + 51, + "bind original deep scan execution settings", + """ + ALTER TABLE deep_scan_runs ADD COLUMN execution_settings_json TEXT; + """, + ), ) diff --git a/plugins/codex-security/scripts/workbench_validation.py b/plugins/codex-security/scripts/workbench_validation.py index a65aff1e7..661eb6bad 100644 --- a/plugins/codex-security/scripts/workbench_validation.py +++ b/plugins/codex-security/scripts/workbench_validation.py @@ -137,6 +137,7 @@ def _valid_measured_scan_usage(usage: object) -> bool: "threadCount", "missingThreadCount", "warnings", + "modelUsage", *SCAN_USAGE_TOKEN_KEYS, } if thread_count == 0 or not set(usage).issubset(allowed_keys): @@ -144,6 +145,19 @@ def _valid_measured_scan_usage(usage: object) -> bool: counts = {key: usage.get(key) for key in SCAN_USAGE_TOKEN_KEYS} if not _valid_scan_token_counts(counts): return False + if "modelUsage" in usage: + parts = usage["modelUsage"] + if not isinstance(parts, list) or not parts: + return False + for part in parts: + if not isinstance(part, dict) or set(part) != {"model", *SCAN_USAGE_TOKEN_KEYS}: + return False + if part["model"] is not None and not isinstance(part["model"], str): + return False + if not _valid_scan_token_counts({key: part[key] for key in SCAN_USAGE_TOKEN_KEYS}): + return False + if any(sum(part[key] for part in parts) != counts[key] for key in SCAN_USAGE_TOKEN_KEYS): + return False missing = usage.get("missingThreadCount", 0) if type(missing) is not int or missing < 0: return False @@ -154,7 +168,7 @@ def _valid_measured_scan_usage(usage: object) -> bool: return True -def parse_scan_cost(value: str | None) -> str | None: +def parse_scan_cost(value: str | None, *, allow_lower_bound: bool = False) -> str | None: if value is None: return None if len(value.encode("utf-8")) > 8192: @@ -163,7 +177,10 @@ def parse_scan_cost(value: str | None) -> str | None: cost = json.loads(value, parse_constant=reject_nonstandard_json_number) except (TypeError, UnicodeError, ValueError) as exc: raise SystemExit("Scan cost must be a valid JSON object.") from exc - if isinstance(cost, dict) and "usage" in cost: + if allow_lower_bound and isinstance(cost, dict) and set(cost) == {"lowerBound"}: + if not _valid_legacy_scan_cost(cost["lowerBound"]): + raise SystemExit("Scan cost lower bound must be a valid measured cost.") + elif isinstance(cost, dict) and "usage" in cost: if ( not set(cost).issubset({"usage", "cost"}) or not _valid_measured_scan_usage(cost["usage"]) @@ -178,6 +195,17 @@ def parse_scan_cost(value: str | None) -> str | None: return json.dumps(cost, separators=(",", ":"), allow_nan=False) +def parse_budget_scan_cost(value: str | None) -> tuple[str | None, Any]: + cost_json = parse_scan_cost(value, allow_lower_bound=True) + if cost_json is None: + raise SystemExit("Budget-exhausted scan completion requires the measured scan cost.") + cost = json.loads(cost_json) + if set(cost) == {"lowerBound"}: + # A priced subtotal proves the stop but is not the saved total estimate. + return None, cost["lowerBound"] + return cost_json, cost.get("cost", cost) + + def bounded_output_text(value: Any, maximum_bytes: int) -> str: encoded = str(value).encode("utf-8")[:maximum_bytes] return encoded.decode("utf-8", errors="ignore") diff --git a/plugins/codex-security/tests/test_accepted_publication_references.py b/plugins/codex-security/tests/test_accepted_publication_references.py new file mode 100644 index 000000000..e7d12d871 --- /dev/null +++ b/plugins/codex-security/tests/test_accepted_publication_references.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import hashlib +import json +from argparse import Namespace + +import pytest +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +def accept_reducer(connection, scan): + result = add_worker(connection, scan) + worker_id = result.parent.name + coverage = { + **scan.coverage, + "completeness": "partial", + "deferred": [{"id": "accepted-follow-up", "reason": "Accepted unresolved review."}], + "reviews": [{"workerId": worker_id, "attempt": 1, "completeness": "partial"}], + } + contents = json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": scan.findings, + "sourceCoverage": coverage, + } + ).encode() + digest = hashlib.sha256(contents).hexdigest() + result.write_bytes(contents) + accepted = result.parent / "accepted" / f"{digest}.json" + accepted.parent.mkdir() + accepted.write_bytes(contents) + result.unlink() + with connection: + connection.execute( + "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none' WHERE id = ?", + (worker_id,), + ) + connection.execute( + "INSERT INTO deep_scan_attempts (scan_id, worker_id, attempt, status, started_at, " + "completed_at, accepted_result_path, accepted_result_sha256) " + "VALUES (?, ?, 1, 'succeeded', ?, ?, ?, ?)", + (scan.scan_id, worker_id, scan.timestamp, scan.timestamp, str(accepted), digest), + ) + connection.execute( + "UPDATE deep_scan_runs SET coordinator_generation = 3 WHERE scan_id = ?", + (scan.scan_id,), + ) + return result, accepted, coverage + + +@pytest.mark.parametrize("selected", [True, False], ids=["accepted", "replaceable-output"]) +def test_legacy_publication_compares_registered_accepted_reference( + workbench_api, workbench_db, publication_scan, selected +): + scan = publication_scan() + result, accepted, _ = accept_reducer(workbench_db, scan) + staged = stage_publication( + scan, generation=3, result_path=accepted if selected else result, title="Accepted aggregate" + ) + before = {path: path.read_bytes() for path in scan.scan_dir.rglob("*.json")} + + if selected: + workbench_api["write_scan_draft"](workbench_db, staged) + else: + with pytest.raises(SystemExit, match="aggregate"): + workbench_api["write_scan_draft"](workbench_db, staged) + assert {path: path.read_bytes() for path in scan.scan_dir.rglob("*.json")} == before + + +def test_stopped_recovery_uses_accepted_bytes_after_replaceable_output_disappears( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + _, accepted, coverage = accept_reducer(workbench_db, scan) + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + contents = accepted.read_bytes() + + stopped = workbench_api["fail_scan"]( + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."), + )["scan"] + + assert stopped["findingCount"] == 1 + published = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert published["reviews"] == coverage["reviews"] + assert coverage["deferred"][0] in published["deferred"] + manifest = (scan.scan_dir / "scan-manifest.json").read_bytes() + workbench_api["preserve_scan_results"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None + ), + ) + assert (scan.scan_dir / "scan-manifest.json").read_bytes() == manifest + assert accepted.read_bytes() == contents diff --git a/plugins/codex-security/tests/test_budget_selection_publication.py b/plugins/codex-security/tests/test_budget_selection_publication.py new file mode 100644 index 000000000..40b7e91b5 --- /dev/null +++ b/plugins/codex-security/tests/test_budget_selection_publication.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import json +import sqlite3 +from argparse import Namespace + +import pytest +from test_accepted_publication_references import accept_reducer +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_publication_stop_interleavings import published_bytes, saved_selection +from test_workbench_db import BUDGET_COST + + +@pytest.mark.parametrize("selected", [False, True], ids=["legacy-v1", "selected-v2"]) +@pytest.mark.parametrize("reason", ["saturated", "capped"]) +@pytest.mark.parametrize("cancel_first", [False, True], ids=["budget-first", "cancel-first"]) +def test_budget_completion_and_cancel_keep_the_committed_outcome( + workbench_api, workbench_db, publication_scan, tmp_path, selected, reason, cancel_first +): + scan = publication_scan() + _, accepted, coverage = accept_reducer(workbench_db, scan) + selection = saved_selection(workbench_db, scan, accepted, reason=reason) if selected else None + scan.coverage = coverage + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute("UPDATE deep_scan_runs SET terminal_reason = ?", (reason,)) + staged = stage_publication( + scan, generation=3, result_path=accepted, title="Selected accepted aggregate" + ) + database_path = tmp_path / "budget-publication.sqlite3" + with sqlite3.connect(database_path) as connection: + workbench_db.backup(connection) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + workbench_api["write_scan_draft"](connection, staged) + accepted_bytes = accepted.read_bytes() + warning = "Scan stopped after reaching its configured cost limit." + budget_args = Namespace( + scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=warning + ) + cancel_args = Namespace(scan_id=scan.scan_id, thread_id=None) + if cancel_first: + workbench_api["cancel_scan"](connection, cancel_args) + frozen = published_bytes(scan) + with pytest.raises(SystemExit, match="running"): + workbench_api["complete_budget_exhausted_scan"](connection, budget_args) + else: + workbench_api["complete_budget_exhausted_scan"](connection, budget_args) + frozen = published_bytes(scan) + with pytest.raises(SystemExit, match="running"): + workbench_api["cancel_scan"](connection, cancel_args) + assert published_bytes(scan) == frozen + + with sqlite3.connect(database_path) as connection: + connection.row_factory = sqlite3.Row + row = connection.execute("SELECT * FROM scans").fetchone() + run = connection.execute("SELECT * FROM deep_scan_runs").fetchone() + assert row["status"] == ("failed" if cancel_first else "complete") + assert bool(row["canceled_at"]) == cancel_first + assert run["terminal_reason"] == reason + assert json.loads(run["finalization_input_json"] or "null") == selection + assert accepted.read_bytes() == accepted_bytes + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert coverage["completeness"] == "partial" + assert any(item["id"] == "accepted-follow-up" for item in coverage["deferred"]) + if not cancel_first: + assert any(item["id"] == "scan-cost-limit" for item in coverage["deferred"]) + assert warning in json.loads(row["completion_warnings_json"]) + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == 1 + with pytest.raises(SystemExit, match="stopped"): + workbench_api["write_scan_draft"](connection, staged) + assert published_bytes(scan) == frozen diff --git a/plugins/codex-security/tests/test_checkpoint_publication_authority.py b/plugins/codex-security/tests/test_checkpoint_publication_authority.py new file mode 100644 index 000000000..7c277d3dd --- /dev/null +++ b/plugins/codex-security/tests/test_checkpoint_publication_authority.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import copy +import json +from argparse import Namespace + +import pytest +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan +from workbench_test_support import write_checkpoint + + +@pytest.mark.parametrize("archived", [False, True], ids=["current", "archived"]) +@pytest.mark.parametrize("has_head", [False], ids=["legacy"]) +@pytest.mark.parametrize("complete", [False, True], ids=["checkpoint", "complete"]) +def test_recovery_honors_rejection_committed_before_result_replacement( + workbench_api, workbench_db, publication_scan, archived, has_head, complete +): + scan = publication_scan() + provisional = copy.deepcopy(scan.findings[0]) + provisional["extensions"] = {"candidateId": "candidate-rejected"} + retained = copy.deepcopy(scan.findings[0]) + retained["identity"]["anchor"] = "independent-finding" + retained["extensions"] = {"candidateId": "candidate-retained"} + retained["locations"][0]["startLine"] = 20 + retained["locations"][0]["endLine"] = 21 + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + result_path = add_worker(workbench_db, scan, status="canceled") + if archived: + result_path = result_path.parent / "attempts" / "attempt-1" / "result.json" + result_path.parent.mkdir(parents=True) + previous = { + "scanId": scan.scan_id, + "complete": True, + "findings": [provisional, retained], + "coverage": scan.coverage, + } + result_path.write_text(json.dumps(previous)) + old_checkpoint = write_checkpoint(result_path.parent / "checkpoints", previous) + rejected = { + **previous, + "complete": complete, + "findings": [retained], + "coverage": { + **scan.coverage, + "surfaces": [ + { + "candidateId": "candidate-rejected", + "label": "Validated candidate disposition", + "disposition": "rejected", + "receiptRefs": [], + } + ], + }, + } + checkpoint = write_checkpoint(result_path.parent / "checkpoints", rejected) + if has_head: + (result_path.parent / "checkpoint-head.json").write_text( + json.dumps({"checkpoint": checkpoint.name}) + ) + saved_bytes = {path: path.read_bytes() for path in (result_path, old_checkpoint, checkpoint)} + + stopped = workbench_api["fail_scan"]( + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."), + )["scan"] + + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert stopped["findingCount"] == len(findings) == (1 if has_head else 2) + assert any(finding["identity"]["anchor"] == "independent-finding" for finding in findings) + assert all(path.read_bytes() == contents for path, contents in saved_bytes.items()) + if has_head: + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert any( + surface.get("candidateId") == "candidate-rejected" + and surface.get("disposition") == "rejected" + for surface in coverage["surfaces"] + ) + + +def save_disposition(scan, directory, disposition): + directory.mkdir(parents=True, exist_ok=True) + finding = copy.deepcopy(scan.findings[0]) + finding["extensions"] = {"candidateId": "candidate-disposition"} + draft = { + "scanId": scan.scan_id, + "complete": True, + "findings": [finding] if disposition == "reported" else [], + "coverage": { + **scan.coverage, + "surfaces": [ + { + "candidateId": "candidate-disposition", + "label": "Validated candidate disposition", + "disposition": disposition, + "receiptRefs": [], + } + ], + }, + } + checkpoint = write_checkpoint(directory / "checkpoints", draft) + (directory / "checkpoint-head.json").write_text(json.dumps({"checkpoint": checkpoint.name})) + return draft + + +def test_legacy_frozen_publication_keeps_result_fallback_without_saved_heads( + workbench_api, workbench_db, publication_scan, monkeypatch +): + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + result = add_worker(workbench_db, scan, status="canceled") + previous = save_disposition(scan, result.parent, "reported") + result.write_text(json.dumps(previous)) + save_disposition(scan, result.parent, "rejected") + (result.parent / "checkpoint-head.json").unlink() + + def fail_before_publication(*args, **kwargs): + raise OSError("Synthetic publication interruption") + + with monkeypatch.context() as patch: + patch.setattr( + workbench_api["saved_results"], + "_write_prepared_scan_finalization", + fail_before_publication, + ) + workbench_api["fail_scan"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped." + ), + ) + with workbench_db: + workbench_db.execute( + "UPDATE scans SET retained_checkpoint_heads_json = NULL WHERE id = ?", (scan.scan_id,) + ) + save_disposition(scan, result.parent, "rejected") + + replayed = workbench_api["preserve_scan_results"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None + ), + )["scan"] + + assert replayed["findingCount"] == 1 diff --git a/plugins/codex-security/tests/test_deep_scan_compatibility.py b/plugins/codex-security/tests/test_deep_scan_compatibility.py new file mode 100644 index 000000000..1fa0f882b --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_compatibility.py @@ -0,0 +1,407 @@ +"""Compatibility checks use real persisted scans and preserve unsupported state.""" + +from __future__ import annotations + +import json +import os +import sqlite3 +import subprocess +import sys +import uuid +from pathlib import Path + +import pytest +from workbench_test_support import SCRIPT, run_workbench + + +def snapshot(state_dir: Path) -> str: + with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: + return "\n".join(connection.iterdump()) + + +def claim_requiring_original_settings( + state: Path, scan_id: str +) -> subprocess.CompletedProcess[str]: + # Exercise the private MCP requirement against both the parent and fixed + # workbench, without adding a public CLI argument. + return subprocess.run( + [ + sys.executable, + "-c", + "\n".join( + [ + "import runpy, sys", + "script = sys.argv.pop(1)", + "main = runpy.run_path(script)['main']", + "namespace = main.__globals__", + "parse = namespace['parse_args']", + "def parse_with_requirement(*args, **kwargs):", + " result = parse(*args, **kwargs)", + " result.require_execution_settings = True", + " return result", + "namespace['parse_args'] = parse_with_requirement", + "main()", + ] + ), + str(SCRIPT), + "claim-deep-scan-coordinator", + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + ], + env={**os.environ, "CODEX_SECURITY_STATE_DIR": str(state)}, + capture_output=True, + text=True, + timeout=30, + ) + + +@pytest.mark.parametrize( + "workflow,settings_version", + [ + ("deep-security-scan/v1", 99), + ("deep-scan-mcp/v1", 99), + ("deep-security-scan/v2", 99), + ("deep-security-scan/v2", None), + ("deep-security-scan/v2", 1), + ], +) +def test_missing_or_unsupported_settings_reject_before_takeover( + tmp_path: Path, workflow: str, settings_version: int | None +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + run = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + "--workflow-version", + "deep-security-scan/v1" if workflow == "deep-security-scan/v2" else workflow, + )["deepScan"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute("UPDATE deep_scan_runs SET workflow_version = ?", (workflow,)) + connection.execute( + "UPDATE deep_scan_runs SET coordinator_generation = 2, " + "phase = 'discovery', updated_at = '2000-01-01T00:00:00Z'" + ) + settings_path = Path(run["scanDir"]) / "artifacts/deep_discovery/execution-settings.json" + saved = None + if settings_version is not None: + settings_path.parent.mkdir(parents=True, exist_ok=True) + saved = json.dumps( + { + "version": settings_version, + "settings": {"codexPath": "/fixture/codex", "codexHome": "/fixture/home"}, + } + ).encode() + settings_path.write_bytes(saved) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + columns = {row[1] for row in connection.execute("PRAGMA table_info(deep_scan_runs)")} + if "execution_settings_json" not in columns: + connection.execute( + "ALTER TABLE deep_scan_runs ADD COLUMN execution_settings_json TEXT" + ) + if settings_version != 1: + connection.execute( + "UPDATE deep_scan_runs SET execution_settings_json = ?", (saved.decode(),) + ) + before = snapshot(state) + result = claim_requiring_original_settings(state, run["scanId"]) + assert snapshot(state) == before, ( + "settings rejection must precede ownership and worker recovery" + ) + assert result.returncode != 0 + assert ( + "newer version to resume execution" if workflow.endswith("/v2") else "execution settings" + ) in result.stderr + assert (settings_path.read_bytes() if settings_path.exists() else None) == saved + + +@pytest.mark.parametrize("workflow", ["deep-security-scan/v1", "deep-scan-mcp/v1"]) +def test_legacy_takeover_does_not_require_or_create_a_new_snapshot( + tmp_path: Path, workflow: str +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + run = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + "--workflow-version", + workflow, + )["deepScan"] + artifact = Path(run["scanDir"]) / "artifacts/deep_discovery/execution-settings.json" + artifact.parent.mkdir(parents=True, exist_ok=True) + planted = json.dumps( + { + "version": 99, + "settings": {"codexPath": "/untrusted/codex", "codexHome": "/untrusted/home"}, + } + ) + artifact.write_text(planted) + result = claim_requiring_original_settings(state, run["scanId"]) + assert result.returncode == 0, result.stderr + observed = json.loads(result.stdout)["deepScan"] + assert observed["workflowVersion"] == workflow + assert observed["config"] == run["config"] + assert artifact.read_text() == planted, "legacy recovery ignores model-writable settings" + + +@pytest.mark.parametrize("completion_only", [False, True]) +def test_observation_and_selected_completion_do_not_require_worker_settings( + tmp_path: Path, completion_only: bool +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + run = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + )["deepScan"] + claim = run_workbench( + state, + "claim-deep-scan-coordinator", + "--scan-id", + run["scanId"], + "--thread-id", + "fixture-thread", + ) + if completion_only: + selection = { + "version": 1, + "resultPath": None, + "resultSha256": None, + "terminalReason": "capped", + "omittedWorkerIds": [], + "selectedAt": run["createdAt"], + } + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "finalization_input_json = ?, " + "updated_at = '2000-01-01T00:00:00Z'", + (json.dumps(selection),), + ) + before = snapshot(state) + result = claim_requiring_original_settings(state, run["scanId"]) + assert result.returncode == 0, result.stderr + observed = json.loads(result.stdout) + if completion_only: + assert observed["coordinatorDisposition"] == "adopted" + assert observed["deepScan"]["finalizationInput"] == selection + else: + assert observed["coordinatorDisposition"] == "observing" + assert ( + observed["deepScan"]["coordinatorGeneration"] + == claim["deepScan"]["coordinatorGeneration"] + ) + assert snapshot(state) == before + + +@pytest.mark.parametrize("version", ["deep-security-scan/v1", "deep-scan-mcp/v1"]) +def test_supported_workflows_keep_their_identity(tmp_path: Path, version: str) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + begun = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + "--workflow-version", + version, + )["deepScan"] + claimed = run_workbench( + state, + "claim-deep-scan-coordinator", + "--scan-id", + str(begun["scanId"]), + "--thread-id", + "fixture-thread", + )["deepScan"] + assert claimed["workflowVersion"] == version + assert claimed["schemaVersion"] == 1 + assert claimed["noNewStreak"] == begun["noNewStreak"] + assert claimed["config"] == begun["config"] + + +@pytest.mark.parametrize("field,value", [("workflow_version", "future/v99")]) +@pytest.mark.parametrize("operation", ["begin", "claim", "handoff"]) +def test_unsupported_execution_does_not_mutate( + tmp_path: Path, + field: str, + value: str | int, + operation: str, +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + begun = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + )["deepScan"] + scan_id = str(begun["scanId"]) + if operation == "claim": + artifact_dir = Path(str(begun["scanDir"])) / "artifacts" / "deep_discovery" / "worker" + artifact_dir.mkdir(parents=True) + prompt = artifact_dir / "prompt.md" + prompt.write_text("Original discovery input") + run_workbench( + state, + "upsert-deep-scan-worker", + "--scan-id", + scan_id, + "--worker-id", + str(uuid.uuid4()), + "--kind", + "discovery", + "--status", + "running", + "--prompt-path", + str(prompt), + "--artifact-dir", + str(artifact_dir), + "--attempt", + "1", + ) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute( + f"UPDATE deep_scan_runs SET {field} = ?, updated_at = ?", + (value, "2000-01-01T00:00:00Z"), + ) + if operation == "handoff": + connection.execute( + "UPDATE scans SET deep_scan_owner_thread_id = NULL, recipe_json = '{}'" + ) + connection.execute("UPDATE workspaces SET thread_id = NULL") + before = snapshot(state) + command = "claim-deep-scan-coordinator" if operation == "claim" else "begin-deep-scan" + result = run_workbench( + state, + command, + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + *(["--model", "observer-model"] if operation != "claim" else []), + check=False, + ) + assert result["returncode"] != 0 + assert "unsupported" in str(result["stderr"]).lower() + assert snapshot(state) == before + if operation != "handoff": + observed = run_workbench( + state, "get-deep-scan", "--scan-id", scan_id, "--thread-id", "fixture-thread" + )["deepScan"] + assert ( + observed["workflowVersion" if field == "workflow_version" else "schemaVersion"] == value + ) + assert snapshot(state) == before + + +@pytest.mark.parametrize("original", [None, "Original discovery context"]) +def test_reader_honors_original_context_when_present(tmp_path: Path, original: str | None) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + begun = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + )["deepScan"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + columns = {row[1] for row in connection.execute("PRAGMA table_info(deep_scan_runs)")} + if "discovery_user_context" not in columns: + connection.execute("ALTER TABLE deep_scan_runs ADD COLUMN discovery_user_context TEXT") + connection.execute("UPDATE deep_scan_runs SET discovery_user_context = ?", (original,)) + connection.execute("UPDATE scans SET user_context = 'Later discussion'") + observed = run_workbench( + state, + "get-deep-scan", + "--scan-id", + str(begun["scanId"]), + "--thread-id", + "fixture-thread", + )["deepScan"] + assert observed["userContext"] == original + + +@pytest.mark.parametrize( + "version,message", [("future/v99", "unsupported"), ("deep-security-scan/v2", "newer version")] +) +def test_unsupported_new_workflow_does_not_claim_registered_scan( + tmp_path: Path, version: str, message: str +) -> None: + state = tmp_path / "state" + target = tmp_path / "target" + target.mkdir() + scan_dir = tmp_path / "scan" + scan_dir.mkdir(mode=0o700) + registered = run_workbench( + state, + "register-cli-scan", + "--scan-dir", + str(scan_dir), + "--repository", + str(target), + "--registration-json-stdin", + input_text=json.dumps( + { + "recipe": { + "config": {}, + "mode": "deep", + "repository": str(target), + "target": {"kind": "repository", "paths": []}, + } + } + ), + ) + before = snapshot(state) + rejected = run_workbench( + state, + "begin-deep-scan", + "--scan-id", + str(registered["scanId"]), + "--thread-id", + "fixture-thread", + "--workflow-version", + version, + check=False, + ) + assert rejected["returncode"] != 0 + assert message in str(rejected["stderr"]).lower() + assert snapshot(state) == before diff --git a/plugins/codex-security/tests/test_deep_scan_finalization_compatibility.py b/plugins/codex-security/tests/test_deep_scan_finalization_compatibility.py new file mode 100644 index 000000000..28f5b2852 --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_finalization_compatibility.py @@ -0,0 +1,123 @@ +"""Selected finalization survives ownership recovery without restarting discovery.""" + +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path + +import pytest +from workbench_test_support import run_workbench + + +@pytest.mark.parametrize("legacy_version", [None, "deep-security-scan/v1", "deep-scan-mcp/v1"]) +def test_new_workflow_default_preserves_existing_run_version( + tmp_path: Path, legacy_version: str | None +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + created = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + *(["--workflow-version", legacy_version] if legacy_version else []), + )["deepScan"] + expected_version = legacy_version or "deep-security-scan/v1" + assert created["workflowVersion"] == expected_version + resumed = run_workbench( + state, + "begin-deep-scan", + "--scan-id", + created["scanId"], + "--thread-id", + "fixture-thread", + )["deepScan"] + assert resumed["workflowVersion"] == expected_version + assert resumed["createdAt"] == created["createdAt"] + + +def selected_scan(tmp_path: Path, version: int) -> tuple[Path, str, dict[str, object]]: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + run = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + )["deepScan"] + selected = { + "version": version, + "resultPath": None, + "resultSha256": None, + "terminalReason": "capped", + "omittedWorkerIds": [], + "selectedAt": "2000-01-01T00:00:00Z", + } + with sqlite3.connect(state / "workbench.sqlite3") as connection: + columns = {row[1] for row in connection.execute("PRAGMA table_info(deep_scan_runs)")} + if "finalization_input_json" not in columns: + connection.execute("ALTER TABLE deep_scan_runs ADD COLUMN finalization_input_json TEXT") + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "finalization_input_json = ?, phase = 'reducing', " + "created_at = '2000-01-01T00:00:00Z', updated_at = '2000-01-01T00:00:00Z'", + (json.dumps(selected),), + ) + return state, str(run["scanId"]), selected + + +def test_claim_preserves_selected_finalization_without_discovery_recovery(tmp_path: Path) -> None: + state, scan_id, selected = selected_scan(tmp_path, 1) + claimed = run_workbench( + state, + "claim-deep-scan-coordinator", + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + )["deepScan"] + assert claimed["finalizationInput"] == selected + assert claimed["phase"] == "reducing" + assert claimed["coordinatorGeneration"] == 2 + assert claimed["dispatchedCount"] == 0 + assert claimed["createdAt"] == "2000-01-01T00:00:00Z" + + +@pytest.mark.parametrize("command", ["begin-deep-scan", "claim-deep-scan-coordinator"]) +def test_unsupported_selection_rejects_without_mutation(tmp_path: Path, command: str) -> None: + state, scan_id, selected = selected_scan(tmp_path, 99) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + before = "\n".join(connection.iterdump()) + rejected = run_workbench( + state, + command, + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + check=False, + ) + assert rejected["returncode"] != 0 + assert "unsupported" in str(rejected["stderr"]).lower() + with sqlite3.connect(state / "workbench.sqlite3") as connection: + assert "\n".join(connection.iterdump()) == before + observed = run_workbench( + state, + "get-deep-scan", + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + )["deepScan"] + assert observed["finalizationInput"] == selected diff --git a/plugins/codex-security/tests/test_deep_scan_persistence.py b/plugins/codex-security/tests/test_deep_scan_persistence.py new file mode 100644 index 000000000..ff3ce985e --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_persistence.py @@ -0,0 +1,242 @@ +from __future__ import annotations + +import json +import sqlite3 +import subprocess +import sys +import uuid +from pathlib import Path + +import pytest +from test_workbench_deep_scan import ( + begin_target_scan, + dispatch_discovery_worker, + upsert_worker, + worker_paths, +) +from workbench_test_support import run_workbench + + +def test_state_snapshot_does_not_mix_concurrent_acceptance( + tmp_path: Path, workbench_api, monkeypatch: pytest.MonkeyPatch +) -> None: + state_dir, codex_home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + initial = begin_target_scan(state_dir, codex_home, target, tmp_path / "scans")["deepScan"] + scan_id = initial["scanId"] + worker_id, _, _, _ = dispatch_discovery_worker( + state_dir, + codex_home, + scan_id=scan_id, + scan_dir=Path(initial["scanDir"]), + name="discovery-1", + succeed=False, + ) + database = state_dir / "workbench.sqlite3" + deep_scan = sys.modules["deep_scan_workbench"] + monkeypatch.setattr(deep_scan, "require_scan", workbench_api["require_scan"]) + original = deep_scan.require_deep_scan_run + + def accept_after_read(connection, requested_scan_id): + run = original(connection, requested_scan_id) + with sqlite3.connect(database) as writer: + writer.execute( + "UPDATE deep_scan_runs SET completion_sequence = 1 WHERE scan_id = ?", (scan_id,) + ) + writer.execute( + "UPDATE deep_scan_workers SET status = 'succeeded', completion_sequence = 1 " + "WHERE id = ?", + (worker_id,), + ) + return run + + monkeypatch.setattr(deep_scan, "require_deep_scan_run", accept_after_read) + with sqlite3.connect(database) as reader: + reader.row_factory = sqlite3.Row + snapshot = deep_scan.deep_scan_state(reader, scan_id) + assert snapshot["completionSequence"] == 0 + assert snapshot["workers"][0]["status"] == "running" + assert not reader.in_transaction + + +def test_state_snapshot_preserves_its_callers_transaction( + tmp_path: Path, workbench_api, monkeypatch: pytest.MonkeyPatch +) -> None: + target = tmp_path / "target" + target.mkdir() + initial = begin_target_scan(tmp_path / "state", tmp_path / "codex", target, tmp_path / "scans") + scan_id = initial["deepScan"]["scanId"] + deep_scan = sys.modules["deep_scan_workbench"] + monkeypatch.setattr(deep_scan, "require_scan", workbench_api["require_scan"]) + with sqlite3.connect(tmp_path / "state" / "workbench.sqlite3") as connection: + connection.row_factory = sqlite3.Row + connection.execute("BEGIN IMMEDIATE") + connection.execute( + "UPDATE deep_scan_runs SET consecutive_errors = 2 WHERE scan_id = ?", (scan_id,) + ) + snapshot = deep_scan.deep_scan_state(connection, scan_id) + assert snapshot["consecutiveErrors"] == 2 + assert connection.in_transaction + connection.rollback() + assert deep_scan.deep_scan_state(connection, scan_id)["consecutiveErrors"] == 0 + + +def test_native_usage_keeps_replaced_failed_canceled_attempts_and_descendants( + tmp_path: Path, workbench_api, monkeypatch: pytest.MonkeyPatch +) -> None: + from datetime import datetime, timedelta + + from test_workbench_scan_usage import _counts, _event, _rollout, _state_graph, _token_event + + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + run = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + worker_id = str(uuid.uuid4()) + prompt, artifacts, _ = worker_paths(Path(run["scanDir"]), "discovery-1") + mutation = dict( + scan_id=run["scanId"], + worker_id=worker_id, + kind="discovery", + prompt_path=prompt, + artifact_dir=artifacts, + ) + upsert_worker(state, home, **mutation, status="running", attempt=3, thread_id="replacement") + upsert_worker(state, home, **mutation, status="canceled", attempt=3, thread_id="replacement") + # The writer release recorded these prior attempts and observed sessions. + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute("PRAGMA foreign_keys = ON") + for attempt, status, thread in ( + (1, "replaced", "old"), + (2, "failed", "old"), + (3, "canceled", "replacement"), + ): + connection.execute( + "INSERT INTO deep_scan_attempts " + "(scan_id, worker_id, attempt, status, started_at) VALUES (?, ?, ?, ?, ?)", + (run["scanId"], worker_id, attempt, status, run["createdAt"]), + ) + connection.execute( + "INSERT INTO deep_scan_attempt_sessions " + "(scan_id, worker_id, attempt, sdk_thread_id, observed_at) VALUES (?, ?, ?, ?, ?)", + (run["scanId"], worker_id, attempt, thread, run["createdAt"]), + ) + terminal = run_workbench( + state, "get-deep-scan", "--scan-id", run["scanId"], "--thread-id", "thread-deep-scan" + )["deepScan"] + assert [item["status"] for item in terminal["attempts"]] == ["replaced", "failed", "canceled"] + environment = { + "CODEX_HOME": str(home), + "CODEX_SQLITE_HOME": str(tmp_path / "native"), + "CODEX_STATE_DB": "", + } + for key, value in environment.items(): + monkeypatch.setenv(key, value) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.row_factory = sqlite3.Row + scan = connection.execute("SELECT * FROM scans WHERE id = ?", (run["scanId"],)).fetchone() + timestamp = datetime.fromisoformat(scan["started_at"]) + timedelta(microseconds=1) + context = _event( + timestamp, "turn_context", {"turn_id": "fixture-turn", "model": "gpt-5.6-sol"} + ) + old = _rollout(tmp_path, "old", [context]) + _state_graph( + environment, + { + "thread-deep-scan": _rollout(tmp_path, "thread-deep-scan", []), + "old": old, + "replacement": _rollout( + tmp_path, "replacement", [context, _token_event(timestamp, 30, 0)] + ), + "child": _rollout( + tmp_path, + "child", + [context, _token_event(timestamp, 5, 0)], + parent_thread_id="old", + ), + "unrelated": _rollout( + tmp_path, + "unrelated", + [context, _token_event(timestamp, 900, 0)], + parent_thread_id="thread-deep-scan", + ), + }, + [("old", "child"), ("thread-deep-scan", "unrelated")], + ) + reader = sys.modules["workbench_scan_usage"] + pending = reader.collect_scan_usage(connection, scan) + assert pending["inputTokens"] == 35 + assert pending["missingThreadCount"] == 2 + old.write_text(old.read_text() + json.dumps(_token_event(timestamp, 20, 0)) + "\n") + measured = reader.collect_scan_usage(connection, scan) + assert measured["inputTokens"] == 55 + assert measured["threadCount"] == 3 + assert measured["missingThreadCount"] == 1 + assert measured["coverage"] == "partial" # Original shared parent turn was unavailable. + assert measured["modelUsage"] == [{"model": "gpt-5.6-sol", **_counts(55, 0, 0)}] + + +def test_acceptance_rejects_mutable_result_behind_checkpoint_head(tmp_path: Path) -> None: + import hashlib + + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + run = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + worker_id = str(uuid.uuid4()) + prompt, artifacts, result = worker_paths(Path(run["scanDir"]), "discovery") + mutation = dict( + scan_id=run["scanId"], + worker_id=worker_id, + kind="discovery", + prompt_path=prompt, + artifact_dir=artifacts, + attempt=1, + ) + upsert_worker(state, home, **mutation, status="running") + draft = {"scanId": run["scanId"], "findings": [], "coverage": {"deferred": ["unresolved"]}} + content = json.dumps(draft).encode() + checkpoint = artifacts / "checkpoints" / f"{hashlib.sha256(content).hexdigest()}.json" + checkpoint.parent.mkdir() + checkpoint.write_bytes(content) + (artifacts / "checkpoint-head.json").write_text(json.dumps({"checkpoint": checkpoint.name})) + result.write_text(json.dumps({**draft, "coverage": {}})) + with pytest.raises(subprocess.CalledProcessError) as failure: + upsert_worker(state, home, **mutation, status="succeeded", result_path=result) + assert "does not match its current checkpoint head" in failure.value.stderr + assert checkpoint.read_bytes() == content + + +def test_claim_replay_preserves_original_inputs_after_concurrent_discovery(tmp_path: Path) -> None: + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + run = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + scan_id, scan_dir = run["scanId"], Path(run["scanDir"]) + inputs = [ + dispatch_discovery_worker( + state, home, scan_id=scan_id, scan_dir=scan_dir, name=f"discovery-{index}" + )[0] + for index in range(2) + ] + prompt, artifacts, _ = worker_paths(scan_dir, "reducer") + args = [ + "claim-deep-scan-dedup", + "--scan-id", + scan_id, + "--worker-id", + str(uuid.uuid4()), + "--prompt-path", + str(prompt), + "--artifact-dir", + str(artifacts), + ] + for worker in inputs: + args.extend(["--input-worker-id", worker]) + claimed = run_workbench(state, *args, environment={"CODEX_HOME": str(home)}) + dispatch_discovery_worker( + state, home, scan_id=scan_id, scan_dir=scan_dir, name="concurrent-discovery" + ) + replayed = run_workbench(state, *args, environment={"CODEX_HOME": str(home)}) + assert replayed["deepScan"]["mergeClaims"] == claimed["deepScan"]["mergeClaims"] + assert replayed["deepScan"]["dedupInputs"] == claimed["deepScan"]["dedupInputs"] + assert ( + replayed["deepScan"]["completionSequence"] == claimed["deepScan"]["completionSequence"] + 1 + ) diff --git a/plugins/codex-security/tests/test_deep_scan_publication_authority.py b/plugins/codex-security/tests/test_deep_scan_publication_authority.py new file mode 100644 index 000000000..b4ff3a0bd --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_publication_authority.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import copy +import json +import uuid +from argparse import Namespace + +import pytest +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +def stage_publication(scan, *, generation, result_path, title): + draft_dir = scan.scan_dir / "drafts" + draft_dir.mkdir(exist_ok=True) + draft_path = draft_dir / f"{uuid.uuid4()}.json" + checkpoint_path = draft_dir / f"{uuid.uuid4()}.checkpoint.json" + findings = copy.deepcopy(scan.findings) + findings[0]["title"] = title + draft = { + "manifest": json.loads((scan.scan_dir / "scan-manifest.json").read_text()), + "findings": {"findings": findings}, + "coverage": scan.coverage, + } + if generation is not None: + draft["deepScanPublication"] = { + "coordinatorGeneration": generation, + "resultPath": str(result_path), + } + draft_path.write_text(json.dumps(draft)) + checkpoint_path.write_text( + json.dumps({"scanId": scan.scan_id, "findings": findings, "coverage": scan.coverage}) + ) + return Namespace( + scan_id=scan.scan_id, + claim_token=None, + draft_path=str(draft_path), + checkpoint_path=str(checkpoint_path), + expected_draft_digest=None, + ) + + +@pytest.mark.parametrize("stale", ["generation", "aggregate", "unfenced"]) +def test_stale_coordinator_cannot_replace_newer_canonical_publication( + workbench_api, workbench_db, publication_scan, stale +): + scan = publication_scan() + old_result = add_worker(workbench_db, scan) + new_result = add_worker(workbench_db, scan) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET coordinator_generation = 3 WHERE scan_id = ?", + (scan.scan_id,), + ) + for result, completed_at in ((old_result, "2026-01-01"), (new_result, "2026-01-02")): + workbench_db.execute( + "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none', completed_at = ? " + "WHERE result_manifest_path = ?", + (completed_at, str(result)), + ) + current = stage_publication( + scan, generation=3, result_path=new_result, title="Current accepted aggregate" + ) + workbench_api["write_scan_draft"](workbench_db, current) + saved = { + path: path.read_bytes() + for path in scan.scan_dir.rglob("*.json") + if "drafts" not in path.parts + } + old = stage_publication( + scan, + generation=None if stale == "unfenced" else 2 if stale == "generation" else 3, + result_path=old_result if stale == "aggregate" else new_result, + title="Superseded aggregate", + ) + + with pytest.raises(SystemExit, match="coordinator|aggregate"): + workbench_api["write_scan_draft"](workbench_db, old) + + assert { + path: path.read_bytes() + for path in scan.scan_dir.rglob("*.json") + if "drafts" not in path.parts + } == saved diff --git a/plugins/codex-security/tests/test_deep_scan_publication_replay.py b/plugins/codex-security/tests/test_deep_scan_publication_replay.py new file mode 100644 index 000000000..e7c361569 --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_publication_replay.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +import json +import sqlite3 +import subprocess +import sys +from argparse import Namespace +from pathlib import Path + +import pytest +from test_accepted_publication_references import accept_reducer +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_publication_stop_interleavings import saved_selection + +_CRASH_PUBLICATION = """ +import json, os, runpy, sqlite3, sys +from argparse import Namespace + +api = runpy.run_path(sys.argv[1], run_name="publication_crash_test") +args = Namespace(**json.loads(sys.argv[3])) +boundary = sys.argv[4] + +class CrashConnection(sqlite3.Connection): + def commit(self): + completing = self.execute( + "SELECT status FROM scans WHERE id = ?", (args.scan_id,) + ).fetchone()[0] == "complete" + if completing and boundary == "sqlite-before": + os._exit(72) + super().commit() + if completing and boundary == "sqlite-after": + os._exit(73) + +connection = sqlite3.connect(sys.argv[2], factory=CrashConnection) +connection.row_factory = sqlite3.Row +connection.execute("PRAGMA foreign_keys = ON") +if boundary.startswith("sqlite-"): + api["complete_scan"](connection, Namespace( + scan_id=args.scan_id, claim_token=None, cost_json=None + )) +else: + saved = api["saved_results"] + original_write = saved.write_scan_local_bytes + def crash_after_write(root, relative, contents): + original_write(root, relative, contents) + if relative == boundary: + os._exit(71) + saved.write_scan_local_bytes = crash_after_write + api["write_scan_draft"](connection, args) +raise AssertionError("publication never reached the requested crash boundary") +""" + + +@pytest.mark.parametrize( + "boundary", + ["findings.json", "coverage.json", "scan-manifest.json", "sqlite-before", "sqlite-after"], +) +@pytest.mark.parametrize("selection_reason", [None, "saturated", "capped"]) +def test_publication_crash_replays_selected_input_without_stale_overwrite( + workbench_api, workbench_db, publication_scan, tmp_path, boundary, selection_reason +): + scan = publication_scan() + if selection_reason is None: + result_path = add_worker(workbench_db, scan) + else: + _, result_path, _ = accept_reducer(workbench_db, scan) + saved_selection(workbench_db, scan, result_path, reason=selection_reason) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET terminal_reason = ? WHERE scan_id = ?", + (selection_reason, scan.scan_id), + ) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET coordinator_generation = 3 WHERE scan_id = ?", + (scan.scan_id,), + ) + workbench_db.execute( + "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none' WHERE scan_id = ?", + (scan.scan_id,), + ) + current = stage_publication( + scan, generation=3, result_path=result_path, title="Selected aggregate" + ) + stale = stage_publication( + scan, generation=2, result_path=result_path, title="Obsolete coordinator draft" + ) + database_path = tmp_path / "publication.sqlite3" + with sqlite3.connect(database_path) as connection: + workbench_db.backup(connection) + connection.row_factory = sqlite3.Row + if boundary.startswith("sqlite-"): + workbench_api["write_scan_draft"](connection, current) + + child = subprocess.run( + [ + sys.executable, + "-c", + _CRASH_PUBLICATION, + str(Path(__file__).resolve().parents[1] / "scripts" / "workbench_db.py"), + str(database_path), + json.dumps(vars(current)), + boundary, + ], + capture_output=True, + text=True, + ) + assert child.returncode == {"sqlite-before": 72, "sqlite-after": 73}.get(boundary, 71), ( + child.stdout, + child.stderr, + ) + interrupted = { + path: path.read_bytes() + for path in scan.scan_dir.rglob("*") + if path.is_file() and "drafts" not in path.parts + } + with sqlite3.connect(database_path) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + row = connection.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone() + assert row["status"] == ("complete" if boundary == "sqlite-after" else "running") + assert bool(row["seal_manifest_digest"]) == (boundary == "sqlite-after") + run_before = dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) + workers_before = [ + dict(row) for row in connection.execute("SELECT * FROM deep_scan_workers") + ] + + with pytest.raises(SystemExit, match="coordinator|stopped"): + workbench_api["write_scan_draft"](connection, stale) + assert all(path.read_bytes() == contents for path, contents in interrupted.items()) + + if not boundary.startswith("sqlite-"): + workbench_api["write_scan_draft"](connection, current) + completed = workbench_api["complete_scan"]( + connection, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + )["scan"] + assert completed["progress"]["status"] == "complete" + assert completed["findingCount"] == 1 + assert dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) == run_before + assert [ + dict(row) for row in connection.execute("SELECT * FROM deep_scan_workers") + ] == workers_before + assert connection.execute("SELECT COUNT(*) FROM finding_occurrences").fetchone()[0] == 1 + published = { + path: path.read_bytes() + for path in scan.scan_dir.rglob("*") + if path.is_file() and "drafts" not in path.parts + } + workbench_api["complete_scan"]( + connection, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + assert all(path.read_bytes() == contents for path, contents in published.items()) + if boundary.startswith("sqlite-"): + assert published == interrupted + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert findings[0]["title"] == "Selected aggregate" diff --git a/plugins/codex-security/tests/test_deep_scan_successful_publication.py b/plugins/codex-security/tests/test_deep_scan_successful_publication.py index a83bb8359..405787c32 100644 --- a/plugins/codex-security/tests/test_deep_scan_successful_publication.py +++ b/plugins/codex-security/tests/test_deep_scan_successful_publication.py @@ -72,7 +72,7 @@ def create(*, mode="deep", scope="."): "INSERT INTO deep_scan_runs (scan_id, schema_version, workflow_version, " "status, phase, workers, subagents, stop_after_no_new, max_discovery_runs, " "manifest_path, terminal_reason, created_at, updated_at, completed_at) " - "VALUES (?, 1, 'publication-test', 'succeeded', 'terminal', 1, 0, 1, 1, " + "VALUES (?, 1, 'deep-security-scan/v1', 'succeeded', 'terminal', 1, 0, 1, 1, " "?, 'saturated', ?, ?, ?)", ( scan_id, @@ -174,6 +174,38 @@ def assert_published_aggregate(scan): assert (scan.scan_dir / "report.md").is_file() +def test_deep_publication_renders_each_source_remediation( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + finding = scan.findings[0] + first = copy.deepcopy(finding) + first.pop("provenance") + first["remediation"] = "Check the destination before writing the archive entry." + first["remediationTests"] = ["Reject an archive entry outside the destination."] + second = copy.deepcopy(first) + second["remediation"] = "Reject symbolic links before opening the destination." + second["remediationTests"] = ["Reject a symbolic link inside the destination."] + second["preventiveControls"] = ["Use a directory-relative file handle."] + finding["remediation"] = first["remediation"] + finding["remediationTests"] = first["remediationTests"] + finding["provenance"]["sourceFindings"] = [ + {"id": "review-1:0", "finding": first}, + {"id": "review-2:0", "finding": second}, + ] + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": scan.findings})) + + complete(workbench_api, workbench_db, scan) + + assert_published_aggregate(scan) + report = (scan.scan_dir / "report.md").read_text() + for source in (first, second): + assert report.count(source["remediation"]) == 1 + for test in source["remediationTests"]: + assert report.count(test) == 1 + assert "Use a directory-relative file handle." in report + + @pytest.mark.parametrize("scope", [".", "subdir"], ids=["repository", "scoped"]) def test_deep_publication_keeps_configured_scope_without_worker_observations( workbench_api, workbench_db, publication_scan, scope @@ -461,3 +493,32 @@ def test_standard_publication_preserves_deliberately_partial_coverage( complete(workbench_api, workbench_db, scan) assert_published_aggregate(scan) + + +def test_deep_publication_write_failure_keeps_original_terminal_cause( + workbench_api, workbench_db, publication_scan, monkeypatch +): + scan = publication_scan() + finalizer_globals = workbench_api["_write_prepared_scan_finalization"].__globals__ + write_bytes = finalizer_globals["write_scan_local_bytes"] + + def fail_report(scan_dir, relative_path, payload, **kwargs): + if relative_path == "report.md": + raise finalizer_globals["ContractError"]("Synthetic report write interruption") + return write_bytes(scan_dir, relative_path, payload, **kwargs) + + with monkeypatch.context() as patch: + patch.setitem(finalizer_globals, "write_scan_local_bytes", fail_report) + with pytest.raises(SystemExit, match="Synthetic report write interruption"): + complete(workbench_api, workbench_db, scan) + + assert ( + workbench_db.execute("SELECT status FROM scans WHERE id = ?", (scan.scan_id,)).fetchone()[0] + == "running" + ) + run = workbench_db.execute( + "SELECT status, terminal_reason FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,) + ).fetchone() + assert tuple(run) == ("succeeded", "saturated") + assert complete(workbench_api, workbench_db, scan)["progress"]["status"] == "complete" + assert_published_aggregate(scan) diff --git a/plugins/codex-security/tests/test_deep_scan_usage_owner.py b/plugins/codex-security/tests/test_deep_scan_usage_owner.py new file mode 100644 index 000000000..d2041342c --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_usage_owner.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import json +import sqlite3 +from datetime import datetime, timezone +from pathlib import Path + +from test_workbench_scan_usage import _event, _rollout, _state_graph +from workbench_test_support import run_workbench + + +def test_original_usage_turn_survives_join_and_coordinator_recovery(tmp_path: Path) -> None: + state = tmp_path / "state" + environment = { + "CODEX_HOME": str(tmp_path / "codex"), + "CODEX_SQLITE_HOME": str(tmp_path / "native"), + "CODEX_STATE_DB": "", + } + timestamp = datetime.now(timezone.utc) + rollout = _rollout( + tmp_path, + "shared-parent", + [ + _event(timestamp, "turn_context", {"turn_id": "original-turn", "model": "gpt-5.6-sol"}), + ], + ) + _state_graph(environment, {"shared-parent": rollout}, []) + target = tmp_path / "target" + target.mkdir() + begun = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "shared-parent", + "--target-path", + str(target), + "--scope", + ".", + "--scan-root", + str(tmp_path / "scans"), + environment=environment, + )["deepScan"] + # A writer recorded the original turn; joining and recovery only read it. + owner = { + "threadId": "shared-parent", + "turnId": "original-turn", + "startedAt": begun["createdAt"], + "dedicated": False, + } + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute( + "UPDATE deep_scan_runs SET usage_owner_json = ? WHERE scan_id = ?", + (json.dumps(owner), begun["scanId"]), + ) + observed = run_workbench( + state, + "get-deep-scan", + "--scan-id", + begun["scanId"], + "--thread-id", + "shared-parent", + environment=environment, + )["deepScan"] + assert observed["usageOwner"] == owner + assert owner["threadId"] == "shared-parent" + assert owner["turnId"] == "original-turn" + assert owner["dedicated"] is False + rollout.write_text( + rollout.read_text() + + json.dumps( + _event(timestamp, "turn_context", {"turn_id": "later-turn", "model": "gpt-6-astra"}) + ) + + "\n" + ) + joined = run_workbench( + state, + "begin-deep-scan", + "--scan-id", + begun["scanId"], + "--thread-id", + "shared-parent", + environment=environment, + )["deepScan"] + assert joined["usageOwner"] == owner + claim_args = [ + "claim-deep-scan-coordinator", + "--scan-id", + begun["scanId"], + "--thread-id", + "shared-parent", + ] + claimed = run_workbench(state, *claim_args, environment=environment)["deepScan"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute( + "UPDATE deep_scan_runs SET updated_at = '2000-01-01T00:00:00+00:00' WHERE scan_id = ?", + (begun["scanId"],), + ) + recovered = run_workbench(state, *claim_args, environment=environment)["deepScan"] + assert recovered["coordinatorGeneration"] == claimed["coordinatorGeneration"] + 1 + assert recovered["usageOwner"] == owner + other_target = tmp_path / "other-target" + other_target.mkdir() + other = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "shared-parent", + "--target-path", + str(other_target), + "--scope", + ".", + "--scan-root", + str(tmp_path / "scans"), + environment=environment, + )["deepScan"] + other_owner = {**owner, "turnId": "later-turn", "startedAt": other["createdAt"]} + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute( + "UPDATE deep_scan_runs SET usage_owner_json = ? WHERE scan_id = ?", + (json.dumps(other_owner), other["scanId"]), + ) + other = run_workbench( + state, + "get-deep-scan", + "--scan-id", + other["scanId"], + "--thread-id", + "shared-parent", + environment=environment, + )["deepScan"] + assert other["usageOwner"]["turnId"] == "later-turn" + original = run_workbench( + state, "get-scan", "--scan-id", begun["scanId"], environment=environment + )["scan"] + assert original["executionAttribution"]["owner"] == owner diff --git a/plugins/codex-security/tests/test_publication_stop_interleavings.py b/plugins/codex-security/tests/test_publication_stop_interleavings.py new file mode 100644 index 000000000..8a6c2ff4f --- /dev/null +++ b/plugins/codex-security/tests/test_publication_stop_interleavings.py @@ -0,0 +1,282 @@ +from __future__ import annotations + +import json +import sqlite3 +import subprocess +import sys +from argparse import Namespace +from pathlib import Path + +import pytest +from test_accepted_publication_references import accept_reducer +from test_checkpoint_publication_authority import save_disposition +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +def saved_selection(connection, scan, accepted, omitted=None, *, reason="saturated"): + selection = { + "version": 1, + "resultPath": accepted.relative_to(scan.scan_dir).as_posix(), + "resultSha256": accepted.stem, + "terminalReason": reason, + "omittedWorkerIds": [omitted.parent.name] if omitted is not None else [], + "selectedAt": scan.timestamp, + } + with connection: + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "finalization_input_json = ?, terminal_reason = ?, phase = 'terminal' " + "WHERE scan_id = ?", + (json.dumps(selection), reason, scan.scan_id), + ) + return selection + + +def stop_scan(api, connection, scan, cause): + if cause == "cancel": + return api["cancel_scan"](connection, Namespace(scan_id=scan.scan_id, thread_id=None)) + return api["fail_scan"]( + connection, + Namespace( + scan_id=scan.scan_id, + claim_token=None, + cost_json=None, + message="Scan stopped after reaching the configured cost limit.", + ), + ) + + +def published_bytes(scan): + return { + path.relative_to(scan.scan_dir).as_posix(): path.read_bytes() + for path in scan.scan_dir.rglob("*") + if path.is_file() and "drafts" not in path.relative_to(scan.scan_dir).parts + } + + +_CRASH_SELECTION_RECOVERY = """ +import os, runpy, sqlite3, sys +from argparse import Namespace + +api = runpy.run_path(sys.argv[1], run_name="selection_recovery_crash_test") +deep = api["deep_scan"] +deep.configure(deep.DeepScanDependencies(**{ + name: api["preserve_stopped_results_after_transition" + if name == "preserve_stopped_results" else name] + for name in deep.DeepScanDependencies.__dataclass_fields__ +})) + +class CrashConnection(sqlite3.Connection): + def commit(self): + if sys.argv[4] == "before": + os._exit(72) + super().commit() + os._exit(73) + +connection = sqlite3.connect(sys.argv[2], factory=CrashConnection) +connection.row_factory = sqlite3.Row +connection.execute("PRAGMA foreign_keys = ON") +deep.claim_deep_scan_coordinator(connection, Namespace( + scan_id=sys.argv[3], thread_id="fixture-owner", + claim_token=None, coordinator_generation=None, +)) +raise AssertionError("recovery never reached the requested commit boundary") +""" + + +@pytest.mark.parametrize("cause", ["cancel", "cost"]) +@pytest.mark.parametrize("cut", ["before-selection", "selected", "published", "sealed"]) +def test_stop_and_publication_keep_the_winning_terminal_outcome( + workbench_api, workbench_db, publication_scan, tmp_path, cause, cut +): + scan = publication_scan() + _, accepted, coverage = accept_reducer(workbench_db, scan) + omitted = add_worker(workbench_db, scan) + rejected = save_disposition(scan, omitted.parent, "reported") + omitted.write_text(json.dumps(rejected)) + save_disposition(scan, omitted.parent, "rejected") + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET merge_state = 'buffered' WHERE id = ?", + (omitted.parent.name,), + ) + if cut in {"before-selection", "selected"}: + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'reducing', " + "terminal_reason = NULL, completed_at = NULL WHERE scan_id = ?", + (scan.scan_id,), + ) + selection = ( + None + if cut == "before-selection" + else saved_selection(workbench_db, scan, accepted, omitted) + ) + staged = stage_publication( + scan, generation=3, result_path=accepted, title="Selected accepted aggregate" + ) + # The stop path must recover the accepted bytes and the rejection disposition. + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + evidence = {accepted: accepted.read_bytes(), omitted: omitted.read_bytes()} + database_path = tmp_path / "stop-publication.sqlite3" + with sqlite3.connect(database_path) as connection: + workbench_db.backup(connection) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + if cut in {"published", "sealed"}: + workbench_api["write_scan_draft"](connection, staged) + complete_args = Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + if cut == "sealed": + workbench_api["complete_scan"](connection, complete_args) + before_stop = published_bytes(scan) + if cut == "sealed": + with pytest.raises(SystemExit, match="running|completed"): + stop_scan(workbench_api, connection, scan, cause) + assert published_bytes(scan) == before_stop + else: + stop_scan(workbench_api, connection, scan, cause) + + # Reconnect after the winning commit, then deliver the old publisher response. + with sqlite3.connect(database_path) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + row = dict(connection.execute("SELECT * FROM scans").fetchone()) + run = dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) + assert row["status"] == ("complete" if cut == "sealed" else "failed") + assert bool(row["canceled_at"]) == (cause == "cancel" and cut != "sealed") + if cause == "cost" and cut != "sealed": + assert row["failure_message"] == ( + "Scan stopped after reaching the configured cost limit." + ) + assert json.loads(run["finalization_input_json"] or "null") == selection + if selection is not None: + assert run["terminal_reason"] == selection["terminalReason"] + if cut != "sealed" and cause == "cancel": + assert run["status"] == "canceled" + if cut in {"published", "sealed"}: + assert run["terminal_reason"] == "saturated" + frozen = published_bytes(scan) + with pytest.raises(SystemExit, match="stopped"): + workbench_api["write_scan_draft"](connection, staged) + if cut == "sealed": + workbench_api["complete_scan"](connection, complete_args) + else: + with pytest.raises(SystemExit): + workbench_api["complete_scan"](connection, complete_args) + workbench_api["preserve_scan_results"]( + connection, + Namespace( + scan_id=scan.scan_id, + claim_token=None, + thread_id=None, + coordinator_generation=None, + ), + ) + assert published_bytes(scan) == frozen + assert dict(connection.execute("SELECT * FROM scans").fetchone()) == row + assert dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) == run + assert all(path.read_bytes() == contents for path, contents in evidence.items()) + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == 1 + assert all( + finding.get("extensions", {}).get("candidateId") != "candidate-disposition" + for finding in findings + ) + if cut != "sealed": + published_coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert coverage["deferred"][0] in published_coverage["deferred"] + assert published_coverage["completeness"] == "partial" + + +@pytest.mark.parametrize("cut", ["before", "after"]) +def test_interrupted_selection_recovery_fences_observers_and_keeps_original_deadline( + workbench_api, workbench_db, publication_scan, tmp_path, cut +): + scan = publication_scan() + _, accepted, _ = accept_reducer(workbench_db, scan) + omitted = add_worker(workbench_db, scan) + selection = saved_selection(workbench_db, scan, accepted, omitted) + with workbench_db: + workbench_db.execute( + "UPDATE scans SET deep_scan_owner_thread_id = 'fixture-owner' WHERE id = ?", + (scan.scan_id,), + ) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', " + "completed_at = NULL, max_time_hours = 1, " + "created_at = '2000-01-01T00:00:00Z', updated_at = '2000-01-01T00:00:00Z' " + "WHERE scan_id = ?", + (scan.scan_id,), + ) + workbench_db.execute( + "UPDATE deep_scan_workers SET merge_state = 'buffered' WHERE id = ?", + (omitted.parent.name,), + ) + database_path = tmp_path / "recovery.sqlite3" + with sqlite3.connect(database_path) as connection: + workbench_db.backup(connection) + original = "\n".join(connection.iterdump()) + before = published_bytes(scan) + child = subprocess.run( + [ + sys.executable, + "-c", + _CRASH_SELECTION_RECOVERY, + str(Path(__file__).resolve().parents[1] / "scripts" / "workbench_db.py"), + str(database_path), + scan.scan_id, + cut, + ], + capture_output=True, + text=True, + ) + assert child.returncode == (72 if cut == "before" else 73), (child.stdout, child.stderr) + deep = workbench_api["deep_scan"] + args = Namespace( + scan_id=scan.scan_id, + thread_id="fixture-owner", + claim_token=None, + coordinator_generation=None, + ) + with sqlite3.connect(database_path) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + if cut == "before": + assert "\n".join(connection.iterdump()) == original + run = connection.execute("SELECT * FROM deep_scan_runs").fetchone() + assert run["coordinator_generation"] == (3 if cut == "before" else 4) + workers = [dict(row) for row in connection.execute("SELECT * FROM deep_scan_workers")] + attempts = [dict(row) for row in connection.execute("SELECT * FROM deep_scan_attempts")] + replayed = deep.claim_deep_scan_coordinator(connection, args) + assert replayed["coordinatorDisposition"] == ("adopted" if cut == "before" else "observing") + assert replayed["deepScan"]["coordinatorGeneration"] == 4 + assert replayed["deepScan"]["finalizationInput"] == selection + assert replayed["deepScan"]["createdAt"] == "2000-01-01T00:00:00Z" + assert replayed["deepScan"]["config"]["maxTimeHours"] == 1 + assert replayed["deepScan"]["phase"] == "terminal" + assert replayed["deepScan"]["terminalReason"] == selection["terminalReason"] + assert deep.deep_scan_deadline_reached( + connection.execute("SELECT * FROM deep_scan_runs").fetchone() + ) + stable = "\n".join(connection.iterdump()) + stale_claim = Namespace(**{**vars(args), "coordinator_generation": 3}) + with pytest.raises(SystemExit, match="generation"): + deep.claim_deep_scan_coordinator(connection, stale_claim) + assert "\n".join(connection.iterdump()) == stable + stale = stage_publication(scan, generation=3, result_path=accepted, title="Old coordinator") + with pytest.raises(SystemExit, match="generation"): + workbench_api["write_scan_draft"](connection, stale) + assert published_bytes(scan) == before + assert [ + dict(row) for row in connection.execute("SELECT * FROM deep_scan_workers") + ] == workers + assert [ + dict(row) for row in connection.execute("SELECT * FROM deep_scan_attempts") + ] == attempts + current = stage_publication( + scan, generation=4, result_path=accepted, title="Recovered selected aggregate" + ) + workbench_api["write_scan_draft"](connection, current) + assert accepted.read_bytes() == before[accepted.relative_to(scan.scan_dir).as_posix()] + assert "\n".join(connection.iterdump()) == stable diff --git a/plugins/codex-security/tests/test_reader_budget_cost.py b/plugins/codex-security/tests/test_reader_budget_cost.py new file mode 100644 index 000000000..09c2d2465 --- /dev/null +++ b/plugins/codex-security/tests/test_reader_budget_cost.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import json +import os +import sqlite3 +import subprocess +import sys + +import pytest +import workbench_test_support +from test_workbench_db import BUDGET_COST, BUDGET_WARNING, budget_scan_fixture + + +@pytest.mark.parametrize("cost_kind", ["full", "lower-bound", "invalid", "unexceeded", "ordinary"]) +def test_reader_budget_cost_preserves_unknown_totals( + workbench_api, monkeypatch, tmp_path, cost_kind +): + script = str(workbench_api["__file__"]) + monkeypatch.setattr(workbench_test_support, "SCRIPT", script) + monkeypatch.setenv("CODEX_HOME", str(tmp_path / "home")) + state_dir, _, scan_dir, scan_id, _ = budget_scan_fixture(tmp_path) + database = state_dir / "workbench.sqlite3" + cost = ( + BUDGET_COST + if cost_kind == "full" + else { + "lowerBound": None + if cost_kind == "invalid" + else {**BUDGET_COST, "estimatedUsd": 0.005} + if cost_kind == "unexceeded" + else BUDGET_COST + } + ) + + def snapshot(): + with sqlite3.connect(database) as connection: + assert connection.execute("SELECT COUNT(*) FROM deep_scan_attempts").fetchone() == (0,) + assert connection.execute( + "SELECT COUNT(*) FROM deep_scan_attempt_sessions" + ).fetchone() == (0,) + return list(connection.iterdump()), { + str(path.relative_to(scan_dir)): path.read_bytes() + for path in scan_dir.rglob("*") + if path.is_file() + } + + before = snapshot() + command = [ + sys.executable, + "-I", + "-B", + script, + "complete-scan" if cost_kind == "ordinary" else "complete-budget-exhausted-scan", + "--scan-id", + scan_id, + "--cost-json", + json.dumps(cost), + *([] if cost_kind == "ordinary" else ["--message", BUDGET_WARNING]), + ] + result = subprocess.run( + command, + env={**os.environ, "CODEX_SECURITY_STATE_DIR": str(state_dir)}, + text=True, + capture_output=True, + ) + after = snapshot() + if cost_kind in {"invalid", "unexceeded", "ordinary"}: + assert result.returncode != 0 + assert after == before + return + assert result.returncode == 0, result.stderr + public = json.loads(result.stdout)["scan"] + with sqlite3.connect(database) as connection: + status, saved = connection.execute("SELECT status, cost_json FROM scans").fetchone() + assert connection.execute( + "SELECT status, workflow_version, finalization_input_json FROM deep_scan_runs" + ).fetchone() == ("succeeded", "deep-security-scan/v1", None) + assert status == "complete" + if cost_kind == "lower-bound": + assert "cost" not in public + saved = json.loads(saved) + assert "cost" not in saved and "estimatedUsd" not in saved + assert saved["usage"]["coverage"] == "unavailable" + else: + assert public["cost"] == BUDGET_COST + assert json.loads(saved) == BUDGET_COST + manifest = json.loads((scan_dir / "scan-manifest.json").read_text()) + assert manifest["scan"]["sealedAt"] diff --git a/plugins/codex-security/tests/test_reader_budget_resume.py b/plugins/codex-security/tests/test_reader_budget_resume.py new file mode 100644 index 000000000..7340e6262 --- /dev/null +++ b/plugins/codex-security/tests/test_reader_budget_resume.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import json +import os +import sqlite3 +import subprocess +import sys +from argparse import Namespace +from pathlib import Path + +import pytest +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_stopped_result_version_boundary import snapshot + + +@pytest.mark.parametrize( + "state", + [ + "selected-budget", + "legacy-finished", + "parent-canceled", + "stopping", + "failed", + "canceled", + "unselected", + ], +) +def test_reader_resume_consumes_saved_selection_without_writes( + workbench_api, workbench_db, publication_scan, tmp_path, state +): + scan = publication_scan() + thread = "a23e657b-c14c-4da7-bd20-baa9e7579390" + workbench_api["set_scan_thread"]( + workbench_db, Namespace(scan_id=scan.scan_id, thread_id=thread) + ) + selection = json.dumps( + { + "version": 1, + "resultPath": None, + "resultSha256": None, + "terminalReason": "capped", + "omittedWorkerIds": [], + "selectedAt": scan.timestamp, + } + ) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET status = ?, cancel_requested = ?, " + "workflow_version = ?, finalization_input_json = ?", + ( + "running" + if state == "stopping" + else state + if state in {"failed", "canceled"} + else "succeeded", + int(state != "legacy-finished"), + "deep-security-scan/v1" if state == "legacy-finished" else "deep-security-scan/v2", + selection if state == "selected-budget" else None, + ), + ) + if state == "parent-canceled": + workbench_db.execute("UPDATE scans SET canceled_at = ?", (scan.timestamp,)) + state_dir = tmp_path / "state" + state_dir.mkdir(mode=0o700) + with sqlite3.connect(state_dir / "workbench.sqlite3") as disk: + workbench_db.backup(disk) + before = snapshot(workbench_db, scan.scan_dir) + script = Path(workbench_api["__file__"]) + result = subprocess.run( + [sys.executable, "-I", "-B", str(script), "get-cli-scan-resume", "--scan-id", scan.scan_id], + env={ + **os.environ, + "CODEX_SECURITY_STATE_DIR": str(state_dir), + "CODEX_HOME": str(tmp_path / "home"), + }, + capture_output=True, + text=True, + ) + with sqlite3.connect(state_dir / "workbench.sqlite3") as disk: + after = snapshot(disk, scan.scan_dir) + assert disk.execute("SELECT COUNT(*) FROM deep_scan_attempts").fetchone() == (0,) + assert disk.execute("SELECT COUNT(*) FROM deep_scan_attempt_sessions").fetchone() == (0,) + assert after == before + if state in {"selected-budget", "legacy-finished"}: + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout)["scanId"] == scan.scan_id + assert json.loads(result.stdout)["threadId"] == thread + else: + assert result.returncode != 0 + assert "cannot resume" in result.stderr diff --git a/plugins/codex-security/tests/test_reader_checkpoint_replay.py b/plugins/codex-security/tests/test_reader_checkpoint_replay.py new file mode 100644 index 000000000..32cd56e6c --- /dev/null +++ b/plugins/codex-security/tests/test_reader_checkpoint_replay.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +import copy +import json +from argparse import Namespace + +import pytest +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan +from workbench_test_support import write_checkpoint + + +def save_checkpoint(scan, result, *, rejected=False): + candidate = copy.deepcopy(scan.findings[0]) + candidate["extensions"] = {"candidateId": "candidate-disposition"} + retained = copy.deepcopy(candidate) + retained["extensions"] = {"candidateId": "candidate-retained"} + retained["identity"]["anchor"] = "independent-finding" + retained["locations"][0]["startLine"] = 20 + retained["locations"][0]["endLine"] = 21 + draft = { + "scanId": scan.scan_id, + "complete": False, + "findings": [retained] if rejected else [candidate, retained], + "coverage": { + **scan.coverage, + "surfaces": [ + { + "candidateId": "candidate-disposition", + "label": "Validated candidate disposition", + "disposition": "rejected" if rejected else "reported", + "receiptRefs": [], + } + ], + }, + } + checkpoint = write_checkpoint(result.parent / "checkpoints", draft) + (result.parent / "checkpoint-head.json").write_text(json.dumps({"checkpoint": checkpoint.name})) + return draft, checkpoint + + +def stop(workbench_api, connection, scan): + return workbench_api["fail_scan"]( + connection, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."), + )["scan"] + + +def preserve(workbench_api, connection, scan): + return workbench_api["preserve_scan_results"]( + connection, + Namespace( + scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None + ), + )["scan"] + + +@pytest.mark.parametrize("has_head", [False, True]) +def test_legacy_stop_does_not_create_frozen_checkpoint_metadata( + workbench_api, workbench_db, publication_scan, has_head +): + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + result = add_worker(workbench_db, scan, status="canceled") + draft, _ = save_checkpoint(scan, result) + result.write_text(json.dumps(draft)) + if not has_head: + (result.parent / "checkpoint-head.json").unlink() + + stopped = stop(workbench_api, workbench_db, scan) + assert stopped["findingCount"] == 2 + assert stopped["failureMessage"] == "Audit stopped." + assert preserve(workbench_api, workbench_db, scan)["findingCount"] == 2 + assert ( + workbench_api["recover_scan_results"](workbench_db, Namespace(scan_id=scan.scan_id))[ + "scan" + ]["findingCount"] + == 2 + ) + row = workbench_db.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone() + assert row["retained_source_digests_json"] + assert row["retained_checkpoint_heads_json"] is None + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text()) + assert "preservedCheckpointHeads" not in manifest["scan"] + + +@pytest.fixture +def frozen_stop(workbench_api, workbench_db, publication_scan, monkeypatch): + import finalize_scan_contract + + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + result = add_worker(workbench_db, scan, status="canceled") + previous, _ = save_checkpoint(scan, result) + result.write_text(json.dumps(previous)) + _, selected = save_checkpoint(scan, result, rejected=True) + directory = result.parent.relative_to(scan.scan_dir).as_posix() + heads = {directory: selected.relative_to(scan.scan_dir).as_posix()} + original_outputs = { + name: (scan.scan_dir / name).read_bytes() + for name in ("findings.json", "coverage.json", "scan-manifest.json") + } + write_bytes = finalize_scan_contract.write_scan_local_bytes + failed_writes = [] + + def obstruct_coverage(directory, relative, payload, **kwargs): + if relative != "coverage.json" or failed_writes: + return write_bytes(directory, relative, payload, **kwargs) + failed_writes.append(json.loads((directory / "findings.json").read_text())) + path = directory / relative + before = path.read_bytes() + path.unlink() + path.mkdir() + try: + return write_bytes(directory, relative, payload, **kwargs) + finally: + path.rmdir() + path.write_bytes(before) + + with monkeypatch.context() as patch: + patch.setattr(finalize_scan_contract, "write_scan_local_bytes", obstruct_coverage) + stop(workbench_api, workbench_db, scan) + assert len(failed_writes) == 1 + assert "scanId" in failed_writes[0] + assert all( + (scan.scan_dir / name).read_bytes() == contents + for name, contents in original_outputs.items() + ) + row = workbench_db.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone() + assert row["retained_source_digests_json"] + assert row["seal_manifest_digest"] is None + # A later writer freezes this existing checkpoint map before output writes. + # Seed its recorded input; this release only consumes that saved authority. + with workbench_db: + workbench_db.execute( + "UPDATE scans SET retained_checkpoint_heads_json = ? WHERE id = ?", + (json.dumps(heads, sort_keys=True), scan.scan_id), + ) + return scan, result, heads + + +@pytest.mark.parametrize("head_change", ["replaced", "removed", "missing-checkpoint"]) +def test_reader_replays_frozen_rejection_after_real_output_fault( + workbench_api, workbench_db, frozen_stop, head_change +): + scan, result, heads = frozen_stop + row = workbench_db.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone() + head = result.parent / "checkpoint-head.json" + if head_change == "replaced": + save_checkpoint(scan, result) + elif head_change == "removed": + head.unlink() + else: + head.write_text(json.dumps({"checkpoint": "a" * 64 + ".json"})) + + replayed = preserve(workbench_api, workbench_db, scan) + assert replayed["findingCount"] == 1 + assert replayed["failureMessage"] == "Audit stopped." + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert findings[0]["identity"]["anchor"] == "independent-finding" + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert coverage["completeness"] == "partial" + assert any( + surface.get("candidateId") == "candidate-disposition" + and surface.get("disposition") == "rejected" + for surface in coverage["surfaces"] + ) + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text()) + assert manifest["scan"]["preservedCheckpointHeads"] == heads + after = workbench_db.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone() + assert after["retained_source_digests_json"] == row["retained_source_digests_json"] + assert after["retained_checkpoint_heads_json"] == row["retained_checkpoint_heads_json"] + assert preserve(workbench_api, workbench_db, scan)["findingCount"] == 1 + + +def test_reader_requires_writer_to_select_new_recovery_heads( + workbench_api, workbench_db, frozen_stop +): + scan, result, _ = frozen_stop + assert preserve(workbench_api, workbench_db, scan)["findingCount"] == 1 + save_checkpoint(scan, result) + before_db = list(workbench_db.iterdump()) + before_files = {path: path.read_bytes() for path in scan.scan_dir.rglob("*") if path.is_file()} + with pytest.raises(SystemExit, match="newer version"): + workbench_api["recover_scan_results"](workbench_db, Namespace(scan_id=scan.scan_id)) + assert list(workbench_db.iterdump()) == before_db + assert {path: path.read_bytes() for path in scan.scan_dir.rglob("*") if path.is_file()} == ( + before_files + ) diff --git a/plugins/codex-security/tests/test_reader_publication_compatibility.py b/plugins/codex-security/tests/test_reader_publication_compatibility.py new file mode 100644 index 000000000..f14b94caa --- /dev/null +++ b/plugins/codex-security/tests/test_reader_publication_compatibility.py @@ -0,0 +1,241 @@ +"""A reader preserves publication identity recorded by a newer writer.""" + +from __future__ import annotations + +import copy +import hashlib +import json +from argparse import Namespace + +import pytest +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +def selected_publication(api, connection, scan): + result = scan.scan_dir / "accepted.json" + result.write_text( + json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": scan.findings, + "sourceCoverage": scan.coverage, + } + ) + ) + row = api["require_scan"](connection, scan.scan_id) + prepared = api["_prepare_scan_finalization"]( + scan.scan_dir, + expected_coverage_mode=api["expected_coverage_mode"](row), + completion_binding=api["workbench_completion_binding"](row, api["now"]()), + ) + manifest = copy.deepcopy(prepared[2]) + for key in ("completedAt", "sealedAt"): + manifest["scan"].pop(key, None) + encoded = json.dumps( + [manifest, prepared[3], prepared[4]], sort_keys=True, separators=(",", ":") + ).encode() + selection = { + "version": 1, + "resultPath": result.name, + "resultSha256": hashlib.sha256(result.read_bytes()).hexdigest(), + "publicationSha256": hashlib.sha256(encoded).hexdigest(), + "terminalReason": "saturated", + "omittedWorkerIds": [], + "selectedAt": scan.timestamp, + } + with connection: + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "coordinator_generation = 2, finalization_input_json = ? WHERE scan_id = ?", + (json.dumps(selection), scan.scan_id), + ) + return result + + +@pytest.mark.parametrize("damage", [None, "findings", "coverage", "selected"]) +def test_reader_completion_validates_saved_publication( + workbench_api, workbench_db, publication_scan, damage +): + scan = publication_scan() + selected = selected_publication(workbench_api, workbench_db, scan) + if damage == "selected": + selected.write_bytes(selected.read_bytes() + b"\n") + elif damage is not None: + path = scan.scan_dir / f"{damage}.json" + document = json.loads(path.read_bytes()) + if damage == "findings": + document["findings"][0]["remediation"] = "Substituted repair." + else: + document["completeness"] = "partial" + document["deferred"] = [ + {"id": "changed-coverage", "reason": "Substituted unresolved review."} + ] + path.write_text(json.dumps(document)) + before = {p: p.read_bytes() for p in scan.scan_dir.rglob("*.json")} + state = "\n".join(workbench_db.iterdump()) + args = Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + if damage is None: + workbench_api["complete_scan"](workbench_db, args) + sealed = {p: p.read_bytes() for p in scan.scan_dir.rglob("*.json")} + workbench_api["complete_scan"](workbench_db, args) + assert {p: p.read_bytes() for p in sealed} == sealed + else: + with pytest.raises(SystemExit, match="publication|changed after acceptance"): + workbench_api["complete_scan"](workbench_db, args) + assert "\n".join(workbench_db.iterdump()) == state + assert {p: p.read_bytes() for p in before} == before + + +@pytest.mark.parametrize("damage", ["draft", "selected"]) +def test_reader_republication_checks_identity_before_writing_checkpoints( + workbench_api, workbench_db, publication_scan, damage +): + scan = publication_scan() + result = selected_publication(workbench_api, workbench_db, scan) + staged = stage_publication( + scan, + generation=2, + result_path=result, + title="Substituted aggregate" if damage == "draft" else scan.findings[0]["title"], + ) + if damage == "selected": + result.write_bytes(result.read_bytes() + b"\n") + before = {p: p.read_bytes() for p in scan.scan_dir.rglob("*.json")} + state = "\n".join(workbench_db.iterdump()) + with pytest.raises( + workbench_api["ContractError"], match="publication|changed after acceptance" + ): + workbench_api["write_scan_draft"](workbench_db, staged) + assert {p: p.read_bytes() for p in scan.scan_dir.rglob("*.json")} == before + assert "\n".join(workbench_db.iterdump()) == state + + +@pytest.mark.parametrize( + "damage", + [ + None, + "findings", + "coverage", + "selected", + "target", + "displayName", + "id", + "includePaths", + "excludePaths", + "null-digest", + "bad-digest", + "claim", + "protocol", + ], +) +@pytest.mark.parametrize("omitted_scope", [False, True]) +def test_reader_budget_validates_publication_before_changing_the_projection( + workbench_api, workbench_db, publication_scan, damage, omitted_scope +): + from test_workbench_db import BUDGET_COST, BUDGET_WARNING + + scan = publication_scan() + scan.coverage["completeness"] = "partial" + scan.coverage["deferred"] = [{"id": "remaining", "reason": "A review remains unresolved."}] + (scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage)) + if omitted_scope: + path = scan.scan_dir / "scan-manifest.json" + manifest = json.loads(path.read_bytes()) + manifest["scan"]["scope"].pop("includePaths") + manifest["scan"]["scope"].pop("excludePaths") + path.write_text(json.dumps(manifest)) + selected = selected_publication(workbench_api, workbench_db, scan) + recipe = json.loads(workbench_api["require_scan"](workbench_db, scan.scan_id)["recipe_json"]) + recipe["maxCostUsd"] = 0.005 + with workbench_db: + workbench_db.execute( + "UPDATE scans SET recipe_json = ? WHERE id = ?", (json.dumps(recipe), scan.scan_id) + ) + if damage == "selected": + selected.write_bytes(selected.read_bytes() + b"\n") + elif damage in { + "findings", + "coverage", + "target", + "displayName", + "id", + "includePaths", + "excludePaths", + }: + name = damage if damage in {"findings", "coverage"} else "scan-manifest" + path = scan.scan_dir / f"{name}.json" + document = json.loads(path.read_bytes()) + if damage == "findings": + document["findings"][0]["remediation"] = "Substituted repair." + elif damage == "coverage": + document["deferred"][0]["reason"] = "Substituted unresolved review." + elif damage in {"includePaths", "excludePaths"}: + document["scan"]["scope"][damage] = ["another-path"] + elif damage == "id": + document["scan"]["id"] = "95a98220-0653-47cb-b6b8-b5f125a5b4e7" + else: + document["scan"]["target"]["targetId" if damage == "target" else damage] = ( + "another-target" + ) + path.write_text(json.dumps(document)) + elif damage in {"null-digest", "bad-digest"}: + selection = json.loads( + workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + ) + selection["publicationSha256"] = None if damage == "null-digest" else "0" * 64 + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET finalization_input_json = ?", (json.dumps(selection),) + ) + elif damage == "claim": + with workbench_db: + workbench_db.execute( + "UPDATE scans SET handoff_claim_token = '407e80b8-a8a0-412a-8ec7-d4954afba06a'" + ) + elif damage == "protocol": + with workbench_db: + workbench_db.execute("UPDATE deep_scan_runs SET workflow_version = 'future/v9'") + + def snapshot(): + return list(workbench_db.iterdump()), { + p.relative_to(scan.scan_dir): p.read_bytes() + for p in scan.scan_dir.rglob("*") + if p.is_file() + } + + before = snapshot() + run_before = dict(workbench_db.execute("SELECT * FROM deep_scan_runs").fetchone()) + args = Namespace( + scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=BUDGET_WARNING + ) + if damage is not None: + with pytest.raises(SystemExit): + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + assert snapshot() == before + return + + result = workbench_api["complete_budget_exhausted_scan"](workbench_db, args)["scan"] + assert result["progress"]["status"] == "complete" + run_after = dict(workbench_db.execute("SELECT * FROM deep_scan_runs").fetchone()) + selection_before = json.loads(run_before.pop("finalization_input_json")) + selection_after = json.loads(run_after.pop("finalization_input_json")) + assert run_after == run_before + assert selection_after.pop("publicationSha256") != selection_before.pop("publicationSha256") + assert selection_after == selection_before + coverage = json.loads((scan.scan_dir / "coverage.json").read_bytes()) + assert coverage["completeness"] == "partial" + assert scan.coverage["deferred"][0] in coverage["deferred"] + assert any(item["id"] == "scan-cost-limit" for item in coverage["deferred"]) + assert [f["remediation"] for f in result["findings"]] == [ + f["remediation"] for f in scan.findings + ] + sealed = snapshot() + workbench_api["complete_scan"]( + workbench_db, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + assert snapshot() == sealed + with pytest.raises(SystemExit, match="Only a running CLI Deep Scan"): + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + assert snapshot() == sealed diff --git a/plugins/codex-security/tests/test_reader_release_persistence.py b/plugins/codex-security/tests/test_reader_release_persistence.py new file mode 100644 index 000000000..f6f0dac40 --- /dev/null +++ b/plugins/codex-security/tests/test_reader_release_persistence.py @@ -0,0 +1,261 @@ +from __future__ import annotations + +import json +import os +import sqlite3 +import subprocess +import sys +from pathlib import Path + +import pytest +from test_workbench_deep_scan import commit_reducer, dispatch_discovery_worker, upsert_worker +from workbench_test_support import SCRIPT, run_workbench + + +def introduced_metadata(state: Path, scan_dir: Path) -> dict: + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.row_factory = sqlite3.Row + tables = { + row[0] + for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'") + } + counts = { + name: connection.execute(f"SELECT COUNT(*) FROM {name}").fetchone()[0] + for name in ( + "deep_scan_attempts", + "deep_scan_attempt_sessions", + "deep_scan_merge_claims", + ) + if name in tables + } + run = connection.execute("SELECT * FROM deep_scan_runs").fetchone() + context = { + name: run[name] + for name in ( + "discovery_user_context", + "usage_owner_json", + "finalization_input_json", + "execution_settings_json", + ) + if name in run.keys() + } + inputs = list(connection.execute("SELECT * FROM deep_scan_dedup_inputs")) + references = [ + { + name: row[name] + for name in ("result_manifest_path", "result_manifest_sha256", "attempt") + if name in row.keys() and row[name] is not None + } + for row in inputs + ] + return { + "tables": counts, + "run": context, + "input_references": [row for row in references if row], + "accepted_copies": sorted( + str(path.relative_to(scan_dir)) for path in scan_dir.rglob("checkpoints/*.json") + ), + } + + +@pytest.mark.parametrize("version", ["deep-security-scan/v1", "deep-scan-mcp/v1"]) +def test_legacy_execution_defers_new_persistence(tmp_path: Path, version: str) -> None: + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + (home / "codex-security").mkdir(parents=True) + (home / "codex-security/config.toml").write_text("[deep_scan]\nmax_time_hours = 3\n") + run = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "reader-owner", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + "--workflow-version", + version, + "--user-context", + "Review the supplied input.", + environment={"CODEX_HOME": str(home)}, + )["deepScan"] + scan_id, scan_dir = run["scanId"], Path(run["scanDir"]) + stages = {"begin": introduced_metadata(state, scan_dir)} + workers = [] + for index in range(2): + worker, prompt, artifacts, result = dispatch_discovery_worker( + state, + home, + scan_id=scan_id, + scan_dir=scan_dir, + name=f"discovery-{index}", + succeed=False, + ) + upsert_worker( + state, + home, + scan_id=scan_id, + worker_id=worker, + kind="discovery", + status="running", + prompt_path=prompt, + artifact_dir=artifacts, + attempt=2, + thread_id=f"replacement-{index}", + ) + result.write_text("{}\n") + upsert_worker( + state, + home, + scan_id=scan_id, + worker_id=worker, + kind="discovery", + status="succeeded", + prompt_path=prompt, + artifact_dir=artifacts, + attempt=2, + thread_id=f"replacement-{index}", + result_path=result, + ) + workers.append(worker) + stages["discovery"] = introduced_metadata(state, scan_dir) + committed = commit_reducer( + state, + home, + scan_id=scan_id, + scan_dir=scan_dir, + name="dedup-1", + input_worker_ids=workers, + new_findings_count=0, + ) + stages["merge"] = introduced_metadata(state, scan_dir) + assert committed["noNewStreak"] == 2 + assert [item["discoveryWorkerId"] for item in committed["dedupInputs"]] == workers + assert all( + item["mergeState"] == "merged" + for item in committed["workers"] + if item["kind"] == "discovery" + ) + stopped = run_workbench( + state, + "fail-deep-scan", + "--scan-id", + scan_id, + "--message", + "Original reader stop.", + environment={"CODEX_HOME": str(home)}, + )["deepScan"] + stages["stop"] = introduced_metadata(state, scan_dir) + assert stopped["status"] == "failed" + assert "Original reader stop." in stopped["error"] + assert stopped["workflowVersion"] == version + assert stopped["createdAt"] == run["createdAt"] + assert stopped["config"]["maxTimeHours"] == 3 + assert stopped["userContext"] == "Review the supplied input." + print(json.dumps({"version": version, "stages": stages}, sort_keys=True)) + for stage in stages.values(): + assert all(count == 0 for count in stage["tables"].values()), stages + assert all(value is None for value in stage["run"].values()), stages + assert stage["input_references"] == [], stages + assert stage["accepted_copies"] == [], stages + + +def test_upgraded_context_schema_rejects_creation_before_mutation(tmp_path: Path) -> None: + state, target, next_target = tmp_path / "state", tmp_path / "target", tmp_path / "next" + target.mkdir() + next_target.mkdir() + scan_root = tmp_path / "scans" + environment = {"CODEX_HOME": str(tmp_path / "home")} + run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "reader-owner", + "--target-path", + str(target), + "--scan-root", + str(scan_root), + environment=environment, + ) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + # An upgraded database distinguishes captured NULL from an uncaptured legacy field. + connection.execute("ALTER TABLE deep_scan_runs ADD COLUMN discovery_user_context TEXT") + before = "\n".join(connection.iterdump()) + before_paths = sorted(str(path.relative_to(scan_root)) for path in scan_root.rglob("*")) + rejected = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "reader-owner", + "--target-path", + str(next_target), + "--scan-root", + str(scan_root), + "--user-context", + "Explicit new review input.", + check=False, + environment=environment, + ) + assert rejected["returncode"] != 0 + assert "newer version" in rejected["stderr"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + assert "\n".join(connection.iterdump()) == before + assert sorted(str(path.relative_to(scan_root)) for path in scan_root.rglob("*")) == before_paths + + +def test_selected_replay_keeps_publication_path_validation(tmp_path: Path) -> None: + state, target = tmp_path / "state", tmp_path / "target" + target.mkdir() + environment = {"CODEX_HOME": str(tmp_path / "home")} + run = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "reader-owner", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + environment=environment, + )["deepScan"] + selection = { + "version": 1, + "resultPath": None, + "resultSha256": None, + "terminalReason": "capped", + "omittedWorkerIds": [], + "selectedAt": "2000-01-01T00:00:00Z", + } + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "finalization_input_json = ?, phase = 'terminal', " + "created_at = '2000-01-01T00:00:00Z'", + (json.dumps(selection),), + ) + before = "\n".join(connection.iterdump()) + rejected = subprocess.run( + [ + sys.executable, + "-I", + "-B", + "-c", + "import runpy,sys; p=sys.argv.pop(1); runpy.run_path(p)['main'](select_finalization=True)", + str(SCRIPT), + "finish-deep-scan", + "--scan-id", + run["scanId"], + "--terminal-reason", + "capped", + "--manifest-path", + str(Path(run["scanDir"]) / "wrong-manifest.json"), + ], + capture_output=True, + text=True, + input=json.dumps({"resultPath": None}), + env={**os.environ, **environment, "CODEX_SECURITY_STATE_DIR": str(state)}, + ) + assert rejected.returncode != 0 + assert "parent manifest" in rejected.stderr + with sqlite3.connect(state / "workbench.sqlite3") as connection: + assert "\n".join(connection.iterdump()) == before diff --git a/plugins/codex-security/tests/test_reader_sealed_budget_resume.py b/plugins/codex-security/tests/test_reader_sealed_budget_resume.py new file mode 100644 index 000000000..d23e55989 --- /dev/null +++ b/plugins/codex-security/tests/test_reader_sealed_budget_resume.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import json +import os +import sqlite3 +import subprocess +import sys + +import pytest +import workbench_test_support +from test_workbench_db import BUDGET_COST, BUDGET_WARNING, budget_scan_fixture + + +@pytest.mark.parametrize( + "state", + [ + "retry", + "changed-findings", + "wrong-scan", + "running-discovery", + "canceled", + "other-owner", + "complete", + ], +) +def test_reader_replays_sealed_budget_without_new_writer_state( + workbench_api, monkeypatch, tmp_path, state +): + script = str(workbench_api["__file__"]) + monkeypatch.setattr(workbench_test_support, "SCRIPT", script) + monkeypatch.setenv("CODEX_HOME", str(tmp_path / "home")) + state_dir, _, scan_dir, scan_id, _ = budget_scan_fixture(tmp_path) + environment = {**os.environ, "CODEX_SECURITY_STATE_DIR": str(state_dir)} + budget_args = [ + "complete-budget-exhausted-scan", + "--scan-id", + scan_id, + "--cost-json", + json.dumps(BUDGET_COST), + "--message", + BUDGET_WARNING, + ] + cut_program = """ +import os, runpy, sys +script, *args = sys.argv[1:] +api = runpy.run_path(script, run_name="sealed_budget_test") +namespace = api["main"].__globals__ +original = namespace["_write_prepared_scan_finalization"] +def after_seal(*args, **kwargs): + original(*args, **kwargs) + os._exit(86) +namespace["_write_prepared_scan_finalization"] = after_seal +sys.argv = [script, *args] +api["main"]() +""" + cut = subprocess.run( + [sys.executable, "-I", "-B", "-c", cut_program, script, *budget_args], + env=environment, + text=True, + capture_output=True, + ) + assert cut.returncode == 86, cut.stderr + manifest_path = scan_dir / "scan-manifest.json" + manifest = json.loads(manifest_path.read_text()) + assert manifest["scan"]["sealedAt"] + assert manifest["scan"]["artifacts"] + database = state_dir / "workbench.sqlite3" + with sqlite3.connect(database) as connection: + assert connection.execute("SELECT status, seal_manifest_digest FROM scans").fetchone() == ( + "running", + None, + ) + assert connection.execute( + "SELECT status, workflow_version, finalization_input_json FROM deep_scan_runs" + ).fetchone() == ("succeeded", "deep-security-scan/v1", None) + + def command(*args): + return subprocess.run( + [sys.executable, "-I", "-B", script, *args], + env=environment, + text=True, + capture_output=True, + ) + + if state == "changed-findings": + path = scan_dir / "findings.json" + findings = json.loads(path.read_text()) + findings["findings"].append({"title": "Changed after sealing"}) + path.write_text(json.dumps(findings)) + elif state == "wrong-scan": + manifest["scan"]["id"] = "69078890-d24c-4416-a6fa-c286825bef88" + manifest_path.write_text(json.dumps(manifest)) + elif state == "running-discovery": + with sqlite3.connect(database) as connection: + connection.execute("UPDATE deep_scan_runs SET status = 'running'") + elif state == "other-owner": + with sqlite3.connect(database) as connection: + connection.execute( + "UPDATE scans SET handoff_claim_token = 'a3292ae4-9b47-430f-8ed6-73ff73db575c'" + ) + elif state == "canceled": + result = command("cancel-scan", "--scan-id", scan_id) + assert result.returncode == 0, result.stderr + elif state == "complete": + result = command( + "complete-scan", "--scan-id", scan_id, "--cost-json", json.dumps(BUDGET_COST) + ) + assert result.returncode == 0, result.stderr + + def snapshot(): + with sqlite3.connect(database) as connection: + assert connection.execute("SELECT COUNT(*) FROM deep_scan_attempts").fetchone() == (0,) + assert connection.execute( + "SELECT COUNT(*) FROM deep_scan_attempt_sessions" + ).fetchone() == (0,) + return list(connection.iterdump()), { + path.relative_to(scan_dir): path.read_bytes() + for path in scan_dir.rglob("*") + if path.is_file() + } + + before = snapshot() + result = command(*budget_args) + after = snapshot() + if state == "retry": + assert result.returncode == 0, result.stderr + assert after[1] == before[1] + with sqlite3.connect(database) as connection: + status, digest = connection.execute( + "SELECT status, seal_manifest_digest FROM scans" + ).fetchone() + assert status == "complete" + assert digest + else: + assert result.returncode != 0 + assert after == before diff --git a/plugins/codex-security/tests/test_reader_settings_claim.py b/plugins/codex-security/tests/test_reader_settings_claim.py new file mode 100644 index 000000000..0541f6048 --- /dev/null +++ b/plugins/codex-security/tests/test_reader_settings_claim.py @@ -0,0 +1,180 @@ +"""Recorded settings are checked before an expired coordinator changes state.""" + +from __future__ import annotations + +import datetime +import json +import sqlite3 +import uuid +from pathlib import Path + +import pytest +from workbench_test_support import run_workbench + + +def database_snapshot(state: Path) -> str: + with sqlite3.connect(state / "workbench.sqlite3") as connection: + return "\n".join(connection.iterdump()) + + +@pytest.mark.parametrize( + "version", ["deep-security-scan/v1", "deep-scan-mcp/v1", "deep-security-scan/v2"] +) +@pytest.mark.parametrize("saved", ["unsupported", "missing-home", "valid", "absent"]) +@pytest.mark.parametrize("live", [False, True], ids=["expired-owner", "live-observer"]) +def test_reader_checks_recorded_settings_before_adoption( + tmp_path: Path, version: str, saved: str, live: bool +) -> None: + state, target = tmp_path / "state", tmp_path / "target" + target.mkdir() + run = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "original-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + "--workflow-version", + "deep-security-scan/v1" if version == "deep-security-scan/v2" else version, + )["deepScan"] + scan_dir = Path(run["scanDir"]) + worker_dir = scan_dir / "artifacts" / "deep_discovery" / "worker" + worker_dir.mkdir(parents=True) + prompt = worker_dir / "prompt.md" + prompt.write_text("Original discovery input") + run_workbench( + state, + "upsert-deep-scan-worker", + "--scan-id", + run["scanId"], + "--worker-id", + str(uuid.uuid4()), + "--kind", + "discovery", + "--status", + "running", + "--prompt-path", + str(prompt), + "--artifact-dir", + str(worker_dir), + "--attempt", + "1", + ) + path = worker_dir.parent / "execution-settings.json" + if saved != "absent": + settings = {"codexPath": "/fixture/codex", "codexHome": "/fixture/original-home"} + if saved == "missing-home": + del settings["codexHome"] + path.write_text( + json.dumps({"version": 99 if saved == "unsupported" else 1, "settings": settings}) + ) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute("UPDATE deep_scan_runs SET workflow_version = ?", (version,)) + if version == "deep-security-scan/v2" and saved != "absent": + connection.execute( + "UPDATE deep_scan_runs SET execution_settings_json = ?", (path.read_text(),) + ) + connection.execute( + "UPDATE deep_scan_runs SET coordinator_generation = 2, updated_at = ?", + ( + datetime.datetime.now(datetime.timezone.utc).isoformat() + if live + else "2000-01-01T00:00:00Z", + ), + ) + before = database_snapshot(state) + files = { + item.relative_to(scan_dir): item.read_bytes() + for item in scan_dir.rglob("*") + if item.is_file() + } + rejects = version == "deep-security-scan/v2" + result = run_workbench( + state, + "claim-deep-scan-coordinator", + "--scan-id", + run["scanId"], + "--thread-id", + "original-thread", + check=not rejects, + ) + if rejects: + assert result["returncode"] != 0 + assert "newer version to resume execution" in result["stderr"] + assert database_snapshot(state) == before + else: + observed = result + assert observed["coordinatorDisposition"] == ("observing" if live else "adopted") + assert observed["deepScan"]["workflowVersion"] == version + if live: + assert database_snapshot(state) == before + assert { + item.relative_to(scan_dir): item.read_bytes() + for item in scan_dir.rglob("*") + if item.is_file() + } == files + + +@pytest.mark.parametrize("workflow", ["deep-security-scan/v1", "deep-security-scan/v2"]) +@pytest.mark.parametrize("bound", [False, True]) +def test_reader_private_settings_projection_preserves_public_output( + tmp_path: Path, workflow: str, bound: bool +) -> None: + import os + import subprocess + import sys + + from workbench_test_support import SCRIPT + + state, target = tmp_path / "state", tmp_path / "target" + target.mkdir() + run = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "reader-owner", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + )["deepScan"] + settings = { + "version": 1, + "settings": {"codexPath": "/fixture/codex", "codexHome": "/fixture/original-home"}, + } + with sqlite3.connect(state / "workbench.sqlite3") as connection: + assert connection.execute( + "SELECT execution_settings_json FROM deep_scan_runs" + ).fetchone() == (None,), "the reader does not write creation settings" + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = ?, execution_settings_json = ?", + (workflow, json.dumps(settings) if bound else None), + ) + before = database_snapshot(state) + args = ["get-deep-scan", "--scan-id", run["scanId"], "--thread-id", "reader-owner"] + public = run_workbench(state, *args) + assert "executionSettings" not in public["deepScan"] + result = subprocess.run( + [ + sys.executable, + "-c", + ( + "import runpy, sys; script=sys.argv.pop(1); " + "runpy.run_path(script)['main'](with_execution_settings=True)" + ), + str(SCRIPT), + *args, + ], + env={**os.environ, "CODEX_SECURITY_STATE_DIR": str(state)}, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + private = json.loads(result.stdout) + assert private["deepScan"].pop("executionSettings") == (settings if bound else None) + assert private == public + assert database_snapshot(state) == before + assert not (Path(run["scanDir"]) / "artifacts/deep_discovery/execution-settings.json").exists() diff --git a/plugins/codex-security/tests/test_reader_unsealed_budget.py b/plugins/codex-security/tests/test_reader_unsealed_budget.py new file mode 100644 index 000000000..c6bdf5eb0 --- /dev/null +++ b/plugins/codex-security/tests/test_reader_unsealed_budget.py @@ -0,0 +1,231 @@ +from __future__ import annotations + +import copy +import hashlib +import json +from argparse import Namespace + +import pytest +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_workbench_db import BUDGET_COST + + +@pytest.fixture +def legacy_budget(workbench_db, publication_scan): + def create(*, named=True, duplicate=True): + scan = publication_scan() + worker_id = add_worker(workbench_db, scan).parent.name + result = ( + scan.scan_dir / "artifacts/deep_discovery/workers/discovery-0001/output/result.json" + ) + result.parent.mkdir(parents=True) + receipt = result.parent / "artifacts/review.md" + receipt.parent.mkdir() + receipt.write_text("Synthetic accepted review evidence.\n") + surfaces = [ + { + "label": "Filesystem boundary", + "disposition": "needs_follow_up", + "notes": "The race remains untested.", + "reason": "The caller policy is unknown.", + "receiptRefs": ["artifacts/review.md"], + "provenance": {"source": "independent-review"}, + }, + { + "label": "Configuration boundary", + "disposition": "rejected", + "reason": "Only trusted configuration reaches this path.", + "receiptRefs": ["artifacts/review.md"], + "provenance": {"source": "independent-review"}, + }, + ] + if named: + for index, surface in enumerate(surfaces): + surface["id"] = f"source-{index}" + source = { + "scanId": scan.scan_id, + "complete": True, + "findings": scan.findings, + "coverage": {**scan.coverage, "completeness": "partial", "surfaces": surfaces}, + } + contents = json.dumps(source).encode() + digest = hashlib.sha256(contents).hexdigest() + accepted = result.parent / "checkpoints" / f"{digest}.json" + accepted.parent.mkdir() + accepted.write_bytes(contents) + result.write_text("Later unaccepted output must not replace the accepted checkpoint.") + provenance = {"workerId": worker_id, "attempt": 1} + prefix = f"{worker_id}-attempt-1" + # This is the persisted old-writer shape at its draft commit: one copy + # lacks descriptive provenance; an interrupted projection adds another. + projected = [] + for index, surface in enumerate(surfaces): + item = copy.deepcopy(surface) + item["id"] = f"{prefix}-surface-{index + 1}" + item["receiptRefs"] = [receipt.relative_to(scan.scan_dir).as_posix()] + item["provenance"] = { + **provenance, + **({"sourceId": surface["id"]} if named else {}), + } + projected.append(item) + coverage = { + **scan.coverage, + "completeness": "partial", + "surfaces": projected, + "reviews": [{**provenance, "completeness": "partial"}], + "deferred": [ + { + "id": f"{prefix}-unmerged", + "provenance": provenance, + "reason": "This accepted discovery was not merged before the scan reached its cost limit.", + }, + { + "id": "scan-cost-limit", + "reason": "Validation was deferred because the scan reached its cost limit.", + }, + ], + } + if duplicate: + for item in copy.deepcopy(projected): + item["provenance"]["source"] = "independent-review" + projected.append(item) + (scan.scan_dir / "coverage.json").write_text(json.dumps(coverage)) + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + selection = { + "version": 1, + "resultPath": None, + "resultSha256": None, + "terminalReason": "capped", + "omittedWorkerIds": [worker_id], + "selectedAt": scan.timestamp, + } + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_workers SET merge_state = 'buffered', artifact_dir = ?, " + "result_manifest_path = ? WHERE id = ?", + (str(result.parent), str(result), worker_id), + ) + workbench_db.execute( + "INSERT INTO deep_scan_attempts (scan_id, worker_id, attempt, status, started_at, " + "completed_at, accepted_result_path, accepted_result_sha256) " + "VALUES (?, ?, 1, 'succeeded', ?, ?, ?, ?)", + (scan.scan_id, worker_id, scan.timestamp, scan.timestamp, str(accepted), digest), + ) + workbench_db.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "terminal_reason = 'capped', finalization_input_json = ?", + (json.dumps(selection),), + ) + scan.accepted, scan.contents, scan.source = accepted, contents, source + scan.receipt, scan.selection, scan.projected = receipt, selection, projected + return scan + + return create + + +def snapshot(connection, scan): + return list(connection.iterdump()), { + str(path.relative_to(scan.scan_dir)): path.read_bytes() + for path in scan.scan_dir.rglob("*") + if path.is_file() + } + + +@pytest.mark.parametrize("named", [False, True]) +@pytest.mark.parametrize("duplicate", [False, True]) +def test_reader_completes_old_unsealed_budget_and_replays( + workbench_api, workbench_db, legacy_budget, named, duplicate +): + scan = legacy_budget(named=named, duplicate=duplicate) + run = dict(workbench_db.execute("SELECT * FROM deep_scan_runs").fetchone()) + attempts = list(workbench_db.execute("SELECT * FROM deep_scan_attempts")) + workbench_api["complete_budget_exhausted_scan"]( + workbench_db, + Namespace(scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=None), + ) + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "complete" + assert dict(workbench_db.execute("SELECT * FROM deep_scan_runs").fetchone()) == run + assert list(workbench_db.execute("SELECT * FROM deep_scan_attempts")) == attempts + coverage = json.loads((scan.scan_dir / "coverage.json").read_bytes()) + assert coverage["completeness"] == "partial" + assert len(coverage["surfaces"]) == 2 + for actual, source in zip( + coverage["surfaces"], scan.source["coverage"]["surfaces"], strict=True + ): + assert actual["disposition"] == source["disposition"] + assert actual["reason"] == source["reason"] + assert actual["provenance"]["source"] == "independent-review" + assert actual["receiptRefs"] == [scan.receipt.relative_to(scan.scan_dir).as_posix()] + assert source["reason"] in (scan.scan_dir / "report.md").read_text() + assert json.loads((scan.scan_dir / "findings.json").read_bytes())["findings"] == [] + assert scan.accepted.read_bytes() == scan.contents + before = snapshot(workbench_db, scan) + workbench_api["complete_scan"]( + workbench_db, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + assert snapshot(workbench_db, scan) == before + + +@pytest.mark.parametrize( + "damage", + [ + "accepted-bytes", + "accepted-null-digest", + "newer-rejection", + "changed-obligation", + "unknown-duplicate", + "null-digest", + ], +) +def test_reader_rejects_damaged_old_budget_before_writes( + workbench_api, workbench_db, legacy_budget, damage +): + scan = legacy_budget() + if damage == "accepted-bytes": + scan.accepted.write_bytes(scan.contents + b" ") + elif damage == "accepted-null-digest": + with workbench_db: + workbench_db.execute("UPDATE deep_scan_attempts SET accepted_result_sha256 = NULL") + elif damage == "newer-rejection": + newer = copy.deepcopy(scan.source) + newer["coverage"]["surfaces"][0]["disposition"] = "rejected" + contents = json.dumps(newer).encode() + name = hashlib.sha256(contents).hexdigest() + ".json" + (scan.accepted.parent / name).write_bytes(contents) + (scan.accepted.parent.parent / "checkpoint-head.json").write_text( + json.dumps({"checkpoint": name}) + ) + elif damage == "null-digest": + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET finalization_input_json = ?", + (json.dumps({**scan.selection, "publicationSha256": None}),), + ) + else: + path = scan.scan_dir / "coverage.json" + coverage = json.loads(path.read_bytes()) + if damage == "changed-obligation": + coverage["surfaces"][0]["reason"] = "A different obligation remains unresolved." + else: + coverage["surfaces"].extend( + [{"id": "other", "label": "Other", "disposition": "needs_follow_up"}] * 2 + ) + path.write_text(json.dumps(coverage)) + before = snapshot(workbench_db, scan) + statements = [] + workbench_db.set_trace_callback(statements.append) + with pytest.raises(SystemExit): + workbench_api["complete_budget_exhausted_scan"]( + workbench_db, + Namespace(scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=None), + ) + workbench_db.set_trace_callback(None) + assert snapshot(workbench_db, scan) == before + assert not any( + statement.lstrip().split()[0].upper() in {"UPDATE", "DELETE", "INSERT", "REPLACE"} + for statement in statements + ) diff --git a/plugins/codex-security/tests/test_report_projection.py b/plugins/codex-security/tests/test_report_projection.py index e88dcb63a..73c61cbd2 100644 --- a/plugins/codex-security/tests/test_report_projection.py +++ b/plugins/codex-security/tests/test_report_projection.py @@ -75,6 +75,26 @@ def test_projection_normalizes_multiline_and_block_structural_text() -> None: assert "Text: ## Injected remediation - unsafe instruction" in markdown +def test_linked_writeup_retains_distinct_source_fixes() -> None: + manifest, findings, coverage = canonical_documents() + finding = findings["findings"][0] + finding["writeup"] = {"reportPath": "findings/parser/parser.md"} + finding["remediation"] = "Validate the record length." + finding["provenance"] = { + "sourceFindings": [ + {"id": "review-1:0", "finding": {"remediation": "Validate the record length."}}, + {"id": "review-2:0", "finding": {"remediation": "Reject duplicate record keys."}}, + {"id": "review-3:0", "finding": {"remediation": "Reject duplicate record keys."}}, + ] + } + + markdown = PROJECTION.build_report_markdown(manifest, findings, coverage) + + assert "findings/parser/parser.md" in markdown + assert markdown.count("Validate the record length.") == 1 + assert markdown.count("Reject duplicate record keys.") == 1 + + def test_projection_renders_inline_code_and_section_code_evidence() -> None: manifest, findings, coverage = canonical_documents() finding = findings["findings"][0] @@ -885,3 +905,24 @@ def test_projection_includes_surface_evidence_receipts() -> None: markdown = PROJECTION.build_report_markdown(manifest, findings, coverage) assert "Reviewed parser entrypoints. Evidence: artifacts/receipts/parser.jsonl" in markdown + + +@pytest.mark.parametrize("notes", [None, "The filesystem race remains untested."]) +def test_coverage_report_preserves_the_recorded_disposition_reason(notes) -> None: + manifest, findings, coverage = canonical_documents() + surface = { + "id": "filesystem-boundary", + "label": "Filesystem boundary", + "disposition": "needs_follow_up", + "reason": "The caller's filesystem policy is unknown.", + "receiptRefs": ["artifacts/filesystem-review.md"], + } + if notes is not None: + surface["notes"] = notes + coverage["completeness"] = "partial" + coverage["surfaces"] = [surface] + markdown = PROJECTION.build_report_markdown(manifest, findings, coverage) + assert surface["reason"] in markdown + if notes is not None: + assert notes in markdown + assert surface["receiptRefs"][0] in markdown diff --git a/plugins/codex-security/tests/test_selected_publication_authority.py b/plugins/codex-security/tests/test_selected_publication_authority.py new file mode 100644 index 000000000..b3bd24ea2 --- /dev/null +++ b/plugins/codex-security/tests/test_selected_publication_authority.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import hashlib +import json + +import pytest +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +@pytest.mark.parametrize("publication", ["selected", "mutable", "stale-generation", "unfenced"]) +def test_publication_uses_committed_finalization_selection( + workbench_api, workbench_db, publication_scan, publication +): + scan = publication_scan() + result = add_worker(workbench_db, scan) + contents = json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": scan.findings, + "coverage": scan.coverage, + } + ).encode() + digest = hashlib.sha256(contents).hexdigest() + accepted = result.parent / "accepted" / f"{digest}.json" + accepted.parent.mkdir() + accepted.write_bytes(contents) + selection = { + "version": 1, + "resultPath": accepted.relative_to(scan.scan_dir).as_posix(), + "resultSha256": digest, + "terminalReason": "saturated", + "omittedWorkerIds": [], + "selectedAt": scan.timestamp, + } + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none' " + "WHERE result_manifest_path = ?", + (str(result),), + ) + workbench_db.execute( + "UPDATE deep_scan_runs SET coordinator_generation = ?, finalization_input_json = ?, " + "workflow_version = 'deep-security-scan/v2' " + "WHERE scan_id = ?", + (1 if publication == "unfenced" else 3, json.dumps(selection), scan.scan_id), + ) + # The accepted bytes survive replacement or deletion of the worker's output. + result.unlink(missing_ok=True) + staged = stage_publication( + scan, + generation=None + if publication == "unfenced" + else 2 + if publication == "stale-generation" + else 3, + result_path=result if publication == "mutable" else accepted, + title="Selected accepted aggregate", + ) + before = {path: path.read_bytes() for path in scan.scan_dir.rglob("*.json")} + + if publication == "selected": + workbench_api["write_scan_draft"](workbench_db, staged) + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert findings[0]["title"] == "Selected accepted aggregate" + else: + with pytest.raises(SystemExit, match="coordinator|publication|aggregate"): + workbench_api["write_scan_draft"](workbench_db, staged) + assert {path: path.read_bytes() for path in scan.scan_dir.rglob("*.json")} == before + assert accepted.read_bytes() == contents + assert ( + json.loads( + workbench_db.execute( + "SELECT finalization_input_json FROM deep_scan_runs WHERE scan_id = ?", + (scan.scan_id,), + ).fetchone()[0] + ) + == selection + ) diff --git a/plugins/codex-security/tests/test_stopped_accepted_digests.py b/plugins/codex-security/tests/test_stopped_accepted_digests.py new file mode 100644 index 000000000..3777f85c0 --- /dev/null +++ b/plugins/codex-security/tests/test_stopped_accepted_digests.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import sqlite3 +from argparse import Namespace + +import pytest +from test_accepted_publication_references import accept_reducer +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +@pytest.mark.parametrize("recorded", [True, False], ids=["accepted-receipt", "legacy"]) +@pytest.mark.parametrize("changed", [False, True], ids=["original", "changed"]) +@pytest.mark.parametrize("historical", [False, True], ids=["current-attempt", "prior-attempt"]) +def test_stopped_recovery_checks_recorded_accepted_bytes( + workbench_api, workbench_db, publication_scan, tmp_path, recorded, changed, historical +): + scan = publication_scan() + result, accepted, _ = accept_reducer(workbench_db, scan) + checkpoint = result.parent / "checkpoints" / accepted.name + checkpoint.parent.mkdir() + accepted.rename(checkpoint) + original = checkpoint.read_bytes() + digest = hashlib.sha256(original).hexdigest() + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_attempts SET accepted_result_path = ? WHERE worker_id = ?", + (str(checkpoint), result.parent.name), + ) + if not recorded: + workbench_db.execute( + "DELETE FROM deep_scan_attempts WHERE worker_id = ?", (result.parent.name,) + ) + if historical: + workbench_db.execute( + "UPDATE deep_scan_workers SET attempt = 2, status = 'running' WHERE id = ?", + (result.parent.name,), + ) + if changed: + damaged = json.loads(original) + damaged["findings"][0]["summary"] = "Unaccepted changed evidence." + checkpoint.write_text(json.dumps(damaged)) + accepted_bytes = checkpoint.read_bytes() + # An unrelated valid source must still survive stopped partial preservation. + healthy = add_worker(workbench_db, scan) + finding = copy.deepcopy(scan.findings[0]) + finding["summary"] = "Independent preserved evidence." + finding["identity"]["anchor"] += ".independent" + finding["locations"][0]["startLine"] = 2 + finding["locations"][0]["endLine"] = 2 + healthy.write_text( + json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": [finding], + "coverage": scan.coverage, + } + ) + ) + healthy_bytes = healthy.read_bytes() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + database = tmp_path / "accepted-digests.sqlite3" + with sqlite3.connect(database) as connection: + workbench_db.backup(connection) + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + stopped = workbench_api["fail_scan"]( + connection, + Namespace( + scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped." + ), + )["scan"] + summaries = {finding["summary"] for finding in stopped["findings"]} + assert "Independent preserved evidence." in summaries + assert ("Unaccepted changed evidence." in summaries) == (changed and not recorded) + if not changed: + assert scan.findings[0]["summary"] in summaries + if recorded: + assert ( + connection.execute( + "SELECT accepted_result_sha256 FROM deep_scan_attempts WHERE worker_id = ?", + (result.parent.name,), + ).fetchone()[0] + == digest + ) + if recorded and changed: + warnings = json.loads( + connection.execute("SELECT completion_warnings_json FROM scans").fetchone()[0] + ) + assert any("changed after acceptance" in warning for warning in warnings) + assert checkpoint.read_bytes() == accepted_bytes + assert healthy.read_bytes() == healthy_bytes diff --git a/plugins/codex-security/tests/test_stopped_result_version_boundary.py b/plugins/codex-security/tests/test_stopped_result_version_boundary.py new file mode 100644 index 000000000..50e42749b --- /dev/null +++ b/plugins/codex-security/tests/test_stopped_result_version_boundary.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import hashlib +import json +from argparse import Namespace + +import pytest +from test_checkpoint_publication_authority import save_disposition +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +def snapshot(connection, scan_dir): + return { + "database": "\n".join(connection.iterdump()), + "files": { + path.relative_to(scan_dir).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() + for path in scan_dir.rglob("*") + if path.is_file() + }, + } + + +@pytest.mark.parametrize("operation", ["preserve", "recover"]) +@pytest.mark.parametrize("protocol", ["supported", "future-workflow", "future-selection"]) +def test_stopped_result_publication_requires_supported_protocol( + workbench_api, workbench_db, publication_scan, monkeypatch, operation, protocol +): + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + result = add_worker(workbench_db, scan, status="canceled") + draft = save_disposition(scan, result.parent, "reported") + result.write_text(json.dumps(draft)) + + def interrupt_publication(*args, **kwargs): + raise OSError("Synthetic publication interruption") + + with monkeypatch.context() as patch: + patch.setattr( + workbench_api["saved_results"], + "_write_prepared_scan_finalization", + interrupt_publication, + ) + workbench_api["fail_scan"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, + claim_token=None, + cost_json=None, + message="Original worker stop.", + ), + ) + row = workbench_db.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone() + assert row["status"] == "failed" + assert row["retained_source_digests_json"] + assert row["seal_manifest_digest"] is None + + with workbench_db: + if protocol == "future-workflow": + workbench_db.execute( + "UPDATE deep_scan_runs SET workflow_version = 'future/v99' WHERE scan_id = ?", + (scan.scan_id,), + ) + elif protocol == "future-selection": + workbench_db.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "finalization_input_json = ? WHERE scan_id = ?", + (json.dumps({"version": 99}), scan.scan_id), + ) + before = snapshot(workbench_db, scan.scan_dir) + error = None + try: + if operation == "preserve": + workbench_api["preserve_scan_results"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, + claim_token=None, + thread_id=None, + coordinator_generation=None, + ), + ) + else: + workbench_api["recover_scan_results"](workbench_db, Namespace(scan_id=scan.scan_id)) + except SystemExit as failure: + error = str(failure) + after = snapshot(workbench_db, scan.scan_dir) + changed_files = sorted( + path + for path in before["files"].keys() | after["files"].keys() + if before["files"].get(path) != after["files"].get(path) + ) + print( + json.dumps( + { + "operation": operation, + "protocol": protocol, + "error": error, + "database_changed": before["database"] != after["database"], + "changed_files": changed_files, + } + ) + ) + if protocol == "supported": + assert error is None + assert before != after + row = workbench_db.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone() + assert row["seal_manifest_digest"] + assert row["failure_message"] == "Original worker stop." + else: + assert error is not None and "unsupported" in error.lower() + assert after == before diff --git a/plugins/codex-security/tests/test_stopped_source_coverage.py b/plugins/codex-security/tests/test_stopped_source_coverage.py new file mode 100644 index 000000000..57b8e6534 --- /dev/null +++ b/plugins/codex-security/tests/test_stopped_source_coverage.py @@ -0,0 +1,167 @@ +from __future__ import annotations + +import copy +import json +from argparse import Namespace + +import pytest +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +@pytest.mark.parametrize("host_coverage", [True, False], ids=["accepted-projection", "legacy"]) +@pytest.mark.parametrize("parent_draft", [True, False], ids=["parent-draft", "no-parent"]) +def test_stopped_recovery_preserves_accepted_coverage_without_worker_id_collisions( + workbench_api, workbench_db, publication_scan, host_coverage, parent_draft +): + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + source_coverage = { + "completeness": "partial", + "surfaces": [], + "explicitExclusions": [], + "deferred": [], + "reviews": [], + } + source_files = [] + for disposition in ("needs_follow_up", "rejected"): + result = add_worker(workbench_db, scan) + worker_id = result.parent.name + surface = { + "id": "surface-1", + "candidateId": "candidate-1", + "label": "Independent review", + "disposition": disposition, + "receiptRefs": [], + } + deferred = {"candidateId": "candidate-1", "reason": "Validation remains unresolved."} + coverage = { + "completeness": "partial" if disposition == "needs_follow_up" else "complete", + "surfaces": [surface], + "explicitExclusions": [], + "deferred": [deferred] if disposition == "needs_follow_up" else [], + } + result.write_text( + json.dumps( + {"scanId": scan.scan_id, "complete": True, "findings": [], "coverage": coverage} + ) + ) + source_files.append(result) + prefix = f"{worker_id}-attempt-1" + provenance = {"workerId": worker_id, "attempt": 1, "candidateId": "candidate-1"} + source_coverage["reviews"].append( + {"workerId": worker_id, "attempt": 1, "completeness": coverage["completeness"]} + ) + source_coverage["surfaces"].append( + { + **surface, + "id": f"{prefix}-surface-1", + "provenance": {**provenance, "sourceId": "surface-1"}, + } + ) + if coverage["deferred"]: + source_coverage["deferred"].append( + { + **deferred, + "id": f"{prefix}-deferred-1", + "candidateId": f"{prefix}-candidate-1", + "provenance": provenance, + } + ) + reducer = add_worker(workbench_db, scan) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none' " + "WHERE result_manifest_path = ?", + (str(reducer),), + ) + aggregate = {"scanId": scan.scan_id, "complete": True, "findings": []} + if host_coverage: + aggregate["sourceCoverage"] = copy.deepcopy(source_coverage) + reducer.write_text(json.dumps(aggregate)) + source_files.append(reducer) + saved_bytes = {path: path.read_bytes() for path in source_files} + if not parent_draft: + for filename in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / filename).unlink() + + stopped = workbench_api["fail_scan"]( + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."), + )["scan"] + + assert stopped["progress"]["status"] == "failed" + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert coverage["completeness"] == "partial" + assert len(coverage["deferred"]) == 2 + assert coverage["deferred"][-1]["id"] == "scan-stopped" + assert len(coverage["surfaces"]) == 2 + if host_coverage: + for field in ("reviews", "surfaces", "deferred"): + assert ( + coverage[field][:-1] if field == "deferred" else coverage[field] + ) == source_coverage[field] + else: + assert coverage["deferred"][0]["candidateId"] == "candidate-1" + manifest = (scan.scan_dir / "scan-manifest.json").read_bytes() + workbench_api["preserve_scan_results"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None + ), + ) + assert (scan.scan_dir / "scan-manifest.json").read_bytes() == manifest + assert all(path.read_bytes() == contents for path, contents in saved_bytes.items()) + + +def test_stopped_recovery_keeps_unmerged_coverage_after_accepted_review( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + accepted = add_worker(workbench_db, scan) + accepted.write_text( + json.dumps( + {"scanId": scan.scan_id, "complete": True, "findings": [], "coverage": scan.coverage} + ) + ) + reducer = add_worker(workbench_db, scan) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none' " + "WHERE result_manifest_path = ?", + (str(reducer),), + ) + reviews = [{"workerId": accepted.parent.name, "attempt": 1, "completeness": "complete"}] + reducer.write_text( + json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": [], + "sourceCoverage": {**scan.coverage, "reviews": reviews}, + } + ) + ) + pending = add_worker(workbench_db, scan, status="canceled") + deferred = {"id": "pending-review", "reason": "The independent review remains unresolved."} + pending.write_text( + json.dumps( + { + "scanId": scan.scan_id, + "complete": False, + "findings": [], + "coverage": {**scan.coverage, "completeness": "partial", "deferred": [deferred]}, + } + ) + ) + + workbench_api["fail_scan"]( + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."), + ) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert coverage["completeness"] == "partial" + assert coverage["reviews"] == reviews + assert deferred in coverage["deferred"] diff --git a/plugins/codex-security/tests/test_workbench_db.py b/plugins/codex-security/tests/test_workbench_db.py index 9cb0301e5..272f6cd7e 100644 --- a/plugins/codex-security/tests/test_workbench_db.py +++ b/plugins/codex-security/tests/test_workbench_db.py @@ -6,6 +6,7 @@ import runpy import sqlite3 import subprocess +import sys import time import uuid from concurrent.futures import ThreadPoolExecutor @@ -13,6 +14,7 @@ from typing import Any import pytest +import workbench_test_support from workbench_test_support import ( SCRIPT, create_saved_git_workspace, @@ -55,6 +57,9 @@ BUDGET_WARNING = "Scan stopped: estimated cost $0.00625 exceeded the $0.005 cost limit." EXPECTED_TABLES = { + "deep_scan_attempts", + "deep_scan_attempt_sessions", + "deep_scan_merge_claims", "deep_scan_dedup_inputs", "deep_scan_runs", "deep_scan_workers", @@ -169,6 +174,198 @@ def budget_scan_fixture( return state_dir, target, scan_dir, scan_id, ledger +@pytest.mark.parametrize("operation", ["complete-scan", "complete-budget-exhausted-scan"]) +@pytest.mark.parametrize("protocol", ["supported", "future-workflow", "future-selection"]) +def test_completion_rejects_unknown_protocol_before_mutation( + workbench_api, monkeypatch, tmp_path, operation, protocol +): + script = str(workbench_api["__file__"]) + monkeypatch.setattr(workbench_test_support, "SCRIPT", script) + monkeypatch.setenv("CODEX_HOME", str(tmp_path / "home")) + state_dir, _, scan_dir, scan_id, _ = budget_scan_fixture(tmp_path) + environment = {**os.environ, "CODEX_SECURITY_STATE_DIR": str(state_dir)} + cost_args = ["--scan-id", scan_id, "--cost-json", json.dumps(BUDGET_COST)] + if operation == "complete-scan": + cut_program = """ +import os, runpy, sys +script, *args = sys.argv[1:] +api = runpy.run_path(script, run_name="completion_version_test") +namespace = api["main"].__globals__ +original = namespace["_write_prepared_scan_finalization"] +def after_seal(*args, **kwargs): + original(*args, **kwargs) + os._exit(86) +namespace["_write_prepared_scan_finalization"] = after_seal +sys.argv = [script, *args] +api["main"]() +""" + cut = subprocess.run( + [ + sys.executable, + "-I", + "-B", + "-c", + cut_program, + script, + "complete-budget-exhausted-scan", + *cost_args, + "--message", + BUDGET_WARNING, + ], + env=environment, + capture_output=True, + text=True, + ) + assert cut.returncode == 86, cut.stderr + assert json.loads((scan_dir / "scan-manifest.json").read_text())["scan"]["sealedAt"] + database = state_dir / "workbench.sqlite3" + with sqlite3.connect(database) as connection: + assert connection.execute("SELECT status FROM scans").fetchone() == ("running",) + if protocol == "future-workflow": + connection.execute("UPDATE deep_scan_runs SET workflow_version = 'future/v99'") + elif protocol == "future-selection": + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "finalization_input_json = ?", + (json.dumps({"version": 99}),), + ) + + def snapshot(): + with sqlite3.connect(database) as connection: + return list(connection.iterdump()), { + str(path.relative_to(scan_dir)): path.read_bytes() + for path in scan_dir.rglob("*") + if path.is_file() + } + + before = snapshot() + result = subprocess.run( + [ + sys.executable, + "-I", + "-B", + script, + operation, + *cost_args, + *([] if operation == "complete-scan" else ["--message", BUDGET_WARNING]), + ], + env=environment, + capture_output=True, + text=True, + ) + after = snapshot() + if protocol == "supported": + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout)["scan"]["progress"]["status"] == "complete" + if operation == "complete-scan": + assert after[1] == before[1] + else: + assert result.returncode != 0 + assert "unsupported" in result.stderr.lower() + assert after == before + + +@pytest.mark.parametrize("legacy_digest", [False, True]) +@pytest.mark.parametrize( + "protocol", ["supported", "legacy-no-run", "future-workflow", "future-selection"] +) +def test_completed_replay_checks_protocol_before_cost_or_digest_writes( + workbench_api, monkeypatch, tmp_path, protocol, legacy_digest +): + monkeypatch.setattr(workbench_test_support, "SCRIPT", workbench_api["__file__"]) + monkeypatch.setenv("CODEX_HOME", str(tmp_path / "home")) + state_dir, _, scan_dir, scan_id, _ = budget_scan_fixture(tmp_path) + complete_budget_scan(state_dir, scan_id) + database = state_dir / "workbench.sqlite3" + with sqlite3.connect(database) as connection: + assert connection.execute("SELECT status FROM scans").fetchone() == ("complete",) + if legacy_digest: + connection.execute("UPDATE scans SET seal_manifest_digest = NULL") + if protocol == "future-workflow": + connection.execute("UPDATE deep_scan_runs SET workflow_version = 'future/v99'") + elif protocol == "future-selection": + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "finalization_input_json = ?", + (json.dumps({"version": 99}),), + ) + elif protocol == "legacy-no-run": + connection.execute("DELETE FROM deep_scan_runs") + + def snapshot(): + with sqlite3.connect(database) as connection: + return list(connection.iterdump()), { + str(path.relative_to(scan_dir)): path.read_bytes() + for path in scan_dir.rglob("*") + if path.is_file() + } + + before = snapshot() + cost = {**BUDGET_COST, "inputTokens": 1500, "estimatedUsd": 0.0075} + result = run_workbench( + state_dir, + "complete-scan", + "--scan-id", + scan_id, + "--cost-json", + json.dumps(cost), + check=not protocol.startswith("future-"), + ) + after = snapshot() + if protocol.startswith("future-"): + assert after == before + assert result["returncode"] != 0 + assert "unsupported" in result["stderr"].lower() + else: + assert result["scan"]["progress"]["status"] == "complete" + assert result["scan"]["cost"] == cost + assert after[1] == before[1] + with sqlite3.connect(database) as connection: + assert connection.execute("SELECT seal_manifest_digest FROM scans").fetchone()[0] + + +@pytest.mark.parametrize("terminal", [True]) +@pytest.mark.parametrize("continuation", ["current", "pending", "claimed"]) +def test_budget_completion_checks_continuation_before_draft_writes( + workbench_api, monkeypatch, tmp_path, terminal, continuation +): + monkeypatch.setattr(workbench_test_support, "SCRIPT", workbench_api["__file__"]) + monkeypatch.setenv("CODEX_HOME", str(tmp_path / "home")) + state_dir, _, scan_dir, scan_id, _ = budget_scan_fixture(tmp_path, terminal=terminal) + database = state_dir / "workbench.sqlite3" + with sqlite3.connect(database) as connection: + connection.execute("UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2'") + if continuation != "current": + connection.execute("UPDATE scans SET handoff_status = 'pending'") + if continuation == "claimed": + run_workbench( + state_dir, + "claim-handoff-delivery", + "--scan-id", + scan_id, + "--claim-token", + str(uuid.uuid4()), + ) + + def snapshot(): + with sqlite3.connect(database) as connection: + return list(connection.iterdump()), { + str(path.relative_to(scan_dir)): path.read_bytes() + for path in scan_dir.rglob("*") + if path.is_file() + } + + before = snapshot() + result = complete_budget_scan(state_dir, scan_id, check=continuation == "current") + after = snapshot() + if continuation == "current": + assert result["scan"]["progress"]["status"] == "complete" + else: + assert after == before + assert result["returncode"] != 0 + assert "owned by another continuation" in result["stderr"] + + def complete_budget_scan(state_dir: Path, scan_id: str, *, check: bool = True) -> dict[str, object]: return run_workbench( state_dir, @@ -1042,7 +1239,7 @@ def test_workbench_persists_progress_and_indexes_completed_findings(tmp_path: Pa ) } assert tables == EXPECTED_TABLES - assert connection.execute("SELECT COUNT(*) FROM schema_migrations").fetchone() == (41,) + assert connection.execute("SELECT COUNT(*) FROM schema_migrations").fetchone() == (46,) assert connection.execute("SELECT COUNT(*) FROM findings").fetchone() == (1,) assert connection.execute("SELECT COUNT(*) FROM finding_locations").fetchone() == (1,) diff --git a/plugins/codex-security/tests/test_workbench_deep_scan.py b/plugins/codex-security/tests/test_workbench_deep_scan.py index c58a274e2..9d598c4bb 100644 --- a/plugins/codex-security/tests/test_workbench_deep_scan.py +++ b/plugins/codex-security/tests/test_workbench_deep_scan.py @@ -279,7 +279,9 @@ def claim() -> dict[str, object]: return claim_deep_scan_coordinator(state_dir, codex_home, scan_id) with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: - assert connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone() == (41,) + migrations = connection.execute( + "SELECT * FROM schema_migrations ORDER BY version" + ).fetchall() assert claim()["deepScan"]["coordinatorGeneration"] == 2 assert claim()["coordinatorDisposition"] == "observing" expire_deep_scan_coordinator(state_dir, scan_id) @@ -311,6 +313,11 @@ def claim() -> dict[str, object]: assert sum(result["coordinatorDisposition"] == "adopted" for result in results) == 1 assert sum(result["coordinatorDisposition"] == "observing" for result in results) == 3 assert {result["deepScan"]["coordinatorGeneration"] for result in results} == {3} + with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: + assert ( + connection.execute("SELECT * FROM schema_migrations ORDER BY version").fetchall() + == migrations + ) def test_legacy_generation_with_active_worker_observes_grace_then_adopts(tmp_path: Path) -> None: @@ -1879,10 +1886,21 @@ def test_failed_reducer_rebuffers_claimed_inputs_for_same_generation_replacement error="fixture reducer exhausted its attempts", coordinator_generation=2, )["deepScan"] - replayed_workers = {worker["id"]: worker for worker in replayed["workers"]} + assert replayed["workerReceipt"] == failed["workerReceipt"] assert replayed["phase"] == "reducing" - assert replayed["consecutiveErrors"] == counter_before_failure - assert all(replayed_workers[worker]["mergeState"] == "merging" for worker in replacement_inputs) + current = run_workbench( + state_dir, + "get-deep-scan", + "--scan-id", + scan_id, + "--thread-id", + "thread-deep-scan", + environment=deep_environment(codex_home), + )["deepScan"] + current_workers = {worker["id"]: worker for worker in current["workers"]} + assert current["phase"] == "reducing" + assert current["consecutiveErrors"] == counter_before_failure + assert all(current_workers[worker]["mergeState"] == "merging" for worker in replacement_inputs) upsert_worker( state_dir, diff --git a/plugins/codex-security/tests/test_workbench_scan_usage.py b/plugins/codex-security/tests/test_workbench_scan_usage.py index 06b7ea425..4671818b3 100644 --- a/plugins/codex-security/tests/test_workbench_scan_usage.py +++ b/plugins/codex-security/tests/test_workbench_scan_usage.py @@ -6,6 +6,7 @@ import sys import tempfile import uuid +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from datetime import datetime, timedelta from pathlib import Path @@ -492,6 +493,32 @@ def test_completion_reports_unavailable_without_fabricating_zero(tmp_path: Path) assert "totalTokens" not in usage +@pytest.mark.parametrize("reported", ["missing", "null-counter", "explicit-zero"]) +def test_completion_distinguishes_missing_token_records_from_zero( + tmp_path: Path, reported: str +) -> None: + fixture = _start_scan(tmp_path) + counted = fixture.started_at + timedelta(microseconds=1) + events = [] + if reported == "explicit-zero": + events.append(_token_event(counted, 0, 0)) + elif reported == "null-counter": + events.append( + _event(counted, "event_msg", {"type": "token_count", "info": None, "rate_limits": None}) + ) + parent = _rollout(tmp_path, "scan-parent", events) + _state_graph(fixture.environment, {"scan-parent": parent}, []) + usage = _complete_scan(fixture)["scan"]["usage"] + if reported == "explicit-zero": + assert usage["coverage"] == "complete" + assert usage["totalTokens"] == 0 + else: + assert usage["coverage"] == "unavailable" + assert "token_usage_unavailable" in usage["warnings"] + assert "token_record_invalid" not in usage["warnings"] + assert "totalTokens" not in usage + + @pytest.mark.skipif(sys.platform != "darwin", reason="macOS system path aliases") @pytest.mark.parametrize("temporary_root", [tempfile.gettempdir(), "/tmp"], ids=["var", "tmp"]) def test_completion_accepts_macos_system_rollout_alias( @@ -565,6 +592,22 @@ def test_completion_counts_deep_sdk_workers_and_descendants(tmp_path: Path) -> N environment, "deep", ) + # Read an owner binding already persisted by a newer writer release. + with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: + connection.execute( + "UPDATE deep_scan_runs SET usage_owner_json = ? WHERE scan_id = ?", + ( + json.dumps( + { + "threadId": "scan-parent", + "turnId": None, + "startedAt": fixture.started_at.isoformat(), + "dedicated": False, + } + ), + scan_id, + ), + ) counted = fixture.started_at + timedelta(microseconds=1) artifact = scan_dir / "artifacts" / "usage-worker" artifact.mkdir(parents=True) @@ -605,11 +648,316 @@ def test_completion_counts_deep_sdk_workers_and_descendants(tmp_path: Path) -> N ) usage = _complete_scan(fixture)["scan"]["usage"] assert usage == { - "coverage": "complete", + "coverage": "partial", "source": "codex_rollout", - **_counts(37, 0, 10), - "threadCount": 3, + **_counts(27, 0, 7), + "threadCount": 2, + "missingThreadCount": 1, + "warnings": ["scan_owner_turn_unavailable"], + "modelUsage": [{"model": None, **_counts(27, 0, 7)}], + } + + +@pytest.mark.parametrize( + "worker_home", + [ + "recorded", + "current", + "inherited-sqlite", + "current-prefix", + "recorded-prefix", + "current-unreadable", + "current-mismatched", + "external-sqlite", + "external-shared-home", + "external-missing-copy", + "external-missing-child", + "unavailable", + ], +) +def test_completion_keeps_owner_and_workers_in_their_recorded_homes( + tmp_path: Path, worker_home: str +) -> None: + current_home = tmp_path / "current-home" + copied_rollout = worker_home in { + "current-prefix", + "recorded-prefix", + "current-unreadable", + "current-mismatched", + } + environment = { + "CODEX_HOME": str(current_home), + "CODEX_SQLITE_HOME": str(current_home / "sqlite"), + "CODEX_STATE_DB": str(current_home / "sqlite" / "state_5.sqlite"), + } + owners = { + f"owner-{index}": _rollout( + tmp_path, + f"owner-{index}", + [_event(datetime.now().astimezone(), "turn_context", {"turn_id": "original"})], + ) + for index in (1, 2) } + _state_graph(environment, owners, []) + + def complete(index: int) -> dict[str, Any]: + root = tmp_path / f"scan-{index}" + target = root / "target" + target.mkdir(parents=True) + selected_home = current_home if worker_home == "current" else root / "original-home" + if worker_home == "external-shared-home": + selected_home = tmp_path / "shared-original-home" + deep = run_workbench( + root / "state", + "begin-deep-scan", + "--thread-id", + f"owner-{index}", + "--target-path", + str(target), + "--scan-root", + str(root / "scans"), + environment=environment, + )["deepScan"] + scan_id, scan_dir = deep["scanId"], Path(deep["scanDir"]) + snapshot = scan_dir / "artifacts/deep_discovery/execution-settings.json" + assert not snapshot.exists() + # These original facts were persisted by a newer writer release. + snapshot.parent.mkdir(parents=True, exist_ok=True) + snapshot.write_text( + json.dumps( + { + "version": 1, + "settings": { + "codexHome": str(selected_home), + "codexPath": sys.executable, + }, + } + ) + ) + owner_json = json.dumps( + { + "threadId": f"owner-{index}", + "turnId": "original", + "startedAt": deep["createdAt"], + "dedicated": False, + } + ) + with sqlite3.connect(root / "state" / "workbench.sqlite3") as connection: + assert connection.execute( + "SELECT usage_owner_json FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) + ).fetchone() == (None,) + connection.execute( + "UPDATE deep_scan_runs SET usage_owner_json = ? WHERE scan_id = ?", + (owner_json, scan_id), + ) + original_bytes = snapshot.read_bytes() + fixture = ScanFixture( + root / "state", + target, + scan_id, + scan_dir, + datetime.fromisoformat(deep["createdAt"]), + environment, + "deep", + ) + counted = fixture.started_at + timedelta(microseconds=1) + with owners[f"owner-{index}"].open("a") as stream: + stream.write(json.dumps(_token_event(counted, index * 10, 2)) + "\n") + stream.write(json.dumps(_event(counted, "turn_context", {"turn_id": "later"})) + "\n") + stream.write(json.dumps(_token_event(counted, 9000, 900)) + "\n") + worker_threads = {} + for kind in ("discovery", "second-discovery"): + thread_id = f"{kind}-{index}" + artifact = scan_dir / "artifacts" / thread_id + artifact.mkdir() + prompt = artifact / "prompt.md" + prompt.write_text("Review the synthetic target.\n") + run_workbench( + fixture.state_dir, + "upsert-deep-scan-worker", + "--scan-id", + scan_id, + "--worker-id", + str(uuid.uuid4()), + "--kind", + "discovery", + "--status", + "running", + "--prompt-path", + str(prompt), + "--artifact-dir", + str(artifact), + "--sdk-thread-id", + thread_id, + environment=environment, + ) + worker_threads[thread_id] = _rollout( + root, + thread_id, + [ + *( + [_event(counted, "turn_context", {"model": "model-alpha"})] + if copied_rollout and kind == "discovery" + else [] + ), + _token_event(counted, index * 20, 3), + _event( + counted, + "turn_context", + { + "turn_id": "resumed", + **( + {"model": "model-beta"} + if copied_rollout and kind == "discovery" + else {} + ), + }, + ), + _token_event(counted, index * 30, 5), + ], + ) + child_id = f"child-{index}" + worker_threads[child_id] = _rollout( + root, + child_id, + [_token_event(counted, index * 7, 1)], + parent_thread_id=f"discovery-{index}", + ) + if worker_home in {"current", "inherited-sqlite"}: + with sqlite3.connect(environment["CODEX_STATE_DB"]) as connection: + connection.executemany( + "INSERT INTO threads VALUES (?, ?)", + [(key, str(path)) for key, path in worker_threads.items()], + ) + connection.execute( + "INSERT INTO thread_spawn_edges VALUES (?, ?)", (f"discovery-{index}", child_id) + ) + if worker_home == "inherited-sqlite": + # Earlier launches used A; the resumed process forwards its + # explicit SQLite home C even while workers keep Codex home A. + _state_graph( + {"CODEX_SQLITE_HOME": str(selected_home)}, + {f"discovery-{index}": worker_threads[f"discovery-{index}"]}, + [], + ) + elif worker_home in { + "recorded", + "current-prefix", + "recorded-prefix", + "current-unreadable", + "current-mismatched", + }: + recorded_threads = dict(worker_threads) + if worker_home != "recorded": + thread_id = f"discovery-{index}" + full = worker_threads[thread_id] + copied = full.with_name(f"copied-{thread_id}.jsonl") + copied.write_bytes(b"\n".join(full.read_bytes().splitlines()[:3]) + b"\n") + if worker_home == "current-unreadable": + copied.write_text("invalid session metadata\n") + elif worker_home == "current-mismatched": + copied.write_text(full.read_text().replace(thread_id, "unrelated-thread")) + current_copy = copied + if worker_home == "recorded-prefix": + recorded_threads[thread_id] = copied + current_copy = full + with sqlite3.connect(environment["CODEX_STATE_DB"]) as connection: + connection.execute( + "INSERT INTO threads VALUES (?, ?)", (thread_id, str(current_copy)) + ) + _state_graph( + {"CODEX_SQLITE_HOME": str(selected_home)}, + recorded_threads, + [(f"discovery-{index}", child_id)], + ) + elif worker_home in { + "external-sqlite", + "external-shared-home", + "external-missing-copy", + "external-missing-child", + }: + # Native keeps rollouts in its Codex home even when its SQLite + # index lives elsewhere and recovery chooses a different index. + sessions = selected_home / "sessions" / "2026" / "01" / "01" + sessions.mkdir(parents=True, exist_ok=True) + for thread_id, path in worker_threads.items(): + recorded = sessions / f"rollout-{thread_id}.jsonl" + path.rename(recorded) + worker_threads[thread_id] = recorded + _state_graph( + {"CODEX_SQLITE_HOME": str(root / "original-external-sqlite")}, + worker_threads, + [(f"discovery-{index}", child_id)], + ) + if worker_home in {"external-missing-copy", "external-missing-child"}: + first_id = f"discovery-{index}" + with sqlite3.connect(environment["CODEX_STATE_DB"]) as connection: + connection.execute( + "INSERT INTO threads VALUES (?, ?)", + ( + first_id, + str(root / "missing-copy.jsonl") + if worker_home == "external-missing-copy" + else str(worker_threads[first_id]), + ), + ) + if worker_home == "external-missing-child": + connection.execute( + "INSERT INTO thread_spawn_edges VALUES (?, ?)", (first_id, child_id) + ) + worker_threads[child_id].unlink() + result = _complete_scan(fixture)["scan"]["usage"] + assert snapshot.read_bytes() == original_bytes + with sqlite3.connect(fixture.state_dir / "workbench.sqlite3") as connection: + assert connection.execute( + "SELECT usage_owner_json FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) + ).fetchone() == (owner_json,) + assert connection.execute("SELECT COUNT(*) FROM deep_scan_attempts").fetchone() == (0,) + assert connection.execute( + "SELECT COUNT(*) FROM deep_scan_attempt_sessions" + ).fetchone() == (0,) + return result + + # Both completions share current home B; each scan retains its own worker home A. + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(complete, (1, 2))) + for index, usage in enumerate(results, 1): + if worker_home == "unavailable": + assert usage["coverage"] == "partial" + assert usage["inputTokens"] == index * 10 + assert usage["outputTokens"] == 2 + assert usage["threadCount"] == 1 + assert usage["missingThreadCount"] == 2 + elif worker_home == "external-missing-child": + assert usage == { + "coverage": "partial", + "source": "codex_rollout", + **_counts(index * 70, 0, 12), + "threadCount": 3, + "missingThreadCount": 1, + "warnings": [ + "codex_state_unavailable", + "rollout_unavailable", + "scan_root_unavailable", + ], + "modelUsage": [{"model": None, **_counts(index * 70, 0, 12)}], + } + else: + assert usage == { + "coverage": "complete", + "source": "codex_rollout", + **_counts(index * 77, 0, 13), + "threadCount": 4, + "modelUsage": ( + [ + {"model": None, **_counts(index * 47, 0, 8)}, + {"model": "model-alpha", **_counts(index * 20, 0, 3)}, + {"model": "model-beta", **_counts(index * 10, 0, 2)}, + ] + if copied_rollout + else [{"model": None, **_counts(index * 77, 0, 13)}] + ), + } def test_completion_preserves_explicit_legacy_cost(tmp_path: Path) -> None: @@ -684,3 +1032,273 @@ def test_failed_scan_preserves_legacy_failure_behavior(tmp_path: Path) -> None: )["scan"] assert failed["progress"]["status"] == "failed" assert "usage" not in failed + + +def test_rollout_usage_reconciles_stale_cumulative_events_and_models( + tmp_path: Path, workbench_api +) -> None: + usage_reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + events = [ + _event(start, "turn_context", {"turn_id": "own-turn", "model": "gpt-5.6-sol"}), + _token_event(start, 100, 10), + _token_event(start, 60, 6), + _event(start, "turn_context", {"turn_id": "own-turn", "model": "gpt-6-astra"}), + _token_event(start, 200, 20), + ] + rollout = _rollout(tmp_path, "worker", events) + counts, warnings = usage_reader._read_rollout_usage( + usage_reader.RolloutSession("worker", None, rollout), + started_at=start, + completed_at=None, + ) + assert counts == _counts(200, 0, 20) + assert warnings == {"token_counter_regressed"} + models = {} + counts, warnings = usage_reader._read_rollout_usage( + usage_reader.RolloutSession("worker", None, rollout), + started_at=start, + completed_at=None, + model_usage=models, + ) + assert counts == _counts(200, 0, 20) + assert warnings == {"token_counter_regressed"} + assert models == {"gpt-5.6-sol": _counts(100, 0, 10), "gpt-6-astra": _counts(100, 0, 10)} + + +def test_shared_parent_usage_requires_original_turn_and_scan_interval( + tmp_path: Path, workbench_api +) -> None: + usage_reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + end = start + timedelta(seconds=5) + events = [ + _event( + start - timedelta(seconds=1), + "turn_context", + {"turn_id": "prior", "model": "gpt-5.6-sol"}, + ), + _token_event(start - timedelta(seconds=1), 100, 10), + _event(start, "turn_context", {"turn_id": "scan-turn", "model": "gpt-6-astra"}), + _token_event(start, 110, 12), + _event(start, "turn_context", {"turn_id": "unrelated", "model": "gpt-5.6-sol"}), + _token_event(start, 910, 92), + _event( + end + timedelta(seconds=1), + "turn_context", + {"turn_id": "scan-turn", "model": "gpt-6-astra"}, + ), + _token_event(end + timedelta(seconds=1), 1000, 100), + ] + models = {} + counts, warnings = usage_reader._read_rollout_usage( + usage_reader.RolloutSession("parent", None, _rollout(tmp_path, "parent", events)), + started_at=start, + completed_at=end, + owner_turn_id="scan-turn", + model_usage=models, + ) + assert counts == _counts(10, 0, 2) + assert warnings == set() + assert models == {"gpt-6-astra": _counts(10, 0, 2)} + + +@pytest.mark.parametrize( + "counter_info,receipt_state,expected_warning", + [ + (None, "complete", None), + ({}, "complete", "token_record_invalid"), + ({"total_token_usage": {"input_tokens": -1}}, "complete", "token_record_invalid"), + (None, "missing-response", "token_receipts_incomplete"), + (None, "incomplete-line", "rollout_record_incomplete"), + (None, "invalid-timestamp", "token_record_invalid"), + ], + ids=[ + "null-counter", + "empty-info", + "malformed-usage", + "missing-response", + "incomplete-line", + "invalid-timestamp", + ], +) +def test_completion_handles_no_usage_counter_without_hiding_incomplete_receipts( + tmp_path: Path, counter_info: Any, receipt_state: str, expected_warning: str | None +) -> None: + fixture = _start_scan(tmp_path) + counted = fixture.started_at + timedelta(microseconds=1) + tokens = dict(input_tokens=100, cached_input_tokens=20, output_tokens=10, total_tokens=110) + response = _event( + counted, + "token_usage_record", + dict( + response_id="response-one", + thread_id="scan-parent", + model="gpt-5.6-sol", + usage=tokens, + thread_token_usage=( + {**tokens, "input_tokens": 150, "total_tokens": 160} + if receipt_state == "missing-response" + else tokens + ), + ), + ) + counter = _event( + counted, + "event_msg", + {"type": "token_count", "info": counter_info, "rate_limits": None}, + ) + events = [counter, response, counter] + if receipt_state == "invalid-timestamp": + events.append( + { + **response, + "timestamp": None, + "payload": {**response["payload"], "response_id": "response-two"}, + } + ) + parent = _rollout(tmp_path, "scan-parent", events) + if receipt_state == "incomplete-line": + with parent.open("a") as stream: + stream.write('{"type":"token_usage_record"') + _state_graph(fixture.environment, {"scan-parent": parent}, []) + usage = _complete_scan(fixture)["scan"]["usage"] + assert usage["totalTokens"] == 110 + assert usage["modelUsage"] == [{"model": "gpt-5.6-sol", **_counts(100, 20, 10)}] + assert usage["coverage"] == ("partial" if expected_warning else "complete") + assert usage.get("warnings", []) == ([expected_warning] if expected_warning else []) + + +@pytest.mark.parametrize("counters", [False, True]) +def test_response_receipts_count_compaction_once_across_resets( + tmp_path: Path, workbench_api, counters: bool +) -> None: + reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + + def usage(input_tokens, cached, output): + return dict( + input_tokens=input_tokens, + cached_input_tokens=cached, + cache_write_input_tokens=0, + output_tokens=output, + reasoning_output_tokens=0, + total_tokens=input_tokens + output, + ) + + def receipt(response, count, cumulative, model="gpt-5.6-sol", turn="scan-turn", second=1): + return _event( + start + timedelta(seconds=second), + "token_usage_record", + dict( + response_id=response, + thread_id="parent", + turn_id=turn, + model=model, + usage=count, + thread_token_usage=cumulative, + ), + ) + + first = receipt("first", usage(100, 80, 10), usage(100, 80, 10)) + compact = receipt("compaction", usage(50, 40, 5), usage(150, 120, 15), model="gpt-6-astra") + second = receipt("second", usage(120, 90, 12), usage(120, 90, 12)) + events = [ + first, + *([_token_event(start + timedelta(seconds=1), 100, 10)] if counters else []), + compact, + _event(start + timedelta(seconds=1), "compacted", {"message": "Synthetic summary"}), + compact, + second, + *([_token_event(start + timedelta(seconds=1), 220, 22)] if counters else []), + first, + receipt("other", usage(900, 0, 0), usage(900, 0, 0), turn="other-turn"), + receipt("post", usage(800, 0, 0), usage(1700, 0, 0), second=11), + ] + models = {} + total, warnings = reader._read_rollout_usage( + reader.RolloutSession("parent", None, _rollout(tmp_path, "parent", events)), + started_at=start, + completed_at=start + timedelta(seconds=10), + owner_turn_id="scan-turn", + model_usage=models, + ) + assert total == _counts(270, 210, 27) + assert warnings == set() + assert models == {"gpt-5.6-sol": _counts(220, 170, 22), "gpt-6-astra": _counts(50, 40, 5)} + + +def test_delayed_response_receipt_resolves_missing_cumulative_usage( + tmp_path: Path, workbench_api +) -> None: + reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + + def receipt(response, tokens, cumulative): + def usage(value): + return dict(input_tokens=value, output_tokens=0, total_tokens=value) + + return _event( + start, + "token_usage_record", + dict( + response_id=response, + thread_id="parent", + model="gpt-5.6-sol", + usage=usage(tokens), + thread_token_usage=usage(cumulative), + ), + ) + + rollout = _rollout(tmp_path, "parent", [receipt("first", 100, 100), receipt("third", 50, 180)]) + session = reader.RolloutSession("parent", None, rollout) + total, warnings = reader._read_rollout_usage(session, started_at=start, completed_at=None) + assert total == _counts(150, 0, 0) + assert warnings == {"token_receipts_incomplete"} + with rollout.open("a") as source: + source.write(json.dumps(receipt("second", 30, 130)) + "\n") + total, warnings = reader._read_rollout_usage(session, started_at=start, completed_at=None) + assert total == _counts(180, 0, 0) + assert warnings == set() + + +def test_exact_receipts_replace_overlapping_legacy_counter(tmp_path: Path, workbench_api) -> None: + reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + + def receipt(response, tokens, cumulative): + def usage(value): + return dict(input_tokens=value, output_tokens=0, total_tokens=value) + + return _event( + start, + "token_usage_record", + dict( + response_id=response, + thread_id="parent", + model="gpt-5.6-sol", + usage=usage(tokens), + thread_token_usage=usage(cumulative), + ), + ) + + rollout = _rollout( + tmp_path, + "parent", + [ + _token_event(start, 100, 0), + receipt("new", 10, 110), + receipt("old", 100, 100), + _token_event(start, 10, 0), + ], + ) + models = {} + total, warnings = reader._read_rollout_usage( + reader.RolloutSession("parent", None, rollout), + started_at=start, + completed_at=None, + model_usage=models, + ) + assert total == _counts(110, 0, 0) + assert warnings == set() + assert models == {"gpt-5.6-sol": _counts(110, 0, 0)} diff --git a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py index 679ee77c1..94dd74fbc 100644 --- a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py +++ b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py @@ -405,7 +405,7 @@ def test_workbench_serializes_concurrent_first_run_migrations(tmp_path: Path) -> {"databasePath": str(state_dir / "workbench.sqlite3")}, ] with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: - assert connection.execute("SELECT COUNT(*) FROM schema_migrations").fetchone() == (41,) + assert connection.execute("SELECT COUNT(*) FROM schema_migrations").fetchone() == (46,) @pytest.mark.parametrize("previous_history", ["main", "comparison-preview"]) @@ -867,6 +867,11 @@ def test_workbench_creates_single_final_schema(tmp_path: Path) -> None: (39, "store dedupe checkpoint bindings in columns"), (40, "index finding identity and comparison history"), (41, "checkpoint finding severity assessments"), + (45, "retain deep scan attempts and exact merge inputs"), + (46, "persist selected deep scan finalization input"), + (47, "freeze stopped scan checkpoint selections"), + (48, "bind original deep scan parent usage turn"), + (51, "bind original deep scan execution settings"), ] assert {row[1] for row in connection.execute("PRAGMA table_info(workspaces)")} >= { "diff_target_kind", @@ -969,7 +974,7 @@ def test_workbench_upgrades_preexisting_database(tmp_path: Path) -> None: connection.execute("ALTER TABLE scans DROP COLUMN handoff_claim_token") run_workbench(state_dir, "database-info") with sqlite3.connect(database) as connection: - assert connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone() == (41,) + assert connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone() == (51,) assert {row[1] for row in connection.execute("PRAGMA table_info(scans)")} >= { "handoff_claimed_at", "handoff_claim_token", @@ -1996,6 +2001,11 @@ def test_workbench_upgrades_released_database_schema(tmp_path: Path) -> None: (39, "store dedupe checkpoint bindings in columns"), (40, "index finding identity and comparison history"), (41, "checkpoint finding severity assessments"), + (45, "retain deep scan attempts and exact merge inputs"), + (46, "persist selected deep scan finalization input"), + (47, "freeze stopped scan checkpoint selections"), + (48, "bind original deep scan parent usage turn"), + (51, "bind original deep scan execution settings"), ] assert "capability_preflight_json" in { row[1] for row in connection.execute("PRAGMA table_info(workspaces)") @@ -2079,6 +2089,11 @@ def test_workbench_upgrades_pre_release_phase_progress_migration(tmp_path: Path) (39, "store dedupe checkpoint bindings in columns"), (40, "index finding identity and comparison history"), (41, "checkpoint finding severity assessments"), + (45, "retain deep scan attempts and exact merge inputs"), + (46, "persist selected deep scan finalization input"), + (47, "freeze stopped scan checkpoint selections"), + (48, "bind original deep scan parent usage turn"), + (51, "bind original deep scan execution settings"), ] assert "continuation_thread_id" in { row[1] for row in connection.execute("PRAGMA table_info(scans)") @@ -2170,6 +2185,11 @@ def test_workbench_upgrades_pre_release_preflight_progress_migration(tmp_path: P (39, "store dedupe checkpoint bindings in columns"), (40, "index finding identity and comparison history"), (41, "checkpoint finding severity assessments"), + (45, "retain deep scan attempts and exact merge inputs"), + (46, "persist selected deep scan finalization input"), + (47, "freeze stopped scan checkpoint selections"), + (48, "bind original deep scan parent usage turn"), + (51, "bind original deep scan execution settings"), ] assert "continuation_thread_id" in { row[1] for row in connection.execute("PRAGMA table_info(scans)") diff --git a/plugins/codex-security/tests/test_workbench_standard_deep_results.py b/plugins/codex-security/tests/test_workbench_standard_deep_results.py index de2b5ec22..58e4b2fd2 100644 --- a/plugins/codex-security/tests/test_workbench_standard_deep_results.py +++ b/plugins/codex-security/tests/test_workbench_standard_deep_results.py @@ -47,8 +47,10 @@ def test_stopped_deep_scan_ignores_late_worker_checkpoints_without_reducer( # The latest incomplete attempt need not be parseable for a saved checkpoint to survive. result_path.write_text("{incomplete") with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: + # This is new incomplete work, not a rewrite of the accepted attempt. connection.execute( - "UPDATE deep_scan_workers SET status = 'running' WHERE id = ?", (worker_id,) + "UPDATE deep_scan_workers SET status = 'running', attempt = 2 WHERE id = ?", + (worker_id,), ) environment = {"CODEX_HOME": str(codex_home)} if termination == "canceled": @@ -366,6 +368,9 @@ def test_explicit_recovery_preserves_sealed_parent_with_empty_source_map( state_dir, codex_home, target, scan_dir, scan_id = deep_scan_fixture(tmp_path) _, result_path = accepted_standard_worker(state_dir, codex_home, scan_dir, scan_id) result_path.unlink() + # Remove the immutable accepted copy too, leaving no recoverable worker source. + for checkpoint in (result_path.parent / "checkpoints").glob("*.json"): + checkpoint.unlink() contract_dir = tmp_path / "contract" contract_dir.mkdir() scripts_dir = Path(__file__).resolve().parents[1] / "scripts" @@ -1012,6 +1017,9 @@ def test_canceled_scan_reports_noop_coordinator_publication(tmp_path: Path) -> N state_dir, codex_home, _, scan_dir, scan_id = deep_scan_fixture(tmp_path) _, result_path = accepted_standard_worker(state_dir, codex_home, scan_dir, scan_id) result_path.write_text("{incomplete") + # A valid immutable copy would let publication recover despite this corruption. + for checkpoint in (result_path.parent / "checkpoints").glob("*.json"): + checkpoint.unlink() scripts_dir = Path(__file__).resolve().parents[1] / "scripts" wrapper = tmp_path / "fail_before_canceled_sources_are_frozen.py" @@ -1391,10 +1399,25 @@ def test_failure_preserves_last_committed_reducer_without_parent_draft(tmp_path: draft = json.loads(result_path.read_text()) draft["findings"] = json.loads((contract_dir / "findings.json").read_text())["findings"] result_path.write_text(json.dumps(draft)) - _, reducer_path, _ = committed_standard_reducer( + reducer_id, reducer_path, _ = committed_standard_reducer( state_dir, codex_home, scan_dir, scan_id, worker_id, result_path ) reduced = json.loads(reducer_path.read_text()) + # The later writer retained this accepted reference before result.json changed. + import hashlib + + accepted = write_checkpoint(reducer_path.parent / "checkpoints", reduced) + with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: + connection.execute("PRAGMA foreign_keys = ON") + connection.execute( + "INSERT INTO deep_scan_attempts " + "(scan_id, worker_id, attempt, status, started_at, completed_at, " + "accepted_result_path, accepted_result_sha256) " + "SELECT scan_id, id, attempt, status, created_at, completed_at, ?, ? " + "FROM deep_scan_workers WHERE id = ?", + (str(accepted), hashlib.sha256(accepted.read_bytes()).hexdigest(), reducer_id), + ) + accepted_summary = reduced["findings"][0]["summary"] reduced["findings"][0]["summary"] = ( "The reducer retained additional independently reviewed evidence." ) @@ -1411,7 +1434,8 @@ def test_failure_preserves_last_committed_reducer_without_parent_draft(tmp_path: failed = run_workbench(state_dir, "get-scan", "--scan-id", scan_id)["scan"] assert failed["progress"]["status"] == "failed" assert failed["findingCount"] == 1 - assert failed["findings"][0]["summary"] == reduced["findings"][0]["summary"] + assert failed["findings"][0]["summary"] == accepted_summary + assert json.loads(reducer_path.read_text()) == reduced def test_stopped_rejection_recovers_malformed_parent_surfaces(tmp_path: Path) -> None: @@ -1584,7 +1608,7 @@ def test_complete_worker_supersedes_obsolete_checkpoint_coverage(tmp_path: Path) }, } checkpoints = result_path.parent / "checkpoints" - checkpoints.mkdir() + checkpoints.mkdir(exist_ok=True) (checkpoints / ("0" * 64 + ".json")).write_text(json.dumps(checkpoint)) run_workbench( @@ -1826,7 +1850,7 @@ def test_recovery_selects_strongest_same_finding_checkpoint(tmp_path: Path) -> N strong["confidence"]["level"] = "high" strong["summary"] = "Later strong checkpoint evidence." checkpoint_dir = result_path.parent / "checkpoints" - checkpoint_dir.mkdir() + checkpoint_dir.mkdir(exist_ok=True) for name, finding in (("0" * 64, weak), ("f" * 64, strong)): (checkpoint_dir / f"{name}.json").write_text( json.dumps( diff --git a/plugins/codex-security/tests/workbench_test_support.py b/plugins/codex-security/tests/workbench_test_support.py index dae12b77e..4a9017c1f 100644 --- a/plugins/codex-security/tests/workbench_test_support.py +++ b/plugins/codex-security/tests/workbench_test_support.py @@ -212,7 +212,7 @@ def create_saved_git_workspace(state_dir: Path, target: Path) -> dict[str, objec def mark_deep_coordinator_succeeded(state_dir: Path, scan_id: str, scan_dir: Path) -> Path: manifest = scan_dir / "artifacts" / "deep_discovery" / "coordinator-manifest.json" - manifest.parent.mkdir(parents=True) + manifest.parent.mkdir(parents=True, exist_ok=True) manifest.write_text('{"status":"succeeded"}\n') with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: connection.execute( diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index af8b260ce..c3a8b9a39 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -800,6 +800,12 @@ four workers. Unknown keys are rejected. `max_time_hours` accepts positive values up to 96, including fractional hours. At the deadline, discovery stops; the scan combines and returns completed findings. +When a Deep scan has a saved accepted aggregate, recovering an interrupted +publication reuses that aggregate and its original stop reason without another +discovery or reducer run. Explicit cancellation and cost stops retain their +stopped or partial-result behavior. +Scans created by earlier versions keep their original workflow when resumed. + `scan --workers` controls discovery workers within one deep scan; `bulk-scan --workers` controls how many repositories are scanned concurrently. @@ -948,6 +954,14 @@ long-context rates. `cost.pricing` records the price source, verification date, processing tier, short-context rates, and verified long-context rates when known. Models without known short-context prices still have no cost estimate. +Deep Scan accounting includes failed, replaced, and canceled worker attempts and +their descendants once. Shared conversation usage is limited to the original +scan turn and scan interval. Missing usage remains unavailable or partial; +reported zero remains zero. When sessions use different models, `cost.modelCosts` +records each model's tokens, estimate, and pricing basis, and `estimatedUsd` sums +those estimates. A partial estimate has `cost.coverage: "partial"`; missing model +attribution leaves the estimate unavailable until usage can be reconciled. + For compatibility, `cacheWriteInputTokens` remains the reported token subtotal. `cacheWriteInputTokensReported: false` means at least one included usage record did not report cache writes. Raw usage uses `cache_write_input_tokens_reported`. diff --git a/sdk/typescript/scripts/check-package.mjs b/sdk/typescript/scripts/check-package.mjs index f78271374..0553d5ac5 100644 --- a/sdk/typescript/scripts/check-package.mjs +++ b/sdk/typescript/scripts/check-package.mjs @@ -166,6 +166,7 @@ const allowedRoot = new Set([ ]); const distFiles = new Set( [ + "accepted-audit", "api", "auth", "bulk-scan-discovery", @@ -175,6 +176,8 @@ const distFiles = new Set( "severity-store", "cloud-publish", "codex-prompt", + "codex-session", + "preflight-config", "component-plan", "component-scan", "config", @@ -183,6 +186,7 @@ const distFiles = new Set( "cost", "cost-model", "custom-validation", + "deep-scan-finalization", "custom-validation-prompt", "custom-publish", "deep-progress", @@ -210,6 +214,7 @@ const distFiles = new Set( "publication-events", "publication-store", "publish", + "reasoning-summary", "result", "runtime", "scan-activity", @@ -327,7 +332,17 @@ assertExpectedGitHead( const internalMarker = /(?:internal\.api\.openai\.org|gateway\.[a-z0-9.-]*internal|\.openai\.org|openai\.firewall\.socket\.dev|socket\x2dfirewall\x2dregistry|openai\.(?:enterprise\.)?slack\.com|app\.slack\.com\/client|(?:app\.notion\.com\/p|notion\.so)\/openai|linear\.app\/openai|(?:github\.com[:/]|api\.github\.com\/repos\/|raw\.githubusercontent\.com\/)openai\/openai(?:\.git)?(?:[^a-z0-9_-]|$)|LicenseRef\x2dProprietary|\/Users\/|\/home\/dev-user|flow\.apps\.openai\.org|(?:^|[^a-z0-9_-])go\/[a-z0-9_-]+)/iu; -const payloads = [archiveBytes.toString("utf8")]; +// Scan compressed assets after decoding them below. Their binary bytes can +// coincidentally match text references; keep scanning every tar header and all +// other entry contents. +const readableArchive = Buffer.from(archiveBytes); +for (const file of files) { + if (!/\.br(?:\.part-[0-9]+)?$/iu.test(file)) continue; + const bytes = archiveFile(file); + const start = bytes.byteOffset - archiveBytes.byteOffset; + readableArchive.fill(0, start, start + bytes.byteLength); +} +const payloads = [readableArchive.toString("utf8")]; const compressedFiles = [...files].filter((file) => /\.br$/iu.test(file)); const compressedParts = new Map(); for (const file of files) { diff --git a/sdk/typescript/scripts/fixtures/package-deep-codex.mjs b/sdk/typescript/scripts/fixtures/package-deep-codex.mjs new file mode 100644 index 000000000..4d645dfb9 --- /dev/null +++ b/sdk/typescript/scripts/fixtures/package-deep-codex.mjs @@ -0,0 +1,175 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; +import { appendFile, writeFile } from "node:fs/promises"; +import { basename, join } from "node:path"; +import { createInterface } from "node:readline"; +import { startRpc } from "./package-rpc.mjs"; + +try { + await run(); + process.exit(0); +} catch (error) { + await trace({ phase: "fixture-error", error: error.stack }); + console.error(error); + process.exit(1); +} + +async function trace(event) { + await appendFile( + process.env.PACKAGE_DEEP_TRACE, + `${JSON.stringify(event)}\n`, + ); +} + +async function run() { + const args = process.argv.slice(2); + if (args.includes("app-server")) { + await trace({ phase: "preflight", args }); + for await (const line of createInterface({ input: process.stdin })) { + const message = JSON.parse(line); + if (message.id === undefined) continue; + let result; + switch (message.method) { + case "initialize": + result = { userAgent: "package-fixture" }; + break; + case "config/read": + result = { + config: { + default_permissions: "codex_security_deep_scan_worker", + permissions: { + codex_security_deep_scan_worker: { + extends: ":read-only", + filesystem: { ":root": "read" }, + network: { enabled: false }, + }, + }, + }, + origins: {}, + layers: null, + }; + break; + case "permissionProfile/list": + result = { + data: [ + { + id: "codex_security_deep_scan_worker", + description: null, + allowed: true, + }, + ], + nextCursor: null, + }; + break; + case "account/read": + result = { account: null, requiresOpenaiAuth: true }; + break; + default: + throw new Error(`Unexpected preflight method: ${message.method}`); + } + console.log(JSON.stringify({ jsonrpc: "2.0", id: message.id, result })); + } + return; + } + let prompt = ""; + for await (const chunk of process.stdin) prompt += chunk; + const config = {}; + for (let index = 0; index < args.length; index++) { + if (args[index] !== "-c" && args[index] !== "--config") continue; + const setting = args[++index]; + const equals = setting.indexOf("="); + config[setting.slice(0, equals)] = setting.slice(equals + 1); + } + const prefix = "mcp_servers.cs_artifacts."; + const env = Object.fromEntries( + Object.entries(config) + .filter(([name]) => name.startsWith(`${prefix}env.`)) + .map(([name, value]) => [ + name.slice(`${prefix}env.`.length), + JSON.parse(value), + ]), + ); + const root = env.CODEX_SECURITY_ARTIFACT_ROOT; + assert.ok(root, "The real worker must supply its bound artifact root."); + assert.equal(config["mcp_servers.codex-security.enabled"], "false"); + const layout = env.CODEX_SECURITY_ARTIFACT_LAYOUT; + const threadId = `package-${layout}-${basename(root)}-${basename(join(root, ".."))}`; + console.log(JSON.stringify({ type: "thread.started", thread_id: threadId })); + if ( + layout === "worker" && + basename(join(root, "..")) === "discovery-0002" && + process.env.PACKAGE_DEEP_HOLD && + existsSync(process.env.PACKAGE_DEEP_HOLD) + ) { + await trace({ phase: "held", scanId: env.CODEX_SECURITY_SCAN_ID }); + await new Promise(() => setInterval(() => {}, 1_000)); + } + const server = await startRpc( + JSON.parse(config[`${prefix}command`]), + JSON.parse(config[`${prefix}args`]), + { cwd: root, env: { ...process.env, ...env } }, + ); + let complete = true; + try { + if (layout === "worker") { + const draft = { + scanId: env.CODEX_SECURITY_SCAN_ID, + findings: [], + coverage: { + completeness: "complete", + surfaces: [], + explicitExclusions: [], + deferred: [], + }, + }; + const marker = process.env.PACKAGE_DEEP_EMPTY_ONCE; + if (marker && !existsSync(marker)) { + await writeFile(marker, "process completed without a final artifact"); + complete = false; + } else { + await server.call("record_codex_security_scan_draft", { + ...draft, + complete: false, + }); + await server.call("record_codex_security_scan_draft", { + ...draft, + complete: true, + }); + } + } else { + assert.equal(layout, "reducer"); + const inputs = await server.call( + "get_codex_security_deep_reducer_inputs", + {}, + ); + assert.ok(inputs.discoveries.length > 0); + await server.call("record_codex_security_deep_reduction", { + scanId: env.CODEX_SECURITY_SCAN_ID, + findings: [], + }); + } + await trace({ + phase: layout, + complete, + resumed: args.includes("resume"), + scanId: env.CODEX_SECURITY_SCAN_ID, + home: process.env.CODEX_HOME, + hasApiKey: process.env.CODEX_API_KEY === "synthetic-package-deep-key", + root, + args, + }); + console.log( + JSON.stringify({ + type: "turn.completed", + usage: { + input_tokens: 1, + cached_input_tokens: 0, + output_tokens: 1, + }, + }), + ); + } finally { + await server.close(); + } +} diff --git a/sdk/typescript/scripts/fixtures/package-deep-scan.mjs b/sdk/typescript/scripts/fixtures/package-deep-scan.mjs new file mode 100644 index 000000000..702704903 --- /dev/null +++ b/sdk/typescript/scripts/fixtures/package-deep-scan.mjs @@ -0,0 +1,595 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; +import { + chmod, + copyFile, + cp, + mkdir, + mkdtemp, + readFile, + realpath, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, dirname, join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { startRpc } from "./package-rpc.mjs"; +import { packageSmokeTimeouts } from "../package-smoke-timeouts.mjs"; + +const installedRoot = await realpath(process.argv[2]); +const root = await realpath( + await mkdtemp(join(tmpdir(), "package deep % fixture-")), +); +const installedPlugin = join(installedRoot, "_bundled_plugin"); +try { + const detachedPlugin = join(root, "standalone plugin %", "codex-security"); + await cp(installedPlugin, detachedPlugin, { recursive: true }); + // Assert physical independence instead of merely changing the working directory. + for (let ancestor = detachedPlugin; ; ancestor = dirname(ancestor)) { + for (const dependency of ["node_modules", join("sdk", "typescript")]) { + await assert.rejects(stat(join(ancestor, dependency)), { + code: "ENOENT", + }); + } + if (ancestor === dirname(ancestor)) break; + } + for (const name of [ + "package-deep-codex.mjs", + "package-rpc.mjs", + "package-deep-spawn.mjs", + ]) { + await copyFile(new URL(name, import.meta.url), join(root, name)); + } + const executable = join( + root, + process.platform === "win32" + ? "package-codex.exe" + : "package-deep-codex.mjs", + ); + if (process.platform === "win32") + await copyFile(process.execPath, executable); + await chmod(executable, 0o700); + + await runInstalledSdk(installedPlugin, executable); + await runDetachedPlugin(detachedPlugin, executable); + console.log( + "Validated installed SDK and detached plugin: real Deep processes, bound artifact tools, checkpoints, reducer acceptance, restart before finalization, and sealed results.", + ); +} catch (error) { + for (const name of ["installed", "detached"]) { + try { + error.message += `\n${await readFile(join(root, name, "executions.jsonl"), "utf8")}`; + } catch (readError) { + if (readError.code !== "ENOENT") throw readError; + } + } + throw error; +} finally { + await rm(root, { + recursive: true, + force: true, + maxRetries: 10, + retryDelay: 100, + }); +} + +async function fixture(name, pluginRoot, executable) { + const directory = join(root, name); + const target = join(directory, "target with spaces %"); + const home = join(directory, "home"); + await mkdir(target, { recursive: true }); + await mkdir(join(home, "codex-security"), { recursive: true }); + await writeFile( + join(target, "fixture.py"), + "print('synthetic package fixture')\n", + ); + await writeFile( + join(home, "codex-security", "config.toml"), + "[deep_scan]\nworkers = 1\nsubagents = 0\nstop_after_no_new = 1\nmax_discovery_runs = 2\n", + ); + const env = Object.fromEntries( + [ + "PATH", + "Path", + "SystemRoot", + "WINDIR", + "ComSpec", + "PATHEXT", + "TMP", + "TEMP", + "TMPDIR", + ] + .filter((key) => process.env[key] !== undefined) + .map((key) => [key, process.env[key]]), + ); + Object.assign(env, { + HOME: home, + USERPROFILE: home, + CODEX_HOME: home, + CODEX_CLI_PATH: executable, + CODEX_SECURITY_PLUGIN_ROOT: pluginRoot, + CODEX_SECURITY_STATE_DIR: join(directory, "state"), + CODEX_SECURITY_SCAN_ROOT: join(directory, "scans"), + PYTHON: process.env.PYTHON || "python3", + OPENAI_API_KEY: "synthetic-package-deep-key", + ...(process.platform === "win32" + ? { + PACKAGE_DEEP_EXECUTABLE: executable, + NODE_OPTIONS: `--import=${pathToFileURL(join(root, "package-deep-spawn.mjs")).href}`, + } + : {}), + PACKAGE_DEEP_TRACE: join(directory, "executions.jsonl"), + }); + return { directory, target, home, env, pluginRoot }; +} + +function metadata(f, owner) { + return { + "openai/threadId": owner, + "codex/sandbox-state-meta": { + permissionProfile: { + type: "managed", + file_system: { + type: "restricted", + entries: [ + { + path: { type: "special", value: { kind: "root" } }, + access: "read", + }, + ], + }, + network: "restricted", + }, + sandboxCwd: pathToFileURL(f.target).href, + }, + "x-codex-turn-metadata": { model: "gpt-5.5", reasoning_effort: "high" }, + }; +} + +function server(f, env = f.env) { + return startRpc( + process.execPath, + [join(f.pluginRoot, "mcp", "server.mjs"), "--stdio"], + { + cwd: f.target, + env, + requestTimeoutMs: packageSmokeTimeouts().commandTimeoutMs, + }, + ); +} + +async function runDetachedPlugin(pluginRoot, executable) { + const f = await fixture("detached", pluginRoot, executable); + const owner = "package-detached-owner"; + f.env.PACKAGE_DEEP_HOLD = join(f.directory, "hold-second-worker"); + await writeFile(f.env.PACKAGE_DEEP_HOLD, "hold"); + let rpc = await server(f); + let scanId; + let scanDir; + let partial; + const handoffClaimToken = randomUUID(); + try { + const opened = await rpc.call( + "open_codex_security_workspace", + { + targetPath: f.target, + scope: ".", + mode: "deep", + }, + metadata(f, owner), + ); + const sessionId = opened.workspace.id; + await rpc.call( + "submit_codex_security_setup", + { + sessionId, + targetPath: f.target, + scope: ".", + mode: "deep", + }, + metadata(f, owner), + ); + const started = await rpc.call( + "start_codex_security_scan", + { sessionId }, + metadata(f, owner), + ); + ({ scanId, scanDir } = started.workspace.results); + await rpc.call( + "claim_codex_security_scan_handoff_delivery", + { + scanId, + claimToken: handoffClaimToken, + }, + metadata(f, owner), + ); + await rpc.call( + "attach_codex_security_scan_continuation_thread", + { + scanId, + claimToken: handoffClaimToken, + threadId: owner, + }, + metadata(f, owner), + ); + const pending = rpc + .call( + "start_codex_security_deep_scan", + { scanId, handoffClaimToken }, + metadata(f, owner), + ) + .catch((error) => error); + const deadline = Date.now() + 30_000; + while (!(await readExecutions(f)).some((entry) => entry.phase === "held")) { + assert.ok( + Date.now() < deadline, + "Second worker did not reach the interruption boundary.", + ); + await new Promise((resolve) => setTimeout(resolve, 50)); + } + partial = ( + await workbench(f, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + owner, + ]) + ).deepScan; + assert.equal( + partial.workers.filter( + (worker) => + worker.kind === "discovery" && worker.status === "succeeded", + ).length, + 1, + ); + await rpc.close(); + await pending; + } finally { + await rpc.close(); + } + // Simulate an expired owner lease without waiting for wall-clock expiry. Keep + // the real stored workers/results and use the production recovery path. + await rm( + join( + scanDir, + "artifacts", + "deep_discovery", + `coordinator-heartbeat-${partial.coordinatorGeneration}.json`, + ), + { force: true }, + ); + await promisify(execFile)( + f.env.PYTHON, + [ + "-c", + "import sqlite3,sys; c=sqlite3.connect(sys.argv[1]); c.execute('UPDATE deep_scan_runs SET updated_at = ? WHERE scan_id = ?', ('2000-01-01T00:00:00Z',sys.argv[2])); c.commit()", + join(f.env.CODEX_SECURITY_STATE_DIR, "workbench.sqlite3"), + scanId, + ], + { env: f.env }, + ); + await rm(f.env.PACKAGE_DEEP_HOLD); + rpc = await server(f); + try { + const result = await rpc.call( + "start_codex_security_deep_scan", + { scanId, handoffClaimToken }, + metadata(f, owner), + ); + await assertDraft(result.manifestPath); + const recovered = ( + await workbench(f, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + owner, + ]) + ).deepScan; + assert.equal( + recovered.coordinatorGeneration, + partial.coordinatorGeneration + 1, + ); + assert.equal(recovered.dispatchedCount, 2); + const retained = partial.workers.find( + (worker) => worker.status === "succeeded", + ); + assert.equal( + recovered.workers.find((worker) => worker.id === retained.id).status, + "succeeded", + ); + } finally { + await rpc.close(); + } + // Publication uses the saved aggregate after the recovered executor exits too. + rpc = await server(f); + try { + await rpc.call( + "complete_codex_security_scan", + { scanId, handoffClaimToken }, + metadata(f, owner), + ); + const completed = await rpc.call( + "get_codex_security_completed_scan", + { scanId, handoffClaimToken }, + metadata(f, owner), + ); + assert.equal(completed.manifest.scan.id, scanId); + assert.equal(completed.manifest.scan.status, "completed"); + assert.ok(completed.manifest.scan.sealedAt); + } finally { + await rpc.close(); + } + await assertSavedState(f, scanId, owner); + await assertExecutions(f, scanId, 4); +} + +async function workbench(f, args) { + const { stdout } = await promisify(execFile)( + f.env.PYTHON, + [join(f.pluginRoot, "scripts", "workbench_db.py"), ...args], + { + env: f.env, + cwd: f.target, + maxBuffer: 4 * 1024 * 1024, + }, + ); + return JSON.parse(stdout); +} + +async function runInstalledSdk(pluginRoot, executable) { + const f = await fixture("installed", pluginRoot, executable); + f.env.PACKAGE_DEEP_EMPTY_ONCE = join( + f.directory, + "missing-result-completion", + ); + const sdk = await import( + pathToFileURL(join(installedRoot, "dist", "index.js")).href + ); + const manifest = JSON.parse( + await readFile(join(pluginRoot, ".codex-plugin", "plugin.json"), "utf8"), + ); + const owner = "package-sdk-owner"; + const postScanPrompt = "Explain the completed synthetic scan."; + const prompts = []; + let threadCount = 0; + let manifestBeforeFollowUp; + let scanId; + const client = new sdk.CodexSecurity( + { pythonPath: f.env.PYTHON }, + { + environment: f.env, + prepareRuntime: async () => ({ + codexHome: f.home, + environment: f.env, + credentialsAvailable: true, + plugin: { + pluginRoot, + marketplaceRoot: pluginRoot, + installedRoot: pluginRoot, + marketplaceName: "codex-security-sdk", + name: manifest.name, + version: manifest.version, + }, + }), + // Replace only the parent model's tool choice. The installed SDK registers + // and finalizes the scan; the packaged MCP runs the real Deep lifecycle. + createCodex({ env, apiKey }) { + return { + startThread() { + threadCount += 1; + return { + id: owner, + async runStreamed(prompt) { + prompts.push(prompt); + return { + events: (async function* () { + yield { type: "thread.started", thread_id: owner }; + if (prompts.length > 1) { + assert.equal(prompt, postScanPrompt); + manifestBeforeFollowUp = await readFile( + join(env.CODEX_SECURITY_SCAN_DIR, "scan-manifest.json"), + "utf8", + ); + const completed = JSON.parse(manifestBeforeFollowUp); + assert.equal(completed.scan.status, "completed"); + assert.ok(completed.scan.sealedAt); + yield { + type: "turn.completed", + usage: { + input_tokens: 100_000, + cached_input_tokens: 0, + output_tokens: 100_000, + }, + }; + return; + } + scanId = env.CODEX_SECURITY_SCAN_ID; + // The pinned SDK maps its apiKey option to this child variable. + const rpc = await server(f, { + ...env, + ...(apiKey ? { CODEX_API_KEY: apiKey } : {}), + }); + try { + const result = await rpc.call( + "start_codex_security_deep_scan", + { scanId }, + metadata(f, owner), + ); + await assertDraft(result.manifestPath); + } finally { + await rpc.close(); + } + yield { + type: "turn.completed", + usage: { + input_tokens: 1, + cached_input_tokens: 0, + output_tokens: 1, + }, + }; + })(), + }; + }, + }; + }, + }; + }, + }, + ); + try { + const result = await client.run(f.target, { + mode: "deep", + auth: "api-key", + workers: 1, + subagents: 0, + maxDiscoveryRuns: 2, + stopAfterNoNew: 1, + postScanPrompt, + outputDir: join(f.directory, "output"), + }); + assert.equal(threadCount, 1); + assert.equal(prompts.length, 2); + assert.equal(prompts[1], postScanPrompt); + assert.equal(result.threadId, owner); + assert.equal(result.manifest.scan.status, "completed"); + assert.ok(result.manifest.scan.sealedAt); + assert.equal(result.manifest.scan.id, scanId); + assert.deepEqual(result.findings.findings, []); + assert.equal( + await readFile(result.manifestPath, "utf8"), + manifestBeforeFollowUp, + ); + assert.ok(result.cost === null || result.cost.inputTokens < 100_000); + assert.equal(result.toJSON().threadId, owner); + assert.ok( + (await readFile(join(f.directory, "output", "report.md"), "utf8")) + .length > 0, + ); + } finally { + await client.close(); + } + await assertSavedState(f, scanId, owner); + await assertExecutions(f, scanId, 4); +} + +async function assertSavedState(f, scanId, owner) { + const { deepScan } = await workbench(f, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + owner, + ]); + assert.equal(deepScan.status, "succeeded"); + if (deepScan.workflowVersion === "deep-security-scan/v2") { + const selected = deepScan.finalizationInput; + assert.ok( + selected, + "The completed v2 scan retains its finalization input.", + ); + assert.equal(selected.version, 1); + assert.equal(selected.terminalReason, deepScan.terminalReason); + assert.deepEqual(selected.omittedWorkerIds, []); + await assertDigest( + resolve(deepScan.scanDir, selected.resultPath), + selected.resultSha256, + ); + const workers = deepScan.workers.filter( + (worker) => worker.status === "succeeded", + ); + assert.equal(workers.length, 3); + for (const worker of workers) { + const attempt = deepScan.attempts.find( + (entry) => + entry.workerId === worker.id && entry.attempt === worker.attempt, + ); + assert.ok(attempt, "Each accepted worker retains its execution attempt."); + assert.equal(attempt.status, "succeeded"); + await assertDigest( + attempt.acceptedResultPath, + attempt.acceptedResultSha256, + ); + if (worker.kind === "dedup") { + assert.equal(selected.resultSha256, attempt.acceptedResultSha256); + } else { + const input = deepScan.dedupInputs.find( + (entry) => entry.discoveryWorkerId === worker.id, + ); + assert.ok(input, "The reducer retains each accepted discovery input."); + assert.equal(input.attempt, worker.attempt); + assert.equal(input.resultManifestSha256, attempt.acceptedResultSha256); + await assertDigest( + input.resultManifestPath, + input.resultManifestSha256, + ); + } + } + assert.equal(deepScan.dedupInputs.length, 2); + } + console.log( + JSON.stringify({ + fixture: basename(f.directory), + workflowVersion: deepScan.workflowVersion, + attempts: deepScan.attempts?.length ?? null, + selectedFinalization: deepScan.finalizationInput != null, + }), + ); +} + +async function assertDigest(path, expected) { + const bytes = await readFile(path); + assert.equal(createHash("sha256").update(bytes).digest("hex"), expected); +} + +async function assertDraft(path) { + const document = JSON.parse(await readFile(path, "utf8")); + const findings = JSON.parse( + await readFile(join(dirname(path), "findings.json"), "utf8"), + ); + assert.deepEqual(findings.findings, []); + assert.ok(document.scan.target); +} + +async function readExecutions(f) { + try { + return (await readFile(f.env.PACKAGE_DEEP_TRACE, "utf8")) + .trim() + .split("\n") + .filter(Boolean) + .map(JSON.parse); + } catch (error) { + if (error.code === "ENOENT") return []; + throw error; + } +} + +async function assertExecutions(f, scanId, preflights = 3) { + const executions = await readExecutions(f); + const workers = executions.filter((entry) => entry.phase === "worker"); + const reducers = executions.filter((entry) => entry.phase === "reducer"); + const incomplete = f.env.PACKAGE_DEEP_EMPTY_ONCE ? 1 : 0; + assert.equal(workers.length, 2 + incomplete); + assert.equal(workers.filter((entry) => !entry.complete).length, incomplete); + assert.equal(workers.filter((entry) => entry.resumed).length, incomplete); + assert.equal(reducers.length, 1); + assert.equal( + executions.filter((entry) => entry.phase === "preflight").length, + preflights, + ); + for (const execution of [...workers, ...reducers]) { + assert.equal(execution.scanId, scanId); + assert.equal(execution.home, f.home); + assert.equal(execution.hasApiKey, true); + assert.equal( + execution.args[execution.args.indexOf("--model") + 1], + "gpt-5.5", + ); + assert.ok(execution.args.includes('approval_policy="never"')); + } +} diff --git a/sdk/typescript/scripts/fixtures/package-deep-spawn.mjs b/sdk/typescript/scripts/fixtures/package-deep-spawn.mjs new file mode 100644 index 000000000..e59440558 --- /dev/null +++ b/sdk/typescript/scripts/fixtures/package-deep-spawn.mjs @@ -0,0 +1,20 @@ +import childProcess from "node:child_process"; +import { syncBuiltinESMExports } from "node:module"; +import { win32 } from "node:path"; +import { fileURLToPath } from "node:url"; + +// Windows cannot execute the POSIX fixture's shebang. Preserve the selected +// native executable and its options, inserting only the deterministic protocol +// script, as in the worker launch tests. Every other child runs unchanged. +const executable = win32.toNamespacedPath(process.env.PACKAGE_DEEP_EXECUTABLE); +const script = fileURLToPath( + new URL("package-deep-codex.mjs", import.meta.url), +); +const spawn = childProcess.spawn; +childProcess.spawn = (command, args, options) => + spawn( + command, + win32.toNamespacedPath(command) === executable ? [script, ...args] : args, + options, + ); +syncBuiltinESMExports(); diff --git a/sdk/typescript/scripts/fixtures/package-rpc.mjs b/sdk/typescript/scripts/fixtures/package-rpc.mjs new file mode 100644 index 000000000..ceb6156f2 --- /dev/null +++ b/sdk/typescript/scripts/fixtures/package-rpc.mjs @@ -0,0 +1,90 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { createInterface } from "node:readline"; + +// This client deliberately uses only Node builtins. Detached plugin tests must +// not resolve an MCP client or SDK from the checkout's node_modules. +export async function startRpc(command, args, options) { + const { requestTimeoutMs = 30_000, ...spawnOptions } = options; + const child = spawn(command, args, { + ...spawnOptions, + stdio: ["pipe", "pipe", "pipe"], + windowsHide: true, + }); + let sequence = 0; + let stderr = ""; + const pending = new Map(); + child.stderr.setEncoding("utf8").on("data", (chunk) => { + stderr += chunk; + }); + createInterface({ input: child.stdout }).on("line", (line) => { + const response = JSON.parse(line); + const waiter = pending.get(response.id); + if (!waiter) return; + pending.delete(response.id); + clearTimeout(waiter.timer); + if (response.error) + waiter.reject(new Error(JSON.stringify(response.error))); + else waiter.resolve(response.result); + }); + const exited = once(child, "exit"); + child.on("exit", (code, signal) => { + for (const waiter of pending.values()) { + clearTimeout(waiter.timer); + waiter.reject( + new Error(`Fixture RPC exited (${code}, ${signal}): ${stderr}`), + ); + } + pending.clear(); + }); + const client = { + child, + request(method, params = {}) { + return new Promise((resolve, reject) => { + const id = ++sequence; + const timer = setTimeout(() => { + pending.delete(id); + reject(new Error(`Fixture RPC timed out: ${method}\n${stderr}`)); + }, requestTimeoutMs); + pending.set(id, { resolve, reject, timer }); + child.stdin.write( + `${JSON.stringify({ jsonrpc: "2.0", id, method, params })}\n`, + ); + }); + }, + async call(name, args, meta) { + const result = await this.request("tools/call", { + name, + arguments: args, + ...(meta ? { _meta: meta } : {}), + }); + assert.notEqual(result.isError, true, JSON.stringify(result)); + return result.structuredContent ?? JSON.parse(result.content[0].text); + }, + async close() { + if (child.exitCode !== null || child.signalCode !== null) return; + child.stdin.end(); + const timeout = setTimeout(() => child.kill("SIGKILL"), 5_000); + try { + await exited; + } finally { + clearTimeout(timeout); + } + }, + }; + try { + await client.request("initialize", { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "installed-deep-fixture", version: "1.0.0" }, + }); + child.stdin.write( + `${JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" })}\n`, + ); + return client; + } catch (error) { + await client.close(); + throw error; + } +} diff --git a/sdk/typescript/scripts/smoke-package.mjs b/sdk/typescript/scripts/smoke-package.mjs index 60945f75d..39f834a5d 100644 --- a/sdk/typescript/scripts/smoke-package.mjs +++ b/sdk/typescript/scripts/smoke-package.mjs @@ -808,6 +808,15 @@ try { ); await smokeNestedDeepScanWorker(installedRoot, consumer); + run( + process.execPath, + [ + join(packageRoot, "scripts", "fixtures", "package-deep-scan.mjs"), + installedRoot, + ], + { cwd: consumer }, + ); + console.log( `Validated installed ${packageManifest.name}@${packageManifest.version}: public import, NodeNext types, CLI, SDK lifecycle, credential locking, ${expectedPluginFiles.length} bundled plugin files, MCP initialization, bundled Codex version, dashboard assets, and a nested worker without global codex.`, ); diff --git a/sdk/typescript/src/accepted-audit.ts b/sdk/typescript/src/accepted-audit.ts new file mode 100644 index 000000000..ee526a955 --- /dev/null +++ b/sdk/typescript/src/accepted-audit.ts @@ -0,0 +1,49 @@ +export interface ScanDraftInput { + scanId: string; + complete?: boolean; + handoffClaimToken?: string; + scope?: Record; + threatModel?: Record; + findings: Record[]; + coverage: Record; +} + +/** Accepted evidence may still describe partial or unknown source coverage. */ +export interface AuditEvidence { + checkpoint?: ScanDraftInput; + accepted?: ScanDraftInput; +} + +export type AuditOutcome = AuditEvidence & + ( + | { status: "accepted"; execution: Execution; accepted: ScanDraftInput } + | { status: "checkpoint"; execution: Execution } + ); + +/** One attempt; enclosing callers own retries and public completion. */ +export async function runAcceptedAudit(input: { + signal: AbortSignal; + execute: () => Promise; + accept: (execution: Execution) => Promise; +}): Promise> { + input.signal.throwIfAborted(); + const execution = await input.execute(); + input.signal.throwIfAborted(); + const evidence = await input.accept(execution); + input.signal.throwIfAborted(); + return evidence.accepted === undefined + ? { ...evidence, execution, status: "checkpoint" } + : { + ...evidence, + execution, + status: "accepted", + accepted: evidence.accepted, + }; +} + +/** Process completion alone does not accept an unfinished audit checkpoint. */ +export function auditEvidence(checkpoint: ScanDraftInput): AuditEvidence { + return checkpoint.complete === false + ? { checkpoint } + : { checkpoint, accepted: checkpoint }; +} diff --git a/sdk/typescript/src/api.ts b/sdk/typescript/src/api.ts index f9d7b4033..3fc5e5948 100644 --- a/sdk/typescript/src/api.ts +++ b/sdk/typescript/src/api.ts @@ -1,5 +1,15 @@ /// +import { scanPreflightCodexConfig } from "./preflight-config.js"; +import { captureOriginalReasoningSummary } from "./reasoning-summary.js"; +export { scanPreflightCodexConfig } from "./preflight-config.js"; +import { resumeSelectedDeepScan } from "./deep-scan-finalization.js"; +import { + auditEvidence, + runAcceptedAudit, + type ScanDraftInput, +} from "./accepted-audit.js"; +import { pathToFileURL } from "node:url"; import { statSync } from "node:fs"; import { chmod, @@ -23,13 +33,15 @@ import { resolve, sep, } from "node:path"; -import { - Codex, - type CodexOptions, - type ThreadOptions, - type TurnOptions, -} from "@openai/codex-sdk"; +import { type CodexOptions, type ThreadOptions } from "@openai/codex-sdk"; import { z } from "incur"; +import { + createCodexClient, + readCodexSessionTurn, + type CodexSessionClient as CodexClientLike, + type CodexSessionThread as CodexThreadLike, + type CodexSessionEvent as ScanEvent, +} from "./codex-session.js"; import { CODEX_AUTH_CONFIG_KEYS, NO_CREDENTIALS_MESSAGE, @@ -68,6 +80,7 @@ import { type ScanCost, type ScanSessionEvent, } from "./cost.js"; +import type { ScanExecutionAttribution } from "./scan-sessions.js"; import { DeepScanProgressTracker, type DeepScanProgress, @@ -216,24 +229,6 @@ import { validateMode, } from "./targets.js"; -interface CodexThreadLike { - readonly id: string | null; - runStreamed( - input: string, - options: TurnOptions, - ): Promise<{ events: AsyncGenerator }>; -} - -interface ScanEvent { - readonly type: string; - readonly [key: string]: unknown; -} - -interface CodexClientLike { - startThread(options: ThreadOptions): CodexThreadLike; - resumeThread?(threadId: string, options: ThreadOptions): CodexThreadLike; -} - interface PreparedRuntime { codexHome: string; persistentCredentialHome?: boolean; @@ -453,7 +448,7 @@ interface ClientDependencies { } const DEFAULT_DEPENDENCIES: ClientDependencies = { - createCodex: (options) => new Codex(options), + createCodex: createCodexClient, environment: process.env, }; @@ -1010,15 +1005,7 @@ export class CodexSecurity { `permissions.${POLICY_PERMISSION_PROFILE}.filesystem=${inlineToml(policyFilesystemPermissions(inputs.gitMetadataPaths))}`, ], ); - const reportCost = (current: Readonly): void => { - const total = addScanCosts(accumulatedCost, current); - if (completeCost) - notifyObserver( - "onCost", - options.onCost, - options.onObserverError, - total, - ); + const enforceCostLimit = (total: Readonly): void => { if ( options.maxCostUsd !== undefined && total.estimatedUsd > options.maxCostUsd @@ -1030,6 +1017,17 @@ export class CodexSecurity { ); } }; + const reportCost = (current: Readonly): void => { + const total = addScanCosts(accumulatedCost, current); + if (completeCost) + notifyObserver( + "onCost", + options.onCost, + options.onObserverError, + total, + ); + enforceCostLimit(total); + }; const outputSchema = securityPolicyStageOutputSchema(); const run = async ( stage: SecurityPolicyStage, @@ -1053,6 +1051,10 @@ export class CodexSecurity { options.onCost === undefined && options.maxCostUsd === undefined ? undefined : reportCost, + onCostLowerBound: + options.maxCostUsd === undefined + ? undefined + : (cost) => enforceCostLimit(addScanCosts(accumulatedCost, cost)), onError: (error) => { if (options.maxCostUsd !== undefined) budgetController.abort(error); else @@ -1213,14 +1215,35 @@ export class CodexSecurity { let preparedTargetWarnings: string[] = []; let runPostScan: (() => ReturnType) | null = null; + let selectedDeepFinalization = false; + let observedScanThreadId: string | undefined; let activeScan: { id: string; options: WorkbenchCommandOptions; + mode: ScanMode; } | null = null; const prepareArtifactRestorer = this.#dependencies.prepareScanArtifactRestorer ?? prepareScanArtifactRestorer; const workbench = this.#dependencies.runWorkbench ?? runWorkbench; + const recoverCompletedScan = async ( + commandOptions: WorkbenchCommandOptions, + scanId: string, + error: unknown, + completionArgs: readonly string[], + ): Promise => { + const saved = await workbench(commandOptions, [ + "get-scan", + "--scan-id", + scanId, + ]).catch(() => null); + const savedScan = saved?.["scan"]; + const progress = isRecord(savedScan) ? savedScan["progress"] : null; + if (!isRecord(progress) || progress["status"] !== "complete") throw error; + // A lost response can follow a durable seal. Only the existing normal + // completion command can validate and return that committed receipt. + return workbench(commandOptions, completionArgs); + }; try { const checkOpen = (): void => { this.#requireOpen(); @@ -1282,13 +1305,12 @@ export class CodexSecurity { const { runtime, runtimeHome, - effectiveConfig, - preflightConfig, modelProvider, authentication, approvalPolicy, python, } = session; + let { effectiveConfig, preflightConfig } = session; releaseCredentialHome = session.releaseCredentialHome; const deepScanConfigPath = mode === "deep" @@ -1347,6 +1369,38 @@ export class CodexSecurity { ); checkOpen(); + if (mode === "deep" && options.resumeScanId === undefined) { + const summary = await captureOriginalReasoningSummary({ + config: session.sessionConfig, + command: this.#codexCommand(), + cwd: scanDir, + environment: { + ...withoutOpenAiApiKeys( + this.#createSessionEnvironment(session, {}, options.auth), + ), + ...(session.externalProvider === null && session.apiKey !== null + ? { CODEX_API_KEY: session.apiKey } + : {}), + }, + signal, + }); + if (summary !== undefined) { + effectiveConfig = { + ...effectiveConfig, + model_reasoning_summary: summary, + }; + preflightConfig = scanPreflightCodexConfig(effectiveConfig); + session.effectiveConfig = effectiveConfig; + session.preflightConfig = preflightConfig; + session.sessionConfig = { + ...session.sessionConfig, + model_reasoning_summary: summary, + }; + if (runtime.configPath !== undefined) + await writeCodexConfig(runtime.configPath, preflightConfig); + } + } + const shellPluginRoot = runtime.plugin.pluginRoot; const canonicalShellPluginRoot = await realpath(shellPluginRoot); const pluginRelativeToHome = relative( @@ -1437,6 +1491,14 @@ export class CodexSecurity { `Could not track scan activity: ${errorMessage(error)}`, ); }; + const enforceCostLimit = (cost: Readonly): boolean => { + if (maxCostUsd === undefined || cost.estimatedUsd <= maxCostUsd) + return false; + costAbortController.abort( + new ScanCostLimitExceededError(maxCostUsd, cost, scanDir), + ); + return true; + }; const tracker = new ScanCostTracker({ codexHome: runtime.codexHome, model, @@ -1477,15 +1539,7 @@ export class CodexSecurity { cost, maxCostUsd, ); - if ( - maxCostUsd !== undefined && - cost.estimatedUsd > maxCostUsd - ) { - costAbortController.abort( - new ScanCostLimitExceededError(maxCostUsd, cost, scanDir), - ); - return; - } + if (enforceCostLimit(cost)) return; const request = options.onBudgetApproaching; if ( request === undefined || @@ -1550,6 +1604,8 @@ export class CodexSecurity { } }); }, + onCostLowerBound: + options.maxCostUsd === undefined ? undefined : enforceCostLimit, onError: reportTrackingError, }); costTracker = tracker; @@ -1722,7 +1778,23 @@ export class CodexSecurity { }, ); } - activeScan = { id: scanId, options: workbenchOptions }; + activeScan = { id: scanId, options: workbenchOptions, mode }; + if (mode === "deep") { + tracker.setAttributionReader(async () => { + const context = await workbench( + { ...workbenchOptions, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + const scan = context["scan"]; + if (isRecord(scan) && !("executionAttribution" in scan)) + return undefined; + return isRecord(scan) && isRecord(scan["executionAttribution"]) + ? (scan[ + "executionAttribution" + ] as unknown as ScanExecutionAttribution) + : null; + }); + } if (mode === "deep" && options.onDeepProgress !== undefined) { let progressWarningReported = false; deepProgressTracker = new DeepScanProgressTracker({ @@ -1897,6 +1969,7 @@ export class CodexSecurity { ); } thread = codex.resumeThread(resumeThreadId, threadOptions); + observedScanThreadId = resumeThreadId; tracker.start(resumeThreadId); if (budgetRecovery !== null) budgetRecovery.threadId = resumeThreadId; await tracker.refresh().catch(reportTrackingError); @@ -1920,22 +1993,63 @@ export class CodexSecurity { if (postScanPrompt?.trim()) { runPostScan = () => thread.runStreamed(postScanPrompt, { signal }); } - const { events } = await thread.runStreamed(prompt, { - signal, - }); + const recoverSelectedCompletion = async () => { + const threadId = observedScanThreadId ?? thread.id; + if (mode !== "deep" || !threadId || signal.aborted) return null; + const saved = await workbench(workbenchOptions, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + threadId, + ]).catch(() => null); + const deep = saved?.["deepScan"]; + if ( + !isRecord(deep) || + !isRecord(deep["finalizationInput"]) || + (deep["status"] !== "running" && deep["status"] !== "succeeded") + ) + return null; + selectedDeepFinalization = true; + await resumeSelectedDeepScan({ + scanId, + threadId, + pluginRoot: runtime.plugin.installedRoot, + signal, + runWorkbench: (args) => workbench(workbenchOptions, args), + }); + return { + status: "completed" as const, + threadId, + finalResponse: "", + usage: null, + lastStreamError: null, + }; + }; + const savedCompletion = resumeThreadId + ? await recoverSelectedCompletion() + : null; + const events = (async function* () { + if (savedCompletion) return; + yield* (await thread.runStreamed(prompt, { signal })).events; + })(); checkOpen(); const result = await runScanEvents({ + savedCompletion: savedCompletion ?? undefined, + recoverCompletion: recoverSelectedCompletion, thread, events, signal, scanDir, + scanId, pluginRoot: runtime.plugin.installedRoot, expectation, authentication, workbenchValidated: true, model, onThreadStarted: async (threadId) => { + observedScanThreadId = threadId; if (resumeThreadId !== undefined) { if (threadId !== resumeThreadId) { throw new CodexSecurityError( @@ -1964,6 +2078,7 @@ export class CodexSecurity { } }, onFinalize: async (usage) => { + await recoverSelectedCompletion(); if (options.validationPrompt !== undefined) { tracker.recordUsage(usage); await tracker.refresh().catch(reportTrackingError); @@ -2029,7 +2144,10 @@ export class CodexSecurity { return { usage, cost: estimateScanCost(model, usage) }; }); throwIfAborted(signal, scanDir); - if (options.maxCostUsd !== undefined && snapshot.cost === null) { + if ( + options.maxCostUsd !== undefined && + (snapshot.cost === null || snapshot.cost.coverage === "partial") + ) { notifyObserver( "onWarning", options.onWarning, @@ -2096,14 +2214,20 @@ export class CodexSecurity { onObserverError: options.onObserverError, }); checkOpen(); - const completion = await workbench(workbenchOptions, [ + const completionArgs = [ "complete-scan", "--scan-id", scanId, ...(completionCost === null ? [] : ["--cost-json", JSON.stringify(completionCost)]), - ]); + ]; + const completion = await workbench( + workbenchOptions, + completionArgs, + ).catch((error) => + recoverCompletedScan(workbenchOptions, scanId, error, completionArgs), + ); activeScan = null; const completedScan = completion["scan"]; if (isRecord(completedScan) && Array.isArray(completedScan["warnings"])) { @@ -2151,7 +2275,7 @@ export class CodexSecurity { let artifactRestorer: ScanArtifactRestorer | null = null; try { artifactRestorer = await prepareArtifactRestorer( - workbenchOptions, + { ...workbenchOptions, signal: undefined }, scanDir, ); await runScanEvents({ @@ -2159,6 +2283,7 @@ export class CodexSecurity { events: (await followUp()).events, signal, scanDir, + scanId, pluginRoot: runtime.plugin.installedRoot, expectation, model, @@ -2168,7 +2293,6 @@ export class CodexSecurity { }); checkOpen(); } catch (error) { - if (signal.aborted || this.#closed) throw error; if (artifactRestorer !== null) { for (const artifact of completedArtifacts) { try { @@ -2185,6 +2309,7 @@ export class CodexSecurity { } } } + if (signal.aborted || this.#closed) throw error; await collectResult( result.turnResult, result.threadId, @@ -2278,17 +2403,57 @@ export class CodexSecurity { options.signal?.aborted !== true ) { try { - const completion = await workbench( - { ...activeScan.options, signal: undefined }, - [ - "complete-budget-exhausted-scan", + const budgetScanId = activeScan.id; + const budgetCost = snapshot?.cost ?? { lowerBound: failure.cost }; + const completionSignal = AbortSignal.any([ + this.#abortController.signal, + ...(options.signal === undefined ? [] : [options.signal]), + ]); + const completionOptions = { + ...activeScan.options, + signal: completionSignal, + }; + const saved = await workbench(completionOptions, [ + "get-deep-scan", + "--scan-id", + activeScan.id, + "--thread-id", + budgetRecovery.threadId, + ]).catch(() => null); + const deep = saved?.["deepScan"]; + if ( + isRecord(deep) && + deep["status"] === "running" && + isRecord(deep["finalizationInput"]) + ) { + // Cost stops model work, but an already selected result can still + // finish through the local publisher. Caller cancellation remains live. + selectedDeepFinalization = true; + await resumeSelectedDeepScan({ + scanId: activeScan.id, + threadId: budgetRecovery.threadId, + pluginRoot: budgetRecovery.pluginRoot, + signal: completionSignal, + runWorkbench: (args) => workbench(completionOptions, args), + }); + } + const completion = await workbench(completionOptions, [ + "complete-budget-exhausted-scan", + "--scan-id", + budgetScanId, + "--cost-json", + JSON.stringify(budgetCost), + "--message", + failure.message.slice(0, 2400), + ]).catch((error) => + recoverCompletedScan(completionOptions, budgetScanId, error, [ + "complete-scan", "--scan-id", - activeScan.id, - "--cost-json", - JSON.stringify(snapshot?.cost ?? failure.cost), - "--message", - failure.message.slice(0, 2400), - ], + budgetScanId, + ...(snapshot?.cost + ? ["--cost-json", JSON.stringify(snapshot.cost)] + : []), + ]), ); activeScan = null; runPostScan = null; @@ -2302,10 +2467,7 @@ export class CodexSecurity { scanDir, budgetRecovery.pluginRoot, budgetRecovery.expectation, - AbortSignal.any([ - this.#abortController.signal, - ...(options.signal === undefined ? [] : [options.signal]), - ]), + completionSignal, true, ); if (result.coverage.completeness !== "partial") { @@ -2344,9 +2506,32 @@ export class CodexSecurity { return result; } catch {} } - // A failed attachment must not turn a resumable coordinator into a terminal failure. - // Deep Scan orchestration persists its own terminal failures and cancellations. - if (activeScan !== null && options.resumeScanId === undefined) { + const callerCanceledDeepScan = + activeScan?.mode === "deep" && options.signal?.aborted; + if (activeScan !== null && callerCanceledDeepScan) { + const workbenchOptions = { ...activeScan.options, signal: undefined }; + // The workbench owns the running-state check and repeated cancellation. + // Selection may have committed before the SDK received its response. + await workbench(workbenchOptions, [ + "cancel-scan", + "--scan-id", + activeScan.id, + ...(observedScanThreadId === undefined + ? [] + : ["--thread-id", observedScanThreadId]), + ]).catch(() => undefined); + } + // Publication failures remain resumable. A cost stop or explicit client close + // still uses the existing failure path to retain partial results and stop work. + if ( + activeScan !== null && + !callerCanceledDeepScan && + ((options.resumeScanId === undefined && !selectedDeepFinalization) || + (selectedDeepFinalization && + !options.signal?.aborted && + (failure instanceof ScanCostLimitExceededError || + this.#abortController.signal.aborted))) + ) { if ( options.validationPrompt !== undefined && !customValidationComplete @@ -2619,13 +2804,11 @@ export class CodexSecurity { } } - #createSessionCodex( + #createSessionEnvironment( session: PreparedSession, runtimePaths: Record, auth: ScanAuthMode = "auto", - config?: JsonObject, - configOverrides: string[] = [], - ): { codex: CodexClientLike; environment: ProcessEnvironment } { + ): ProcessEnvironment { const { runtime, python, @@ -2661,6 +2844,23 @@ export class CodexSecurity { if (session.safetyIdentifier !== undefined) { environment[SAFETY_IDENTIFIER_ENV] = session.safetyIdentifier; } + return environment; + } + + #createSessionCodex( + session: PreparedSession, + runtimePaths: Record, + auth: ScanAuthMode = "auto", + config?: JsonObject, + configOverrides: string[] = [], + ): { codex: CodexClientLike; environment: ProcessEnvironment } { + const { externalProvider, apiKey, sessionConfig } = session; + const commandAuth = hasCommandAuth(sessionConfig); + const environment = this.#createSessionEnvironment( + session, + runtimePaths, + auth, + ); const sdkCodexConfig = { ...(config ?? sessionConfig) }; // Projects and permissions already live in generated TOML files; the SDK // cannot safely encode their path and selector keys as dotted overrides. @@ -3612,6 +3812,11 @@ async function removeTargetPathsFile(path: string | null): Promise { } interface ScanEventRunOptions { + scanId?: string; + savedCompletion?: Awaited>; + recoverCompletion?: () => Promise + > | null>; thread: CodexThreadLike; events: AsyncGenerator; signal: AbortSignal; @@ -3645,112 +3850,167 @@ export async function runScanEvents( let scanStarted = false; let tacStatusReported = false; try { - const turn = await readCodexTurn({ - thread: options.thread, - events: options.events, - onEvent: async (event) => { - if (!tacStatusReported) { - const tacStatus = trustedAccessStatusFromEvent(event); - if (tacStatus !== null) { - tacStatusReported = true; - notifyObserver( - "onTrustedAccessStatus", - options.onTrustedAccessStatus, - options.onObserverError, - tacStatus, - ); - if (tacStatus !== "granted") { + let completedTurn: + | (Awaited> & { + threadId: string; + status: "completed"; + }) + | undefined; + const execute = async () => { + const turn = + options.savedCompletion ?? + (await readCodexTurn({ + thread: options.thread, + events: options.events, + onEvent: async (event) => { + if (!tacStatusReported) { + const tacStatus = trustedAccessStatusFromEvent(event); + if (tacStatus !== null) { + tacStatusReported = true; + notifyObserver( + "onTrustedAccessStatus", + options.onTrustedAccessStatus, + options.onObserverError, + tacStatus, + ); + if (tacStatus !== "granted") { + notifyObserver( + "onWarning", + options.onWarning, + options.onObserverError, + trustedAccessWarning(tacStatus, options.authentication), + ); + } + } + } + for (const activity of scanActivitiesFromEvent( + event, + options.expectation.repository, + )) { notifyObserver( - "onWarning", - options.onWarning, + "onActivity", + options.onActivity, options.onObserverError, - trustedAccessWarning(tacStatus, options.authentication), + activity, ); } - } - } - for (const activity of scanActivitiesFromEvent( - event, - options.expectation.repository, - )) { - notifyObserver( - "onActivity", - options.onActivity, - options.onObserverError, - activity, - ); - } - for (const progress of scanProgressUpdatesFromEvent(event)) { - if ( - options.expectedFilesTotal !== undefined && - progress.filesTotal !== options.expectedFilesTotal - ) { - continue; - } - notifyObserver( - "onProgress", - options.onProgress, - options.onObserverError, - progress, - ); - } - const workerStatus = workerStatusFromEvent(event); - if (workerStatus !== null) { - notifyObserver( - "onWorkerStatus", - options.onWorkerStatus, - options.onObserverError, - workerStatus, - ); - } - if (event.type === "thread.started") { - const startedThreadId = event["thread_id"]; - if (typeof startedThreadId === "string") { - await options.onThreadStarted?.(startedThreadId); - } - if (!scanStarted) { - scanStarted = true; + for (const progress of scanProgressUpdatesFromEvent(event)) { + if ( + options.expectedFilesTotal !== undefined && + progress.filesTotal !== options.expectedFilesTotal + ) { + continue; + } + notifyObserver( + "onProgress", + options.onProgress, + options.onObserverError, + progress, + ); + } + const workerStatus = workerStatusFromEvent(event); + if (workerStatus !== null) { + notifyObserver( + "onWorkerStatus", + options.onWorkerStatus, + options.onObserverError, + workerStatus, + ); + } + if (event.type === "thread.started") { + const startedThreadId = event["thread_id"]; + if (typeof startedThreadId === "string") { + await options.onThreadStarted?.(startedThreadId); + } + if (!scanStarted) { + scanStarted = true; + notifyObserver( + "onScanStarted", + options.onScanStarted, + options.onObserverError, + ); + } + } + }, + onReconnect: (message, reconnect) => { notifyObserver( - "onScanStarted", - options.onScanStarted, + "onReconnect", + options.onReconnect, options.onObserverError, + ...reconnect, + reconnectDetails(message), ); - } - } - }, - onReconnect: (message, reconnect) => { - notifyObserver( - "onReconnect", - options.onReconnect, - options.onObserverError, - ...reconnect, - reconnectDetails(message), + }, + })); + const { status, threadId, lastStreamError } = turn; + if (status !== "completed") { + throw new IncompleteScanError( + lastStreamError ?? + "Codex Security event stream ended before the turn completed.", ); - }, - }); - const { status, threadId, finalResponse, lastStreamError } = turn; - let { usage } = turn; - if (options.signal.aborted) { - throw new ScanInterruptedError( - `Codex Security scan was interrupted; partial output remains at ${options.scanDir}.`, - options.scanDir, - ); - } - if (status !== "completed") { - throw new IncompleteScanError( - lastStreamError ?? - "Codex Security event stream ended before the turn completed.", + } + if (threadId === null) { + throw new IncompleteScanError( + "Codex Security did not report a thread ID.", + ); + } + return (completedTurn = { ...turn, threadId, status }); + }; + const accept = async () => { + // Matching, custom validation and the canonical seal remain with the caller. + const [manifest, findings, coverage] = await Promise.all( + ["scan-manifest.json", "findings.json", "coverage.json"].map( + async (name) => + JSON.parse( + ( + await readScanFile(options.scanDir, name, name, options.signal) + ).toString("utf8"), + ), + ), ); + const helper = ( + await import( + pathToFileURL(join(options.pluginRoot, "mcp/helpers.mjs")).href + ) + ).default; + const draft: ScanDraftInput = helper.parseCanonicalScanDraft({ + scanId: options.scanId ?? manifest.scan.id, + manifest, + findings, + coverage, + }); + return auditEvidence(draft); + }; + let audit; + try { + audit = await runAcceptedAudit({ + signal: options.signal, + execute, + accept, + }); + } catch (error) { + if (options.signal.aborted) throw error; + const saved = await options.recoverCompletion?.(); + if (!saved || saved.status !== "completed" || saved.threadId === null) + throw error; + // The enclosing scan publishes its saved selection; acceptance then reads it. + const recovered = completedTurn ?? { ...saved, threadId: saved.threadId }; + audit = await runAcceptedAudit({ + signal: options.signal, + execute: async () => recovered, + accept, + }); } - if (threadId === null) { + if (audit.status === "checkpoint") throw new IncompleteScanError( - "Codex Security did not report a thread ID.", + "Codex Security produced only an unfinished audit checkpoint.", ); - } + const { status, threadId, finalResponse } = audit.execution; + let { usage } = audit.execution; if (options.onFinalize !== undefined) { usage = (await options.onFinalize(usage)) ?? usage; } - const result = await collectResult( + return await collectResult( { status, finalResponse, @@ -3764,13 +4024,6 @@ export async function runScanEvents( options.signal, options.workbenchValidated, ); - if (options.signal.aborted) { - throw new ScanInterruptedError( - `Codex Security scan was interrupted; partial output remains at ${options.scanDir}.`, - options.scanDir, - ); - } - return result; } catch (error) { if (options.signal.reason instanceof ScanCostLimitExceededError) { throw options.signal.reason; @@ -3798,61 +4051,28 @@ async function readCodexTurn(options: { usage: unknown; lastStreamError: string | null; }> { - let threadId = options.thread.id; - let status: "in_progress" | "completed" = "in_progress"; - let finalResponse = ""; - let usage: unknown = null; - let lastStreamError: string | null = null; - for await (const event of eventsWithOptionalUsage(options.events)) { - await options.onEvent?.(event); - if ( - event.type === "thread.started" && - typeof event["thread_id"] === "string" - ) { - threadId = event["thread_id"]; - } else if ( - event.type === "item.completed" && - isRecord(event["item"]) && - event["item"]["type"] === "agent_message" && - typeof event["item"]["text"] === "string" - ) { - finalResponse = event["item"]["text"]; - } else if (event.type === "turn.completed") { - status = "completed"; - usage = event["usage"]; - } else if (event.type === "turn.failed") { - throw new CodexSecurityError(turnFailureMessage(event["error"])); - } else if (event.type === "error" && typeof event["message"] === "string") { - const message = event["message"]; - const classification = classifyConnectionFailure(message); - if (classification === "unauthorized" || classification === "forbidden") { - throw new CodexSecurityError(message); + return readCodexSessionTurn({ + ...options, + onEvent: async (event) => { + await options.onEvent?.(event); + if (event.type === "turn.failed") { + throw new CodexSecurityError(turnFailureMessage(event["error"])); } - const reconnect = reconnectAttempt(message); - if (reconnect === null) throw new CodexSecurityError(message); - lastStreamError = message; - options.onReconnect?.(message, reconnect); - } - } - return { threadId, status, finalResponse, usage, lastStreamError }; -} - -async function* eventsWithOptionalUsage( - events: AsyncGenerator, -): AsyncGenerator { - try { - yield* events; - } catch (error) { - if ( - error instanceof TypeError && - /\b(?:null|undefined)\b/u.test(error.message) && - /\bcache_write_input_tokens\b/u.test(error.message) - ) { - yield { type: "turn.completed", usage: null }; - return; - } - throw error; - } + if (event.type === "error" && typeof event["message"] === "string") { + const message = event["message"]; + const classification = classifyConnectionFailure(message); + if ( + classification === "unauthorized" || + classification === "forbidden" + ) { + throw new CodexSecurityError(message); + } + const reconnect = reconnectAttempt(message); + if (reconnect === null) throw new CodexSecurityError(message); + options.onReconnect?.(message, reconnect); + } + }, + }); } function trustedAccessStatusFromEvent( @@ -4142,6 +4362,19 @@ function addScanCosts( previous.cacheWriteInputTokens + current.cacheWriteInputTokens, outputTokens: previous.outputTokens + current.outputTokens, estimatedUsd: previous.estimatedUsd + current.estimatedUsd, + ...(previous.coverage === "partial" || current.coverage === "partial" + ? { coverage: "partial" as const } + : {}), + ...(previous.modelCosts || + current.modelCosts || + previous.model !== current.model + ? { + modelCosts: [ + ...(previous.modelCosts ?? [previous]), + ...(current.modelCosts ?? [current]), + ], + } + : {}), ...(previous.cacheWriteInputTokensReported === false || current.cacheWriteInputTokensReported === false ? { cacheWriteInputTokensReported: false } @@ -4623,136 +4856,6 @@ function sharedCredentialCodexConfig( return scanRuntimeCodexConfig(shared, credentialHome); } -export function scanPreflightCodexConfig(config: JsonObject): JsonObject { - const safeString = (value: unknown): value is string => - typeof value === "string" && - value.length > 0 && - !/[\u0000-\u001f\u007f]/u.test(value); - const safeProfileName = (value: unknown): value is string => - safeString(value) && /^[A-Za-z0-9_-]+$/u.test(value); - const safeInteger = (value: unknown): value is number => - typeof value === "number" && Number.isSafeInteger(value) && value >= 0; - const capabilityFeatures = (value: unknown): JsonObject => { - if (!isRecord(value)) return {}; - const result: JsonObject = {}; - for (const key of ["goals", "multi_agent", "enable_fanout"]) { - if (typeof value[key] === "boolean") result[key] = value[key]; - } - const multiAgent = value["multi_agent_v2"]; - if (typeof multiAgent === "boolean") { - result["multi_agent_v2"] = multiAgent; - } else if (isRecord(multiAgent)) { - const sanitized: JsonObject = {}; - if (typeof multiAgent["enabled"] === "boolean") { - sanitized["enabled"] = multiAgent["enabled"]; - } - const capacity = multiAgent["max_concurrent_threads_per_session"]; - if (safeInteger(capacity)) { - sanitized["max_concurrent_threads_per_session"] = capacity; - } - if (Object.keys(sanitized).length > 0) { - result["multi_agent_v2"] = sanitized; - } - } - return result; - }; - const executionConfig = (source: JsonObject): JsonObject => { - const result: JsonObject = {}; - for (const key of [ - "model", - "model_reasoning_effort", - "model_reasoning_summary", - "model_provider", - "service_tier", - ]) { - const value = source[key]; - if (safeString(value)) result[key] = value; - } - const features = capabilityFeatures(source["features"]); - if (Object.keys(features).length > 0) result["features"] = features; - const agents = source["agents"]; - if (isRecord(agents)) { - const sanitized: JsonObject = {}; - for (const key of ["max_threads", "max_depth"]) { - const value = agents[key]; - if (safeInteger(value)) sanitized[key] = value; - } - if (Object.keys(sanitized).length > 0) result["agents"] = sanitized; - } - const multiagent = source["multiagent_config"]; - if (isRecord(multiagent) && safeInteger(multiagent["max_concurrency"])) { - result["multiagent_config"] = { - max_concurrency: multiagent["max_concurrency"], - }; - } - return result; - }; - const result = executionConfig(config); - // Keep the effective summary even when preflight filters the profile name. - const reasoningSummary = - resolveCodexProfile(config)["model_reasoning_summary"]; - if (safeString(reasoningSummary)) { - result["model_reasoning_summary"] = reasoningSummary; - } - const selectedProfile = safeProfileName(config["profile"]) - ? config["profile"] - : undefined; - if (selectedProfile !== undefined) { - result["profile"] = selectedProfile; - } - const profiles = config["profiles"]; - if (isRecord(profiles)) { - const sanitized: JsonObject = {}; - for (const [name, profile] of Object.entries(profiles)) { - if (!safeProfileName(name) || !isRecord(profile)) continue; - const projected = executionConfig(profile as JsonObject); - if (Object.keys(projected).length === 0) continue; - sanitized[name] = projected; - } - if (Object.keys(sanitized).length > 0) result["profiles"] = sanitized; - } - const modelProvider = scanModelProvider(result); - if (isExternalModelProvider(modelProvider)) { - result["model_providers"] = { - [modelProvider]: { ...EXTERNAL_CODEX_PROVIDERS[modelProvider] }, - }; - } else if (modelProvider === "amazon-bedrock") { - const providers = config["model_providers"]; - const provider = isRecord(providers) ? providers[modelProvider] : undefined; - const aws = isRecord(provider) ? provider["aws"] : undefined; - if (isRecord(aws)) { - const sanitized: JsonObject = {}; - for (const key of ["region", "profile"]) { - const value = aws[key]; - if (safeString(value)) sanitized[key] = value; - } - if (Object.keys(sanitized).length > 0) { - result["model_providers"] = { - [modelProvider]: { aws: sanitized }, - }; - } - } - } - const rootMarkers = config["project_root_markers"]; - if (Array.isArray(rootMarkers)) { - result["project_root_markers"] = rootMarkers.filter(safeString); - } - const projects = config["projects"]; - if (isRecord(projects)) { - const sanitized: JsonObject = {}; - for (const [path, project] of Object.entries(projects)) { - if (!safeString(path) || !isAbsolute(path) || !isRecord(project)) { - continue; - } - const trust = project["trust_level"]; - if (trust !== "trusted" && trust !== "untrusted") continue; - sanitized[path] = { trust_level: trust }; - } - if (Object.keys(sanitized).length > 0) result["projects"] = sanitized; - } - return result; -} - async function pluginSupportsIsolatedDeepScanConfig( pluginRoot: string, ): Promise { diff --git a/sdk/typescript/src/codex-session.ts b/sdk/typescript/src/codex-session.ts new file mode 100644 index 000000000..630df3625 --- /dev/null +++ b/sdk/typescript/src/codex-session.ts @@ -0,0 +1,94 @@ +import { + Codex, + type CodexOptions, + type ThreadOptions, + type TurnOptions, +} from "@openai/codex-sdk"; + +export interface CodexSessionEvent { + readonly type: string; + readonly [key: string]: unknown; +} + +export interface CodexSessionThread { + readonly id: string | null; + runStreamed( + input: string, + options: TurnOptions, + ): Promise<{ events: AsyncGenerator }>; +} + +export interface CodexSessionClient { + startThread(options: ThreadOptions): CodexSessionThread; + resumeThread?(threadId: string, options: ThreadOptions): CodexSessionThread; +} + +export const createCodexClient = (options: CodexOptions): CodexSessionClient => + new Codex(options); + +/** Reduce a single stream; callers retain error, retry and acceptance policy. */ +export async function readCodexSessionTurn(options: { + thread: CodexSessionThread; + events: AsyncGenerator; + onEvent: (event: CodexSessionEvent) => Promise | void; + stopOnCompletion?: boolean; +}): Promise<{ + threadId: string | null; + status: "in_progress" | "completed"; + finalResponse: string; + usage: unknown; + lastStreamError: string | null; +}> { + let threadId = options.thread.id; + let status: "in_progress" | "completed" = "in_progress"; + let finalResponse = ""; + let usage: unknown = null; + let lastStreamError: string | null = null; + for await (const event of eventsWithOptionalUsage(options.events)) { + await options.onEvent(event); + if ( + event.type === "thread.started" && + typeof event["thread_id"] === "string" + ) { + threadId = event["thread_id"]; + } else if ( + event.type === "item.completed" && + isRecord(event["item"]) && + event["item"]["type"] === "agent_message" && + typeof event["item"]["text"] === "string" + ) { + finalResponse = event["item"]["text"]; + } else if (event.type === "turn.completed") { + status = "completed"; + usage = event["usage"] ?? null; + if (options.stopOnCompletion) break; + } else if (event.type === "error" && typeof event["message"] === "string") { + lastStreamError = event["message"]; + } + } + return { threadId, status, finalResponse, usage, lastStreamError }; +} + +async function* eventsWithOptionalUsage( + events: AsyncGenerator, +): AsyncGenerator { + try { + yield* events; + } catch (error) { + // The pinned SDK accesses this field before yielding a completion with + // absent usage. Preserve completion without inventing a zero-token receipt. + if ( + error instanceof TypeError && + /\b(?:null|undefined)\b/u.test(error.message) && + /\bcache_write_input_tokens\b/u.test(error.message) + ) { + yield { type: "turn.completed", usage: null }; + return; + } + throw error; + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} diff --git a/sdk/typescript/src/cost-model.ts b/sdk/typescript/src/cost-model.ts index 3d1f22eab..26c3a9b3d 100644 --- a/sdk/typescript/src/cost-model.ts +++ b/sdk/typescript/src/cost-model.ts @@ -7,6 +7,8 @@ export interface ScanCost { outputTokens: number; /** Short-context baseline retained for compatibility and spending limits. */ estimatedUsd: number; + coverage?: "partial"; + modelCosts?: readonly ScanCost[]; /** Standard token-cost bounds for the observed usage, not a billing total. */ estimatedUsdRange?: { min: number; @@ -135,6 +137,112 @@ export function tokenUsage(value: unknown): ScanTokenUsage | null { export function estimateScanCost( model: string | undefined, usage: unknown, +): ScanCost | null { + if (isRecord(usage) && Array.isArray(usage["modelUsage"])) { + const total = tokenUsage(usage); + if (total === null || usage["modelUsage"].length === 0) return null; + const costs: ScanCost[] = []; + for (const part of usage["modelUsage"]) { + if (!isRecord(part) || typeof part["model"] !== "string") return null; + const cost = estimateModelCost(part["model"], part); + if (cost === null) return null; + costs.push(cost); + } + const sum = ( + key: + | "inputTokens" + | "cachedInputTokens" + | "cacheWriteInputTokens" + | "outputTokens" + | "estimatedUsd", + ) => costs.reduce((value, cost) => value + cost[key], 0); + if ( + sum("inputTokens") !== total.input_tokens || + sum("cachedInputTokens") !== total.cached_input_tokens || + sum("cacheWriteInputTokens") !== total.cache_write_input_tokens || + sum("outputTokens") !== total.output_tokens + ) + return null; + return { + model: model ?? costs[0]!.model, + inputTokens: total.input_tokens, + cachedInputTokens: total.cached_input_tokens, + cacheWriteInputTokens: total.cache_write_input_tokens, + ...(total.cache_write_input_tokens_reported === false + ? { cacheWriteInputTokensReported: false } + : {}), + outputTokens: total.output_tokens, + estimatedUsd: sum("estimatedUsd"), + estimatedUsdRange: { + min: sum("estimatedUsd"), + max: costs.some((cost) => cost.estimatedUsdRange?.max == null) + ? null + : costs.reduce( + (value, cost) => value + cost.estimatedUsdRange!.max!, + 0, + ), + context: "unknown", + }, + modelCosts: costs, + ...(costs.length === 1 ? { pricing: costs[0]!.pricing } : {}), + ...(usage["coverage"] === "partial" + ? { coverage: "partial" as const } + : {}), + }; + } + const cost = estimateModelCost(model, usage); + return cost && isRecord(usage) && usage["coverage"] === "partial" + ? { ...cost, coverage: "partial" } + : cost; +} + +// Internal budget enforcement only. The public estimate remains unavailable +// when some attributed usage has no price. +export function estimateScanCostLowerBound( + model: string | undefined, + usage: unknown, +): ScanCost | null { + if (!isRecord(usage) || !Array.isArray(usage["modelUsage"])) return null; + const total = tokenUsage(usage); + if (total === null) return null; + const keys = [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "reasoning_output_tokens", + ] as const; + const observed = Object.fromEntries(keys.map((key) => [key, 0])); + const priced = Object.fromEntries(keys.map((key) => [key, 0])); + const parts: Record[] = []; + let cacheWritesReported = true; + for (const part of usage["modelUsage"]) { + const normalized = tokenUsage(part); + if (!isRecord(part) || normalized === null) return null; + for (const key of keys) observed[key]! += normalized[key]; + if ( + typeof part["model"] !== "string" || + estimateModelCost(part["model"], part) === null + ) + continue; + parts.push(part); + for (const key of keys) priced[key]! += normalized[key]; + if (normalized.cache_write_input_tokens_reported === false) + cacheWritesReported = false; + } + // A malformed partition is not evidence of an enforceable lower bound. + if (keys.some((key) => observed[key] !== total[key])) return null; + return estimateScanCost(model, { + ...priced, + cache_write_input_tokens_reported: cacheWritesReported, + modelUsage: parts, + coverage: "partial", + }); +} + +function estimateModelCost( + model: string | undefined, + usage: unknown, ): ScanCost | null { if (model === undefined) return null; const pricingModel = model.startsWith("openai.") diff --git a/sdk/typescript/src/cost.ts b/sdk/typescript/src/cost.ts index 2263f136f..860e705bb 100644 --- a/sdk/typescript/src/cost.ts +++ b/sdk/typescript/src/cost.ts @@ -1,7 +1,9 @@ -import { open, readdir } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { open, readdir, readFile, realpath } from "node:fs/promises"; import { join } from "node:path"; import { estimateScanCost, + estimateScanCostLowerBound, tokenUsage, type ScanCost, type ScanTokenUsage, @@ -11,9 +13,12 @@ import { type ScanActivity, } from "./scan-activity.js"; import { + attributedScanThreads, + isAttributedScanEvent, isScanArtifactDirectory, sessionParentThreadId, sessionStartedAt, + type ScanExecutionAttribution, } from "./scan-sessions.js"; import { scanProgressUpdatesFromEvent, @@ -48,6 +53,16 @@ interface SessionUsage { inheritedUsage: ScanTokenUsage | null; replaying: boolean; usage: ScanTokenUsage | null; + counterUsage: ScanTokenUsage | null; + model: string | null; + modelUsage: Map; + currentTurnId: string | null; + previousUsage: ScanTokenUsage | null; + responseIds: Set; + responseUsageObserved: boolean; + responseTokens: number; + expectedResponseTokens: number; + counterRegressed: boolean; calls: Map; activities: ScanActivity[]; progress: ScanProgress[]; @@ -56,7 +71,8 @@ interface SessionUsage { prose: Set; reasoning: SessionReasoning | null; reasoningCount: number; - events?: Record[]; + eventIndex: number; + events?: { index: number; event: Record }[]; } interface ScanCostTrackerOptions { @@ -67,6 +83,8 @@ interface ScanCostTrackerOptions { maxCostUsd?: number; expectedFilesTotal?: number; onCost?: (cost: Readonly) => void; + // Only reported when the full public estimate is unavailable. + onCostLowerBound?: (cost: Readonly) => void; onActivity?: (activity: ScanActivity) => void; onProgress?: (progress: ScanProgress) => void; onSessionEvent?: (event: ScanSessionEvent) => void; @@ -94,6 +112,16 @@ function createSessionUsage(): SessionUsage { inheritedUsage: null, replaying: false, usage: null, + counterUsage: null, + model: null, + modelUsage: new Map(), + currentTurnId: null, + previousUsage: null, + responseIds: new Set(), + responseUsageObserved: false, + responseTokens: 0, + expectedResponseTokens: 0, + counterRegressed: false, calls: new Map(), activities: [], progress: [], @@ -102,6 +130,7 @@ function createSessionUsage(): SessionUsage { prose: new Set(), reasoning: null, reasoningCount: 0, + eventIndex: 0, }; } @@ -112,13 +141,18 @@ export class ScanCostTracker { readonly #workers = new Map(); readonly #workerProgress = new Map(); readonly #reportedProgress = new Set(); + readonly #reportedSessionEvents = new Map>(); #threadId: string | null = null; #timer: NodeJS.Timeout | null = null; #pending: Promise = Promise.resolve(); #snapshot: ScanCostSnapshot = { usage: null, cost: null }; #lastCost: string | null = null; + #lastCostLowerBound: string | null = null; #highestFilesCompleted = 0; #expectedFilesTotal: number | undefined; + #attribution: ScanExecutionAttribution | null = null; + #readAttribution: + (() => Promise) | undefined; public constructor(options: ScanCostTrackerOptions) { this.#options = options; @@ -129,10 +163,23 @@ export class ScanCostTracker { this.#expectedFilesTotal = filesTotal; } + public setAttributionReader( + reader: () => Promise, + ): void { + this.#readAttribution = reader; + } + public recordUsage(usage: unknown, threadId = this.#threadId): void { const normalized = tokenUsage(usage); if (threadId !== null) { - this.#receipts.set(threadId, normalized); + const previous = this.#receipts.get(threadId); + if ( + previous == null || + (normalized !== null && + normalized.total_tokens >= previous.total_tokens) + ) { + this.#receipts.set(threadId, normalized); + } } } @@ -142,6 +189,7 @@ export class ScanCostTracker { if ( this.#options.maxCostUsd === undefined && this.#options.onCost === undefined && + this.#options.onCostLowerBound === undefined && this.#options.onActivity === undefined && this.#options.onProgress === undefined && this.#options.onSessionEvent === undefined @@ -189,35 +237,97 @@ export class ScanCostTracker { } if (fallbackUsage !== undefined) this.recordUsage(fallbackUsage); await this.refresh(); - if (this.#receipts.size > 0 || this.#snapshot.usage !== null) + if ( + this.#readAttribution || + this.#receipts.size > 0 || + this.#snapshot.usage !== null + ) return this.#snapshot; const cost = estimateScanCost(this.#options.model, fallbackUsage); this.#snapshot = { usage: fallbackUsage ?? null, cost }; - this.#reportCost(cost); + this.#reportCost(cost, fallbackUsage); return this.#snapshot; } async #readSessions(): Promise { if (this.#threadId === null) return; + if (this.#readAttribution) { + const record = await this.#readAttribution(); + if (record === null) return; + const attribution = record === undefined || record.legacy ? null : record; + if ( + attribution && + (!this.#attribution || + attribution.completedAt !== this.#attribution.completedAt) + ) { + this.#sessions.clear(); + } + this.#attribution = attribution; + } const unreadable: Array<{ session: SessionUsage; error: unknown }> = []; - for await (const path of sessionFiles( - join(this.#options.codexHome, "sessions"), - )) { - let session = this.#sessions.get(path); - if (session === undefined) { - session = createSessionUsage(); - this.#sessions.set(path, session); + const homes = new Set([this.#options.codexHome]); + if (this.#options.scanDirectory !== undefined) { + try { + const saved: unknown = JSON.parse( + await readFile( + join( + this.#options.scanDirectory, + "artifacts", + "deep_discovery", + "execution-settings.json", + ), + "utf8", + ), + ); + if ( + isRecord(saved) && + saved["version"] === 1 && + isRecord(saved["settings"]) + ) { + const home = saved["settings"]["codexHome"]; + if (typeof home === "string" && home !== "") homes.add(home); + } + } catch (error) { + if (!isMissingFile(error)) throw error; } + } + // Recovery restores workers to their recorded home; the SDK parent can + // continue in the current home. Apply the same scan membership to both. + const directories = new Set(); + for (const home of homes) { + let directory: string; try { - await readSessionUsage(path, session, this.#options.repository); + directory = await realpath(join(home, "sessions")); } catch (error) { - if (session.threadId === null) throw error; - unreadable.push({ session, error }); + if (isMissingFile(error)) continue; + throw error; + } + if (directories.has(directory)) continue; + directories.add(directory); + for await (const path of sessionFiles(directory)) { + let session = this.#sessions.get(path); + if (session === undefined) { + session = createSessionUsage(); + this.#sessions.set(path, session); + } + try { + await readSessionUsage( + path, + session, + this.#options.repository, + this.#attribution, + ); + } catch (error) { + if (session.threadId === null) throw error; + unreadable.push({ session, error }); + } } } - const included = new Set([this.#threadId, ...this.#receipts.keys()]); - if (this.#options.scanDirectory !== undefined) { + const included = this.#attribution + ? attributedScanThreads(this.#sessions.values(), this.#attribution) + : new Set([this.#threadId, ...this.#receipts.keys()]); + if (!this.#attribution && this.#options.scanDirectory !== undefined) { const scanStartedAt = [...this.#sessions.values()].find( (session) => session.threadId === this.#threadId, @@ -243,7 +353,7 @@ export class ScanCostTracker { } } } - let changed = true; + let changed = this.#attribution === null; while (changed) { changed = false; for (const session of this.#sessions.values()) { @@ -262,7 +372,21 @@ export class ScanCostTracker { if (included.has(session.threadId!)) throw error; } - const usages = new Map(this.#receipts); + let incomplete = false; + const usages = new Map( + [...this.#receipts].filter( + ([threadId]) => + included.has(threadId) && + (!this.#attribution || + this.#attribution.executionThreadIds.includes(threadId)), + ), + ); + if (this.#attribution) { + for (const threadId of included) { + if (!usages.has(threadId)) usages.set(threadId, null); + } + } + const usageSessions = new Map(); for (const [path, tracked] of this.#sessions) { const threadId = tracked.threadId; if (threadId === null || !included.has(threadId)) continue; @@ -274,7 +398,12 @@ export class ScanCostTracker { // Replay only newly associated sessions, including their early events. session = createSessionUsage(); session.events = []; - await readSessionUsage(path, session, this.#options.repository); + await readSessionUsage( + path, + session, + this.#options.repository, + this.#attribution, + ); this.#sessions.set(path, session); } let worker: number | undefined; @@ -282,7 +411,20 @@ export class ScanCostTracker { worker = this.#workers.get(threadId) ?? this.#workers.size + 1; this.#workers.set(threadId, worker); } - for (const event of session.events?.splice(0) ?? []) { + for (const { index, event } of session.events?.splice(0) ?? []) { + let reported = this.#reportedSessionEvents.get(threadId); + if (reported === undefined) { + reported = new Set(); + this.#reportedSessionEvents.set(threadId, reported); + } + // A physical copy keeps each event's position, including repeated + // identical events. Positions count unfiltered records so attribution + // changes can replay the same log without changing occurrence identity. + const identity = `${index}:${createHash("sha256") + .update(JSON.stringify(event)) + .digest("hex")}`; + if (reported.has(identity)) continue; + reported.add(identity); this.#options.onSessionEvent?.({ threadId, parentThreadId: session.parentThreadId, @@ -300,6 +442,22 @@ export class ScanCostTracker { } this.#reportWorkerProgress(session); } + // A copied prefix must not supply model usage for a more complete log. + const previous = usageSessions.get(threadId); + if ( + previous === undefined || + (session.usage?.total_tokens ?? -1) > + (previous.usage?.total_tokens ?? -1) + ) { + usageSessions.set(threadId, session); + } + if ( + session.counterUsage && + session.counterUsage.total_tokens > + (usages.get(threadId)?.total_tokens ?? -1) + ) { + usages.set(threadId, session.counterUsage); + } const receipt = usages.get(threadId); if ( session.usage !== null && @@ -310,19 +468,74 @@ export class ScanCostTracker { ) { usages.set(threadId, session.usage); } + if (!usages.has(threadId)) usages.set(threadId, null); + if ( + (session.counterRegressed && !session.responseUsageObserved) || + session.expectedResponseTokens > session.responseTokens + ) + incomplete = true; + if (session.pendingLineBytes > 0 && !this.#receipts.get(threadId)) + incomplete = true; } let usage: ScanTokenUsage | null = null; for (const value of usages.values()) { if (value === null) { + if (this.#attribution) { + incomplete = true; + continue; + } this.#snapshot = { usage: null, cost: null }; return; } usage = addTokenUsage(usage, value); } - if (usage === null) return; - const cost = estimateScanCost(this.#options.model, usage); - this.#snapshot = { usage, cost }; - this.#reportCost(cost); + if (usage === null) { + this.#snapshot = { usage: null, cost: null }; + return; + } + const modelUsage = new Map(); + let observedModel = false; + for (const [threadId, value] of usages) { + if (value === null) continue; + const session = usageSessions.get(threadId); + for (const [model, tokens] of session?.modelUsage ?? []) { + if (model !== null) observedModel = true; + modelUsage.set( + model, + addTokenUsage(modelUsage.get(model) ?? null, tokens), + ); + } + const remainder = session?.usage + ? subtractTokenUsage(value, session.usage) + : value; + if (remainder !== null && remainder.total_tokens > 0) { + const model = + this.#attribution || (session?.modelUsage.size ?? 0) > 0 + ? null + : (session?.model ?? + (threadId === this.#threadId ? this.#options.model : null)); + modelUsage.set( + model, + addTokenUsage(modelUsage.get(model) ?? null, remainder), + ); + } + } + const reconciled = + observedModel || this.#attribution !== null + ? { + ...usage, + modelUsage: [...modelUsage].map(([model, tokens]) => ({ + model, + ...tokens, + })), + } + : usage; + const measured = incomplete + ? { ...reconciled, coverage: "partial" } + : reconciled; + const cost = estimateScanCost(this.#options.model, measured); + this.#snapshot = { usage: measured, cost }; + this.#reportCost(cost, measured); } #reportWorkerProgress(session: SessionUsage): void { @@ -365,8 +578,17 @@ export class ScanCostTracker { } } - #reportCost(cost: ScanCost | null): void { - if (cost === null) return; + #reportCost(cost: ScanCost | null, usage: unknown): void { + if (cost === null) { + if (this.#options.onCostLowerBound === undefined) return; + const lowerBound = estimateScanCostLowerBound(this.#options.model, usage); + if (lowerBound === null) return; + const signature = JSON.stringify(lowerBound); + if (signature === this.#lastCostLowerBound) return; + this.#lastCostLowerBound = signature; + this.#options.onCostLowerBound(lowerBound); + return; + } const signature = JSON.stringify(cost); if (signature === this.#lastCost) return; this.#lastCost = signature; @@ -396,6 +618,7 @@ async function readSessionUsage( path: string, session: SessionUsage, repository?: string, + attribution: ScanExecutionAttribution | null = null, ): Promise { if (session.unreadable) return; let file; @@ -417,7 +640,12 @@ async function readSessionUsage( if (bytesRead === 0) return; session.offset += bytesRead; try { - readSessionChunk(buffer.subarray(0, bytesRead), session, repository); + readSessionChunk( + buffer.subarray(0, bytesRead), + session, + repository, + attribution, + ); } catch (error) { session.unreadable = true; session.pendingLine = []; @@ -434,6 +662,7 @@ function readSessionChunk( contents: Buffer, session: SessionUsage, repository?: string, + attribution: ScanExecutionAttribution | null = null, ): void { let lineStart = 0; while (lineStart < contents.length) { @@ -451,13 +680,19 @@ function readSessionChunk( } if (session.pendingLineBytes === 0) { - readSessionEvent(fragment.toString("utf8"), session, repository); + readSessionEvent( + fragment.toString("utf8"), + session, + repository, + attribution, + ); } else { if (fragment.length > 0) session.pendingLine.push(Buffer.from(fragment)); readSessionEvent( Buffer.concat(session.pendingLine, lineBytes).toString("utf8"), session, repository, + attribution, ); session.pendingLine = []; session.pendingLineBytes = 0; @@ -470,6 +705,7 @@ function readSessionEvent( line: string, session: SessionUsage, repository?: string, + attribution: ScanExecutionAttribution | null = null, ): void { if (line.length === 0) return; let event: unknown; @@ -480,10 +716,11 @@ function readSessionEvent( } if (!isRecord(event) || !isRecord(event["payload"])) return; const payload = event["payload"]; + const index = session.eventIndex++; if (event["type"] === "session_meta") { if (session.threadId !== null) { session.replaying = payload["id"] !== session.threadId; - if (!session.replaying) session.events?.push(event); + if (!session.replaying) session.events?.push({ index, event }); return; } if (typeof payload["id"] === "string") { @@ -492,9 +729,10 @@ function readSessionEvent( if (typeof payload["cwd"] === "string") { session.workingDirectory = payload["cwd"]; } + if (typeof payload["model"] === "string") session.model = payload["model"]; session.startedAt = sessionStartedAt(payload["timestamp"]); session.parentThreadId = sessionParentThreadId(payload); - session.events?.push(event); + session.events?.push({ index, event }); return; } if (session.replaying) { @@ -515,12 +753,87 @@ function readSessionEvent( : turnOrder !== null && turnOrder >= threadOrder; if (owned) { session.replaying = false; - session.events?.push(event); + session.events?.push({ index, event }); } } return; } - session.events?.push(event); + if ( + (event["type"] === "turn_context" || payload["type"] === "task_started") && + typeof payload["turn_id"] === "string" + ) { + session.currentTurnId = payload["turn_id"]; + } + if ( + event["type"] === "turn_context" && + typeof payload["model"] === "string" + ) { + session.model = payload["model"]; + } + if (event["type"] === "token_usage_record") { + const responseId = payload["response_id"]; + const usage = tokenUsage(payload["usage"]); + if ( + typeof responseId !== "string" || + usage === null || + (typeof payload["thread_id"] === "string" && + payload["thread_id"] !== session.threadId) || + session.responseIds.has(responseId) + ) + return; + session.responseIds.add(responseId); + const cumulative = tokenUsage(payload["thread_token_usage"]); + if (cumulative) + session.expectedResponseTokens = Math.max( + session.expectedResponseTokens, + cumulative.total_tokens, + ); + if (!session.responseUsageObserved) { + // Exact receipts include compaction and survive counter resets. Keep the + // legacy counter as an independent lower bound, never add it to receipts. + session.responseUsageObserved = true; + session.usage = null; + session.modelUsage.clear(); + } + session.responseTokens += usage.total_tokens; + const turnId = + typeof payload["turn_id"] === "string" + ? payload["turn_id"] + : session.currentTurnId; + if ( + attribution && + !isAttributedScanEvent( + attribution, + session.threadId!, + turnId, + event["timestamp"], + ) + ) + return; + const model = + typeof payload["model"] === "string" ? payload["model"] : session.model; + session.usage = addTokenUsage(session.usage, usage); + session.modelUsage.set( + model, + addTokenUsage(session.modelUsage.get(model) ?? null, usage), + ); + session.events?.push({ index, event }); + return; + } + const attributable = + attribution === null || + isAttributedScanEvent( + attribution, + session.threadId!, + session.currentTurnId, + event["timestamp"], + ); + if (attributable) session.events?.push({ index, event }); + if ( + !attributable && + !(event["type"] === "event_msg" && payload["type"] === "token_count") + ) + return; if (event["type"] === "response_item") { session.progress.push(...sessionProgressUpdates(payload)); if (repository === undefined) return; @@ -658,7 +971,41 @@ function readSessionEvent( session.inheritedUsage === null ? usage : subtractTokenUsage(usage, session.inheritedUsage); - if (ownUsage !== null) session.usage = ownUsage; + if (ownUsage !== null) { + const delta = + session.previousUsage === null + ? ownUsage + : subtractTokenUsage(ownUsage, session.previousUsage); + if ( + session.previousUsage !== null && + ownUsage.total_tokens < session.previousUsage.total_tokens + ) { + session.counterRegressed = true; + return; + } + session.previousUsage = ownUsage; + if ( + attribution && + !isAttributedScanEvent( + attribution, + session.threadId!, + session.currentTurnId, + event["timestamp"], + ) + ) + return; + if (delta !== null && !session.responseUsageObserved) { + session.modelUsage.set( + session.model, + addTokenUsage(session.modelUsage.get(session.model) ?? null, delta), + ); + } + session.counterUsage = + attribution && delta !== null + ? addTokenUsage(session.counterUsage, delta) + : ownUsage; + if (!session.responseUsageObserved) session.usage = session.counterUsage; + } } function uuid7Order(value: unknown): bigint | null { diff --git a/sdk/typescript/src/deep-scan-finalization.ts b/sdk/typescript/src/deep-scan-finalization.ts new file mode 100644 index 000000000..e18bf9f58 --- /dev/null +++ b/sdk/typescript/src/deep-scan-finalization.ts @@ -0,0 +1,26 @@ +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +type WorkbenchRunner = (args: string[]) => Promise>; + +/** Load the same publisher used by native Deep from the installed plugin. */ +export async function resumeSelectedDeepScan(input: { + scanId: string; + threadId: string; + pluginRoot: string; + runWorkbench: WorkbenchRunner; + signal: AbortSignal; +}): Promise { + const helper = ( + await import(pathToFileURL(join(input.pluginRoot, "mcp/helpers.mjs")).href) + ).default as { + resumeSelectedDeepScan: (input: { + scanId: string; + threadId: string; + pluginRoot: string; + runWorkbench: WorkbenchRunner; + signal: AbortSignal; + }) => Promise; + }; + await helper.resumeSelectedDeepScan(input); +} diff --git a/sdk/typescript/src/preflight-config.ts b/sdk/typescript/src/preflight-config.ts new file mode 100644 index 000000000..5c1c59cf9 --- /dev/null +++ b/sdk/typescript/src/preflight-config.ts @@ -0,0 +1,142 @@ +import { isAbsolute } from "node:path"; +import { + EXTERNAL_CODEX_PROVIDERS, + isExternalModelProvider, + resolveCodexProfile, + scanModelProvider, + type JsonObject, +} from "./config.js"; + +export function scanPreflightCodexConfig(config: JsonObject): JsonObject { + const safeString = (value: unknown): value is string => + typeof value === "string" && + value.length > 0 && + !/[\u0000-\u001f\u007f]/u.test(value); + const safeProfileName = (value: unknown): value is string => + safeString(value) && /^[A-Za-z0-9_-]+$/u.test(value); + const safeInteger = (value: unknown): value is number => + typeof value === "number" && Number.isSafeInteger(value) && value >= 0; + const capabilityFeatures = (value: unknown): JsonObject => { + if (!isRecord(value)) return {}; + const result: JsonObject = {}; + for (const key of ["goals", "multi_agent", "enable_fanout"]) { + if (typeof value[key] === "boolean") result[key] = value[key]; + } + const multiAgent = value["multi_agent_v2"]; + if (typeof multiAgent === "boolean") { + result["multi_agent_v2"] = multiAgent; + } else if (isRecord(multiAgent)) { + const sanitized: JsonObject = {}; + if (typeof multiAgent["enabled"] === "boolean") { + sanitized["enabled"] = multiAgent["enabled"]; + } + const capacity = multiAgent["max_concurrent_threads_per_session"]; + if (safeInteger(capacity)) { + sanitized["max_concurrent_threads_per_session"] = capacity; + } + if (Object.keys(sanitized).length > 0) { + result["multi_agent_v2"] = sanitized; + } + } + return result; + }; + const executionConfig = (source: JsonObject): JsonObject => { + const result: JsonObject = {}; + for (const key of [ + "model", + "model_reasoning_effort", + "model_reasoning_summary", + "model_provider", + "service_tier", + ]) { + const value = source[key]; + if (safeString(value)) result[key] = value; + } + const features = capabilityFeatures(source["features"]); + if (Object.keys(features).length > 0) result["features"] = features; + const agents = source["agents"]; + if (isRecord(agents)) { + const sanitized: JsonObject = {}; + for (const key of ["max_threads", "max_depth"]) { + const value = agents[key]; + if (safeInteger(value)) sanitized[key] = value; + } + if (Object.keys(sanitized).length > 0) result["agents"] = sanitized; + } + const multiagent = source["multiagent_config"]; + if (isRecord(multiagent) && safeInteger(multiagent["max_concurrency"])) { + result["multiagent_config"] = { + max_concurrency: multiagent["max_concurrency"], + }; + } + return result; + }; + const result = executionConfig(config); + // Keep the effective summary even when preflight filters the profile name. + const reasoningSummary = + resolveCodexProfile(config)["model_reasoning_summary"]; + if (safeString(reasoningSummary)) { + result["model_reasoning_summary"] = reasoningSummary; + } + const selectedProfile = safeProfileName(config["profile"]) + ? config["profile"] + : undefined; + if (selectedProfile !== undefined) { + result["profile"] = selectedProfile; + } + const profiles = config["profiles"]; + if (isRecord(profiles)) { + const sanitized: JsonObject = {}; + for (const [name, profile] of Object.entries(profiles)) { + if (!safeProfileName(name) || !isRecord(profile)) continue; + const projected = executionConfig(profile as JsonObject); + if (Object.keys(projected).length === 0) continue; + sanitized[name] = projected; + } + if (Object.keys(sanitized).length > 0) result["profiles"] = sanitized; + } + const modelProvider = scanModelProvider(result); + if (isExternalModelProvider(modelProvider)) { + result["model_providers"] = { + [modelProvider]: { ...EXTERNAL_CODEX_PROVIDERS[modelProvider] }, + }; + } else if (modelProvider === "amazon-bedrock") { + const providers = config["model_providers"]; + const provider = isRecord(providers) ? providers[modelProvider] : undefined; + const aws = isRecord(provider) ? provider["aws"] : undefined; + if (isRecord(aws)) { + const sanitized: JsonObject = {}; + for (const key of ["region", "profile"]) { + const value = aws[key]; + if (safeString(value)) sanitized[key] = value; + } + if (Object.keys(sanitized).length > 0) { + result["model_providers"] = { + [modelProvider]: { aws: sanitized }, + }; + } + } + } + const rootMarkers = config["project_root_markers"]; + if (Array.isArray(rootMarkers)) { + result["project_root_markers"] = rootMarkers.filter(safeString); + } + const projects = config["projects"]; + if (isRecord(projects)) { + const sanitized: JsonObject = {}; + for (const [path, project] of Object.entries(projects)) { + if (!safeString(path) || !isAbsolute(path) || !isRecord(project)) { + continue; + } + const trust = project["trust_level"]; + if (trust !== "trusted" && trust !== "untrusted") continue; + sanitized[path] = { trust_level: trust }; + } + if (Object.keys(sanitized).length > 0) result["projects"] = sanitized; + } + return result; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} diff --git a/sdk/typescript/src/reasoning-summary.ts b/sdk/typescript/src/reasoning-summary.ts new file mode 100644 index 000000000..a313bc00c --- /dev/null +++ b/sdk/typescript/src/reasoning-summary.ts @@ -0,0 +1,69 @@ +import { + inlineToml, + resolveCodexProfile, + scanModelConfiguration, + type JsonObject, +} from "./config.js"; +import { scanPreflightCodexConfig } from "./preflight-config.js"; +import { + runCodexCommand, + type CodexCommand, + type ProcessEnvironment, +} from "./runtime.js"; + +/** Capture a future owner's selected model default; never infer historical settings. */ +export async function captureOriginalReasoningSummary(options: { + config: JsonObject; + command: CodexCommand; + cwd: string; + environment: ProcessEnvironment; + signal: AbortSignal; +}): Promise { + // Keep explicit selections and invalid values with their existing native validator. + if ( + scanPreflightCodexConfig(options.config)["model_reasoning_summary"] !== + undefined || + resolveCodexProfile(options.config)["model_reasoning_summary"] !== undefined + ) + return undefined; + const { model } = scanModelConfiguration(options.config); + // The same per-session overrides protect the lookup from concurrent home edits. + const config = JSON.parse(JSON.stringify(options.config)) as JsonObject; + const args = [ + "debug", + "models", + ...Object.entries(config).flatMap(([key, value]) => [ + "--config", + `${key}=${inlineToml(value)}`, + ]), + ]; + const result = await runCodexCommand( + options.command, + args, + options.environment, + undefined, + options.signal, + options.cwd, + ); + // Older native executables and models absent from their catalog provide no + // recoverable value. Preserve omission instead of choosing another default. + if (!result.success) return undefined; + let catalog: unknown; + try { + catalog = JSON.parse(result.stdout); + } catch { + return undefined; + } + if (!isRecord(catalog) || !Array.isArray(catalog["models"])) return undefined; + const selected = catalog["models"].find( + (entry: unknown) => isRecord(entry) && entry["slug"] === model, + ); + return isRecord(selected) && + typeof selected["default_reasoning_summary"] === "string" + ? selected["default_reasoning_summary"] + : undefined; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/sdk/typescript/src/runtime.ts b/sdk/typescript/src/runtime.ts index 8d2d2a442..f85477f8a 100644 --- a/sdk/typescript/src/runtime.ts +++ b/sdk/typescript/src/runtime.ts @@ -2702,12 +2702,14 @@ export async function runCodexCommand( environment: ProcessEnvironment, input?: string | Uint8Array, signal?: AbortSignal, + cwd?: string, ): Promise { const child = spawn(executablePathForSpawn(command.command), [...args], { env: environment, stdio: ["pipe", "pipe", "pipe"], windowsHide: true, signal, + ...(cwd === undefined ? {} : { cwd }), }); let stdout = ""; let stderr = ""; diff --git a/sdk/typescript/src/scan-logs.ts b/sdk/typescript/src/scan-logs.ts index 851a8f2da..dbe33be22 100644 --- a/sdk/typescript/src/scan-logs.ts +++ b/sdk/typescript/src/scan-logs.ts @@ -5,9 +5,12 @@ import { sessionFiles } from "./cost.js"; import { CodexSecurityError } from "./errors.js"; import type { JsonObject } from "./config.js"; import { + attributedScanThreads, + isAttributedScanEvent, isScanArtifactDirectory, sessionParentThreadId, sessionStartedAt, + type ScanExecutionAttribution, } from "./scan-sessions.js"; interface ScanLogOptions { @@ -19,6 +22,7 @@ interface ScanLogOptions { scanDirectory?: string; completedAt?: string | null; allowMissingRoot?: boolean; + executionAttribution?: ScanExecutionAttribution | null; } export type ScanLogSource = JsonObject & { @@ -26,6 +30,7 @@ export type ScanLogSource = JsonObject & { continuationThreadId?: string; threadIds?: string[]; executionThreadIds?: string[]; + executionAttribution?: ScanExecutionAttribution | null; mode?: string; scanDir?: string; progress?: { status?: string; updatedAt?: string }; @@ -47,6 +52,7 @@ export function readSavedScanLogs( threadId: threadId ?? scan.threadIds?.[0], threadIds: scan.threadIds, executionThreadIds: scan.executionThreadIds ?? [], + executionAttribution: scan.executionAttribution, codexHome, allowMissingRoot: options.allowMissingRoot, scanDirectory: scan.mode === "deep" ? scan.scanDir : undefined, @@ -126,15 +132,20 @@ export async function readScanLogs(options: ScanLogOptions) { ); } - const included = new Set([ - ...(options.threadId ? [options.threadId] : []), - ...(options.threadIds ?? []), - ...(options.executionThreadIds ?? []), - ]); + const attribution = options.executionAttribution?.legacy + ? null + : options.executionAttribution; + const included = attribution + ? attributedScanThreads(logs.values(), attribution) + : new Set([ + ...(options.threadId ? [options.threadId] : []), + ...(options.threadIds ?? []), + ...(options.executionThreadIds ?? []), + ]); // A Desktop owner can contain other work. Include its log without treating // the whole conversation tree as part of this scan. const traversed = new Set(options.executionThreadIds ?? included); - const pending = [...traversed]; + const pending = attribution ? [] : [...traversed]; for (const parentId of pending) { const parent = logs.get(parentId); for (const session of logs.values()) { @@ -160,8 +171,17 @@ export async function readScanLogs(options: ScanLogOptions) { const events: Record[] = []; for (const session of sessions) { let replaying = false; + let turnId: string | null = null; for await (const event of sessionEvents(session.path)) { const payload = event["payload"]; + if ( + isRecord(payload) && + (event["type"] === "turn_context" || + payload["type"] === "task_started") && + typeof payload["turn_id"] === "string" + ) { + turnId = payload["turn_id"]; + } if (event["type"] === "session_meta" && isRecord(payload)) { replaying = payload["id"] !== session.threadId; } @@ -178,7 +198,22 @@ export async function readScanLogs(options: ScanLogOptions) { } replaying = false; } - events.push({ threadId: session.threadId, event }); + if ( + !attribution || + event["type"] === "session_meta" || + isAttributedScanEvent( + attribution, + session.threadId, + event["type"] === "token_usage_record" && + isRecord(payload) && + typeof payload["turn_id"] === "string" + ? payload["turn_id"] + : turnId, + event["timestamp"], + ) + ) { + events.push({ threadId: session.threadId, event }); + } } } diff --git a/sdk/typescript/src/scan-sessions.ts b/sdk/typescript/src/scan-sessions.ts index 676f7a10b..47b4c9091 100644 --- a/sdk/typescript/src/scan-sessions.ts +++ b/sdk/typescript/src/scan-sessions.ts @@ -1,5 +1,64 @@ import { isAbsolute, join, relative, sep } from "node:path"; +export interface ScanExecutionAttribution { + formatVersion: 1; + legacy?: true; + executionThreadIds: string[]; + owner: { threadId: string | null; turnId: string | null; startedAt: string }; + startedAt: string; + completedAt: string | null; +} + +export function attributedScanThreads( + sessions: Iterable<{ + threadId: string | null; + parentThreadId: string | null; + }>, + attribution: ScanExecutionAttribution, +): Set { + const included = new Set(attribution.executionThreadIds); + const pending = [...included]; + const all = [...sessions]; + for (const parent of pending) { + for (const session of all) { + if ( + session.threadId !== null && + session.parentThreadId === parent && + !included.has(session.threadId) + ) { + included.add(session.threadId); + pending.push(session.threadId); + } + } + } + if (attribution.owner.threadId) included.add(attribution.owner.threadId); + return included; +} + +export function isAttributedScanEvent( + attribution: ScanExecutionAttribution, + threadId: string, + turnId: string | null, + timestamp: unknown, +): boolean { + const time = sessionStartedAt(timestamp); + if ( + time === null || + time < Date.parse(attribution.startedAt) || + (attribution.completedAt !== null && + time > Date.parse(attribution.completedAt)) + ) + return false; + if ( + threadId !== attribution.owner.threadId || + attribution.executionThreadIds.includes(threadId) + ) + return true; + return ( + attribution.owner.turnId !== null && turnId === attribution.owner.turnId + ); +} + export function sessionStartedAt(timestamp: unknown): number | null { const startedAt = typeof timestamp === "string" ? Date.parse(timestamp) : Number.NaN; diff --git a/sdk/typescript/tests-ts/api-audit-admission.test.ts b/sdk/typescript/tests-ts/api-audit-admission.test.ts new file mode 100644 index 000000000..0bfc8c269 --- /dev/null +++ b/sdk/typescript/tests-ts/api-audit-admission.test.ts @@ -0,0 +1,395 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { afterEach, expect, test } from "bun:test"; +import { build } from "esbuild"; +import { runScanEvents } from "../src/api.js"; +import type { ScanDraftInput } from "../src/accepted-audit.js"; +import { PLUGIN_ROOT } from "./plugin-root.js"; +import { + completedEvents, + createApiTestFixtures, +} from "./support/api-events.js"; + +const { temporaryDirectory, copyCompletedScan, cleanup } = + createApiTestFixtures(); +afterEach(cleanup); + +const bundle = await build({ + stdin: { + resolveDir: fileURLToPath( + new URL("../../../plugins/codex-security/mcp-app/", import.meta.url), + ), + contents: `export * from "./src/artifact-scan-draft.ts"; + export * from "./src/deep-scan/artifacts.ts"; + export * from "./src/deep-scan/artifact-validation.ts"; + export * from "./src/deep-scan/worker-runner.ts";`, + }, + bundle: true, + format: "esm", + platform: "node", + loader: { ".md": "text" }, + write: false, +}); +const bundlePath = join(await temporaryDirectory(), "deep-admission.mjs"); +await writeFile(bundlePath, bundle.outputFiles[0]!.contents); +const { + createDeepScanArtifacts, + recordCodexSecurityScanDraft, + readDiscoveryAuditDraft, + DeepScanWorkerRunner, +} = await import(pathToFileURL(bundlePath).href); + +const scanId = "811aef98-3709-4c2d-8b7a-742977521865"; +type Mutation = + | "missing-findings" + | "contradictory-coverage" + | "inverted-lines" + | "wrong-scan" + | "legacy-details"; +const cases: { + name: string; + coverage: "complete" | "partial" | "unknown"; + complete?: boolean; + mutation?: Mutation; + accepted: boolean; +}[] = [ + { + name: "complete coverage", + coverage: "complete", + complete: true, + accepted: true, + }, + { + name: "partial coverage", + coverage: "partial", + complete: true, + accepted: true, + }, + { + name: "unknown coverage", + coverage: "unknown", + complete: true, + accepted: true, + }, + { name: "omitted completion marker", coverage: "partial", accepted: true }, + { + name: "persisted legacy details", + coverage: "partial", + mutation: "legacy-details", + accepted: true, + }, + { + name: "unfinished checkpoint", + coverage: "partial", + complete: false, + accepted: false, + }, + { + name: "missing findings", + coverage: "partial", + mutation: "missing-findings", + accepted: false, + }, + { + name: "complete coverage with deferred work", + coverage: "partial", + mutation: "contradictory-coverage", + accepted: false, + }, + { + name: "inverted finding lines", + coverage: "partial", + mutation: "inverted-lines", + accepted: false, + }, + { + name: "mismatched canonical scan ID", + coverage: "partial", + mutation: "wrong-scan", + accepted: false, + }, +]; + +for (const scenario of cases) { + test(`Standard and Deep production admission: ${scenario.name}`, async () => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const standardRoot = join(root, "standard"); + const deepRoot = join(root, "deep"); + await Promise.all([ + mkdir(repository), + mkdir(standardRoot, { mode: 0o700 }), + mkdir(deepRoot, { mode: 0o700 }), + ]); + const semantic: ScanDraftInput = { + scanId, + ...(scenario.complete === undefined + ? {} + : { complete: scenario.complete }), + scope: { summary: "Archive extraction." }, + threatModel: { summary: "An untrusted caller supplies archive entries." }, + findings: [ + { + ruleId: "path-traversal.archive", + title: "Unsafe archive extraction", + summary: "An archive entry reaches a filesystem write.", + severity: { level: "high" }, + confidence: { level: "high", rationale: "Source review." }, + taxonomy: { category: "path-traversal", cwe: ["CWE-22"] }, + locations: [{ path: "extract.py", startLine: 4, endLine: 7 }], + remediation: "Validate the resolved output path before writing.", + provenance: { source: "local_plugin", candidateId: "archive-entry" }, + }, + ], + coverage: { + completeness: scenario.coverage, + surfaces: [{ label: "Archive extraction", disposition: "reported" }], + explicitExclusions: [], + deferred: + scenario.coverage === "complete" + ? [] + : [ + { + id: "deployment", + reason: "Deployment controls remain unverified.", + }, + ], + }, + }; + await recordCodexSecurityScanDraft( + { + root: standardRoot, + repoRoot: repository, + layout: "scan", + scanId, + mode: "standard", + status: "running", + scope: ".", + targetContract: { + target: { + allowedKinds: ["directory_snapshot"], + targetId: "target_example", + displayName: "example", + requiredSnapshotDigest: `codex-security-snapshot/v1:sha256:${"a".repeat(64)}`, + }, + scope: { requiredIncludePaths: ["."], requiredExcludePaths: [] }, + diffTarget: null, + }, + }, + semantic, + ); + const submitted = mutateDraft(semantic, scenario.mutation); + const findings = { + scanId: submitted.scanId, + findings: submitted.findings?.map((finding) => ({ + ...finding, + findingId: "finding_example", + occurrenceId: "occurrence_example", + fingerprints: { identity: "synthetic" }, + })), + }; + const coverage = JSON.parse( + await readFile(join(standardRoot, "coverage.json"), "utf8"), + ); + Object.assign(coverage, submitted.coverage); + await Promise.all([ + writeFile(join(standardRoot, "findings.json"), JSON.stringify(findings)), + writeFile(join(standardRoot, "coverage.json"), JSON.stringify(coverage)), + ]); + const standard = await observeStandardAdmission( + root, + repository, + standardRoot, + scanId, + ); + + const artifacts = createDeepScanArtifacts(deepRoot); + const acceptedPaths: string[] = []; + let executions = 0; + const runner = new DeepScanWorkerRunner({ + run: { + scanId, + scanDir: deepRoot, + targetPath: repository, + scope: ".", + config: { subagents: 0 }, + }, + artifacts, + pluginRoot: PLUGIN_ROOT, + signal: new AbortController().signal, + retryDelaysMs: [], + random: () => 0, + log: () => {}, + clock: { now: () => Date.now(), sleep: async () => {} }, + executor: { + async run(request: { + artifactContext: { root: string }; + onThreadStarted?: (id: string) => Promise; + }) { + executions++; + await request.onThreadStarted?.("deep-thread"); + await writeFile( + join(request.artifactContext.root, "result.json"), + JSON.stringify(submitted), + ); + return { threadId: "deep-thread" }; + }, + }, + store: { + async updateWorker(update: { + status: string; + resultManifestPath?: string; + }) { + if (update.status === "succeeded") { + acceptedPaths.push(update.resultManifestPath!); + return { ...update, completionSequence: 1 }; + } + return update; + }, + }, + }); + const deepResult = await runner.runDiscoveryWorker( + "worker-1", + "discovery-1", + ); + expect(executions).toBe(1); + if (scenario.accepted) { + expect(standard.error).toBe(standard.finalization); + expect(standard.finalizations).toBe(1); + expect(standard.drafts).toHaveLength(1); + expect(deepResult.status).toBe("succeeded"); + expect(acceptedPaths).toEqual([deepResult.worker.resultPath]); + const deepDraft: ScanDraftInput = await readDiscoveryAuditDraft( + artifacts, + deepResult.worker.resultPath, + scanId, + ); + expect(standard.drafts[0]!.findings).toEqual(deepDraft.findings); + expect(standard.drafts[0]!.coverage).toEqual(deepDraft.coverage); + expect(standard.drafts[0]!.scope).toEqual(deepDraft.scope); + expect(standard.drafts[0]!.threatModel).toEqual(deepDraft.threatModel); + if (scenario.mutation === "legacy-details") { + expect(deepDraft.findings[0]!["validation"]).toEqual({ + limitations: ["Legacy persisted limitation."], + }); + } + } else { + expect(standard.error).toBeInstanceOf(Error); + expect(standard.error).not.toBe(standard.finalization); + expect(standard.finalizations).toBe(0); + expect(deepResult.status).toBe("failed"); + expect(acceptedPaths).toHaveLength(0); + } + expect(standard.calls).toBe(1); + const manifest = JSON.parse( + await readFile(join(standardRoot, "scan-manifest.json"), "utf8"), + ); + expect(manifest.scan.sealedAt).toBeUndefined(); + await expect(readFile(join(standardRoot, "report.md"))).rejects.toThrow(); + }); +} + +test("Standard admission preserves existing canonical scan IDs", async () => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + await mkdir(repository); + const scanDir = await copyCompletedScan(root); + const manifest = JSON.parse( + await readFile(join(scanDir, "scan-manifest.json"), "utf8"), + ); + const standard = await observeStandardAdmission(root, repository, scanDir); + expect(manifest.scan.id).toBe("scan_example_001"); + expect(standard.error).toBe(standard.finalization); + expect(standard.finalizations).toBe(1); + expect(standard.drafts).toHaveLength(1); + expect(standard.calls).toBe(1); + expect(standard.drafts[0]!.scanId).toBe(manifest.scan.id); +}); + +async function observeStandardAdmission( + root: string, + repository: string, + scanDir: string, + scanId?: string, +) { + const pluginRoot = join(root, "observed-plugin"); + const helperPath = join(pluginRoot, "mcp", "helpers.mjs"); + await mkdir(join(pluginRoot, "mcp"), { recursive: true }); + // The bundled exports are immutable getters. Observe the real parser through + // a local delegator instead of mocking its module or copying its semantics. + await writeFile( + helperPath, + `import helpers from ${JSON.stringify(pathToFileURL(join(PLUGIN_ROOT, "mcp", "helpers.mjs")).href)}; +export const drafts = []; +export let calls = 0; +export default { ...helpers, parseCanonicalScanDraft(input) { + calls++; + const draft = helpers.parseCanonicalScanDraft(input); + drafts.push(draft); + return draft; +} };`, + ); + const observed: { drafts: ScanDraftInput[]; calls: number } = await import( + pathToFileURL(helperPath).href + ); + const finalization = new Error("The enclosing finalizer owns the next step."); + let finalizations = 0; + const error = await runScanEvents({ + scanId, + thread: { + id: "standard-thread", + async runStreamed() { + return { events: completedEvents("standard-thread") }; + }, + }, + events: completedEvents("standard-thread"), + signal: new AbortController().signal, + scanDir, + pluginRoot, + expectation: { + repository, + repositoryRevision: null, + target: { kind: "repository", paths: [] }, + mode: "standard", + pluginVersion: "0.1.0", + }, + onFinalize: async () => { + finalizations++; + throw finalization; + }, + }).catch((error: unknown) => error); + return { + error, + finalization, + finalizations, + drafts: observed.drafts, + calls: observed.calls, + }; +} + +function mutateDraft( + input: ScanDraftInput, + mutation?: Mutation, +): ScanDraftInput { + const draft = structuredClone(input); + if (mutation === "missing-findings") + return { ...draft, findings: undefined } as unknown as ScanDraftInput; + if (mutation === "wrong-scan") + draft.scanId = "553a0c18-dcdf-4a3b-8e39-2751a8187bce"; + if (mutation === "contradictory-coverage") + draft.coverage["completeness"] = "complete"; + if (mutation === "inverted-lines") { + const locations = draft.findings[0]!["locations"] as Record< + string, + unknown + >[]; + locations[0]!["endLine"] = 1; + } + if (mutation === "legacy-details") + draft.findings[0]!["validation"] = { + method: null, + limitations: "Legacy persisted limitation.", + }; + return draft; +} diff --git a/sdk/typescript/tests-ts/api-events.test.ts b/sdk/typescript/tests-ts/api-events.test.ts index 28d9b1212..2f7d79c73 100644 --- a/sdk/typescript/tests-ts/api-events.test.ts +++ b/sdk/typescript/tests-ts/api-events.test.ts @@ -1,4 +1,4 @@ -import { mkdir, stat } from "node:fs/promises"; +import { mkdir, rm, stat } from "node:fs/promises"; import { existsSync } from "node:fs"; import { join } from "node:path"; import { @@ -477,7 +477,8 @@ describe("one-shot scan events", () => { test("lets the workbench seal artifacts before validating completed scans", async () => { const root = await temporaryDirectory(); - const scanDir = join(root, "scan"); + const scanDir = await copyCompletedScan(root); + await rm(join(scanDir, "report.md")); const events = completedEvents(); let finalized = false; @@ -506,7 +507,8 @@ describe("one-shot scan events", () => { cache_write_input_tokens: 0, output_tokens: 3, }); - expect(existsSync(join(scanDir, "scan-manifest.json"))).toBe(false); + expect(existsSync(join(scanDir, "scan-manifest.json"))).toBe(true); + expect(existsSync(join(scanDir, "report.md"))).toBe(false); await copyCompletedScan(root); finalized = true; }, diff --git a/sdk/typescript/tests-ts/api-policy.test.ts b/sdk/typescript/tests-ts/api-policy.test.ts index f69bc8fdf..c3b1fac36 100644 --- a/sdk/typescript/tests-ts/api-policy.test.ts +++ b/sdk/typescript/tests-ts/api-policy.test.ts @@ -1538,6 +1538,93 @@ describe("CodexSecurity policy API", () => { await f.security.close(); }); + test.each(["architecture", "threat_model"] as const)( + "enforces priced policy usage despite an unpriced remainder in %s", + async (crossingStage) => { + const costs: number[] = []; + const f = await setup({ + config: { codexOverrides: { model: "gpt-5.6-sol" } }, + stream: async function* (stage, signal) { + if (stage !== crossingStage) { + yield* events(stage); + return; + } + const directory = join(f.root, "codex-home", "sessions"); + await mkdir(directory, { recursive: true }); + const thread = `policy-${stage}`; + const input = stage === "architecture" ? 1_200 : 1_000; + for (const [id, model, parent, inputTokens] of [ + [thread, "gpt-5.6-sol", undefined, input], + ["unpriced-policy-worker", "synthetic-unpriced-model", thread, 100], + ] as const) { + await writeFile( + join(directory, `${id}.jsonl`), + [ + JSON.stringify({ + type: "session_meta", + payload: { + id, + ...(parent === undefined + ? {} + : { parent_thread_id: parent }), + }, + }), + JSON.stringify({ type: "turn_context", payload: { model } }), + JSON.stringify({ + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: inputTokens, + output_tokens: 0, + }, + }, + }, + }), + "", + ].join("\n"), + ); + } + yield { type: "thread.started", thread_id: thread }; + await new Promise((resolve) => { + if (signal.aborted) resolve(); + else + signal.addEventListener("abort", () => resolve(), { once: true }); + }); + throw signal.reason; + }, + }); + const keepAlive = setTimeout(() => {}, 10_000); + try { + await expect( + f.security.generatePolicy(f.repository, { + outputDir: f.outputDir, + maxCostUsd: 0.0045, + signal: AbortSignal.timeout(5_000), + onCost: (cost) => costs.push(cost.estimatedUsd), + }), + ).rejects.toThrow("exceeded its $0.0045 cost limit"); + expect(f.threads).toHaveLength( + crossingStage === "architecture" ? 1 : 2, + ); + if (crossingStage === "architecture") expect(costs).toEqual([]); + else { + expect(costs.length).toBeGreaterThan(0); + for (const cost of costs) expect(cost).toBeCloseTo(0.0006, 12); + } + if (crossingStage === "threat_model") { + expect( + await readFile(join(f.outputDir, "project-spec.md"), "utf8"), + ).toContain("src/service.ts:1"); + } + } finally { + clearTimeout(keepAlive); + await f.security.close(); + } + }, + ); + test("enforces one cost budget across stages and preserves completed evidence", async () => { const f = await setup(); await expect( diff --git a/sdk/typescript/tests-ts/api.test.ts b/sdk/typescript/tests-ts/api.test.ts index f69ba7977..35f04ff98 100644 --- a/sdk/typescript/tests-ts/api.test.ts +++ b/sdk/typescript/tests-ts/api.test.ts @@ -4767,6 +4767,183 @@ describe("CodexSecurity orchestration", () => { }, ); + test.each([ + ["standard", false], + ["deep", false], + ["standard", true], + ["deep", true], + ] as const)( + "enforces priced usage with an unpriced remainder (%s, raised limit: %s)", + async (mode, raised) => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const codexHome = join(root, "codex-home"); + const scanDir = join(root, "scan"); + await Promise.all([ + mkdir(repository), + mkdir(codexHome), + mkdir(scanDir, { mode: 0o700 }), + ]); + const commands: Array = []; + const costs: number[] = []; + let turns = 0; + let releaseIncrease!: () => void; + const increased = new Promise((resolve) => { + releaseIncrease = resolve; + }); + const knownUsage = { + input_tokens: raised ? 2_500 : 1_250, + cached_input_tokens: 200, + output_tokens: 30, + }; + const expectedCost = estimateScanCost("gpt-5.6-sol", knownUsage)!; + const writePricedUsage = async (usage: Record) => { + const path = await writeUsageSession(codexHome, "scan-thread", usage); + const lines = (await readFile(path, "utf8")).split("\n"); + lines.splice( + 1, + 0, + JSON.stringify({ + type: "turn_context", + payload: { model: "gpt-5.6-sol" }, + }), + ); + await writeFile(path, lines.join("\n")); + }; + const writeUnpricedUsage = async () => { + const path = await writeUsageSession( + codexHome, + "unpriced-worker", + { input_tokens: 100, output_tokens: 10 }, + "scan-thread", + ); + const lines = (await readFile(path, "utf8")).split("\n"); + lines.splice( + 1, + 0, + JSON.stringify({ + type: "turn_context", + payload: { model: "synthetic-unpriced-model" }, + }), + ); + await writeFile(path, lines.join("\n")); + }; + const client = new TestClient( + {}, + { + environment: {}, + prepareRuntime: async () => preparedRuntime(codexHome), + resolvePluginPython: async () => "/managed/python", + prepareOutputDir: async () => scanDir, + repositoryRevision: async () => "deadbeef", + runWorkbench: async (_options, args, input) => { + commands.push(args); + if (args[0] === "get-scan") + return { scan: { id: "scan_example_001" } }; + if (args[0] === "complete-budget-exhausted-scan") + throw new Error("Synthetic canonical output is not ready"); + return mockWorkbench(args, input); + }, + createCodex: () => ({ + startThread: () => ({ + id: null, + async runStreamed( + _input: string, + options: { signal: AbortSignal }, + ) { + turns++; + async function* events(): AsyncGenerator { + if (raised) { + await writePricedUsage({ + input_tokens: 800, + output_tokens: 0, + }); + } else { + await writePricedUsage(knownUsage); + await writeUnpricedUsage(); + } + yield { type: "thread.started", thread_id: "scan-thread" }; + if (raised) { + await increased; + await writeUnpricedUsage(); + const path = join( + codexHome, + "sessions", + "2026", + "07", + "26", + "rollout-scan-thread.jsonl", + ); + await appendFile( + path, + JSON.stringify({ + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: knownUsage }, + }, + }) + "\n", + ); + } + await new Promise((resolve) => { + if (options.signal.aborted) resolve(); + else + options.signal.addEventListener( + "abort", + () => resolve(), + { once: true }, + ); + }); + throw new DOMException("aborted", "AbortError"); + } + return { events: events() }; + }, + }), + }), + }, + ); + const keepAlive = setTimeout(() => {}, 10_000); + try { + const failure = await client + .run(repository, { + mode, + maxCostUsd: 0.004, + signal: AbortSignal.timeout(5_000), + postScanPrompt: "No model work after the budget stop.", + ...(raised ? { onBudgetApproaching: () => 0.008 } : {}), + onCost: (cost, limit) => { + costs.push(cost.estimatedUsd); + if (limit === 0.008) releaseIncrease(); + }, + }) + .catch((error: unknown) => error); + expect(failure).toBeInstanceOf(ScanCostLimitExceededError); + expect(failure).toMatchObject({ + maxCostUsd: raised ? 0.008 : 0.004, + cost: { + estimatedUsd: expectedCost.estimatedUsd, + inputTokens: knownUsage.input_tokens, + coverage: "partial", + }, + }); + expect(costs.every((cost) => raised && cost === 0.0032)).toBe(true); + expect(turns).toBe(1); + expect( + commands.some((args) => args[0] === "complete-budget-exhausted-scan"), + ).toBe(mode === "deep"); + if (raised) + expect( + commands + .filter((args) => args[0] === "set-scan-cost-limit") + .map((args) => args.at(-1)), + ).toEqual(["0.008"]); + } finally { + clearTimeout(keepAlive); + await client.close(); + } + }, + ); + test("stops and records a scan as soon as its live cost exceeds the limit", async () => { const root = await temporaryDirectory(); const repository = join(root, "repository"); @@ -4930,6 +5107,10 @@ describe("CodexSecurity orchestration", () => { input?: string, ): Promise => { commands.push(args); + if (args[0] === "get-scan") { + // Older workbench readers return a scan without execution attribution. + return { scan: { id: "scan_example_001" } }; + } if (args[0] !== "complete-budget-exhausted-scan") { return mockWorkbench(args, input); } @@ -7298,6 +7479,211 @@ if ([basename(process.argv[1]), ...process.argv.slice(2)].join(" ") !== "login s expect(scanSignal?.aborted).toBe(false); }); + test.each([ + ["standard", false], + ["deep", false], + ["deep", true], + ] as const)( + "isolates concurrent managed %s sessions at the Codex child boundary (capture=%s)", + async (mode, captureSummary) => { + const clients: TestClient[] = []; + try { + const outcomes = await Promise.allSettled( + ["first", "second"].map(async (name) => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const codexHome = join(root, "codex-home"); + const scanDir = join(root, "scan"); + const preload = join(root, "fake-codex.mjs"); + const marker = join(root, "invocation.jsonl"); + await Promise.all([ + mkdir(repository), + mkdir(codexHome), + mkdir(scanDir, { mode: 0o700 }), + ]); + const model = `fixture-${name}-model`; + const summary = name === "first" ? "none" : "concise"; + const configPath = join(codexHome, "config-preflight.toml"); + let recipeConfig: JsonObject | undefined; + await writeFile( + preload, + [ + 'import { appendFileSync } from "node:fs";', + 'let prompt = ""; for await (const chunk of process.stdin) prompt += chunk;', + `appendFileSync(${JSON.stringify(marker)}, JSON.stringify({args:process.argv, executable:process.execPath, cwd:process.cwd(), home:process.env.CODEX_HOME, key:process.env.CODEX_API_KEY, value:process.env.FIXTURE_SCAN_VALUE, prompt}) + "\\n");`, + `if (process.argv.includes("models")) { console.log(JSON.stringify({models:[{slug:${JSON.stringify(model)},default_reasoning_summary:${JSON.stringify(summary)}}]})); process.exit(0); }`, + `console.log(JSON.stringify({type:"thread.started",thread_id:${JSON.stringify(`fixture-${name}-thread`)}}));`, + 'console.log(JSON.stringify({type:"item.completed",item:{type:"agent_message",text:"scan complete"}}));', + 'console.log(JSON.stringify({type:"turn.completed",usage:null}));', + "process.exit(0);", + ].join("\n"), + ); + const fake = nodeCodex(preload); + const provider = `fixture-${name}-provider`; + const client = new TestClient( + { + codexOverrides: { + model, + model_provider: provider, + model_reasoning_effort: "ultra", + // JavaScript callers can omit the merged default with undefined. + model_reasoning_summary: captureSummary + ? (undefined as unknown as string) + : summary, + service_tier: name === "first" ? "flex" : "fast", + features: { + multi_agent_v2: { max_concurrent_threads_per_session: 4 }, + }, + }, + }, + { + environment: { + OPENAI_API_KEY: `synthetic-${name}-key`, + CODEX_CLI_PATH: fake.command.command, + }, + prepareRuntime: async () => ({ + ...preparedRuntime(codexHome), + configPath, + environment: { + ...fake.environment, + FIXTURE_SCAN_VALUE: name, + }, + }), + resolvePluginPython: async () => "/managed/python", + prepareOutputDir: async () => scanDir, + repositoryRevision: async () => "deadbeef", + runWorkbench: async (_options, args, input) => { + if (args[0] === "register-cli-scan") + recipeConfig = JSON.parse(input!).recipe.config; + return mockWorkbench(args, input); + }, + createCodex: (options: CodexOptions) => { + const codex = new Codex(options); + return { + startThread: (threadOptions: ThreadOptions) => { + const thread = codex.startThread(threadOptions); + return { + get id() { + return thread.id; + }, + runStreamed: async ( + ...args: Parameters + ) => { + if (thread.id === null) { + await copyCompletedScan(root); + if (mode === "deep") { + const coveragePath = join( + scanDir, + "coverage.json", + ); + const coverage = JSON.parse( + await readFile(coveragePath, "utf8"), + ); + coverage.mode = "deep_repository"; + const coverageBytes = JSON.stringify(coverage); + await writeFile(coveragePath, coverageBytes); + const manifestPath = join( + scanDir, + "scan-manifest.json", + ); + const manifest = JSON.parse( + await readFile(manifestPath, "utf8"), + ); + manifest.scan.artifacts.find( + (artifact: { path: string }) => + artifact.path === "coverage.json", + ).sha256 = createHash("sha256") + .update(coverageBytes) + .digest("hex"); + await writeFile( + manifestPath, + JSON.stringify(manifest), + ); + } + } + return thread.runStreamed(...args); + }, + }; + }, + }; + }, + }, + ); + clients.push(client); + const postScanPrompt = "Summarize the completed synthetic scan."; + const result = await client.run(repository, { + mode, + postScanPrompt, + }); + expect(result.threadId).toBe(`fixture-${name}-thread`); + expect(result.turnResult.usage).toBeNull(); + const invocations = (await readFile(marker, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + const lookups = invocations.filter((child) => + child.args.includes("models"), + ); + const children = invocations.filter( + (child) => !child.args.includes("models"), + ); + expect(recipeConfig?.["model_reasoning_summary"]).toBe(summary); + expect(lookups).toHaveLength(captureSummary ? 1 : 0); + for (const lookup of lookups) { + expect(await realpath(lookup.cwd)).toBe(await realpath(scanDir)); + expect(lookup.home).toBe(codexHome); + expect(lookup.key).toBe(`synthetic-${name}-key`); + expect(lookup.value).toBe(name); + expect(lookup.args).toContain(`model=${JSON.stringify(model)}`); + } + const preflight = await readFile(configPath, "utf8"); + expect(parseToml(preflight)["model_reasoning_summary"]).toBe( + summary, + ); + expect(preflight).not.toContain(`synthetic-${name}-key`); + expect(children).toHaveLength(2); + expect(children[1].prompt).toBe(postScanPrompt); + expect(children[1].args).toContain("resume"); + expect(children[1].args).toContain(`fixture-${name}-thread`); + for (const child of children) { + expect(child.executable).toBe( + process.platform === "win32" + ? win32.toNamespacedPath(fake.command.command) + : fake.command.command, + ); + expect(child.home).toBe(codexHome); + expect(child.key).toBe(`synthetic-${name}-key`); + expect(child.value).toBe(name); + expect(child.args).toContain(`model=${JSON.stringify(model)}`); + expect(child.args).toContain( + `model_provider=${JSON.stringify(provider)}`, + ); + expect(child.args).toContain('model_reasoning_effort="ultra"'); + expect(child.args).toContain( + `model_reasoning_summary=${JSON.stringify(name === "first" ? "none" : "concise")}`, + ); + expect(child.args).toContain( + `service_tier=${JSON.stringify(name === "first" ? "flex" : "fast")}`, + ); + expect(child.args).toContain( + "features.multi_agent_v2.max_concurrent_threads_per_session=4", + ); + expect(child.args).toContain( + 'default_permissions="codex_security_scan"', + ); + expect(child.args).toContain('approval_policy="on-request"'); + } + }), + ); + for (const outcome of outcomes) { + if (outcome.status === "rejected") throw outcome.reason; + } + } finally { + await Promise.all(clients.map((client) => client.close())); + } + }, + ); + test("closes a real Codex subprocess cleanly after a streamed terminal failure", async () => { const root = await temporaryDirectory(); const repository = join(root, "repository"); diff --git a/sdk/typescript/tests-ts/build-plugin.test.ts b/sdk/typescript/tests-ts/build-plugin.test.ts index ecd012ee1..e01a22e03 100644 --- a/sdk/typescript/tests-ts/build-plugin.test.ts +++ b/sdk/typescript/tests-ts/build-plugin.test.ts @@ -1,19 +1,23 @@ import { execFile } from "node:child_process"; import { chmod, + copyFile, + cp, mkdir, mkdtemp, readFile, readdir, rm, stat, + symlink, writeFile, } from "node:fs/promises"; import { tmpdir } from "node:os"; import { delimiter, join } from "node:path"; -import { fileURLToPath } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { promisify } from "node:util"; import { afterEach, describe, expect, test } from "bun:test"; +import { transform } from "esbuild"; import { buildBundledPlugin } from "../scripts/build-plugin.mjs"; import { assertGeneratedPluginUntracked } from "../scripts/check-plugin-source.mjs"; @@ -69,8 +73,163 @@ afterEach(async () => { }); describe("bundled plugin build", () => { - test("builds the MCP runtime without invoking an npm launcher", async () => { + test("bundles the native policy proof with only SDK dependencies", async () => { const root = await temporaryDirectory(); + const plugin = join(root, "plugins", "codex-security"); + const native = join(plugin, "native"); + const sdk = join(root, "sdk", "typescript"); + const source = new URL("../../../plugins/codex-security/", import.meta.url); + await mkdir(native, { recursive: true }); + await mkdir(sdk, { recursive: true }); + await symlink( + fileURLToPath(new URL("../node_modules", import.meta.url)), + join(sdk, "node_modules"), + process.platform === "win32" ? "junction" : "dir", + ); + for (const name of ["schemas", "mcp-app/src"]) { + await cp(new URL(name, source), join(plugin, name), { recursive: true }); + } + await copyFile( + new URL("mcp-app/helpers-main.ts", source), + join(plugin, "mcp-app", "helpers-main.ts"), + ); + for (const name of [ + "binding", + "platform", + "windows-binding", + "windows-flags", + "windows-files", + "proof-policy-windows", + ]) { + const compiled = await transform( + await readFile(new URL(`native/${name}.mts`, source), "utf8"), + { loader: "ts", format: "esm", target: "node20" }, + ); + await writeFile(join(native, `${name}.mjs`), compiled.code); + } + const { nativeTarget } = await import( + pathToFileURL(join(native, "platform.mjs")).href + ); + const binary = process.platform === "win32" ? "windows.node" : "unix.node"; + // The build copies this artifact; this portable test does not load native code. + await writeFixture( + native, + `dist/${nativeTarget}/${binary}`, + "native fixture", + ); + await expect( + stat(join(plugin, "mcp-app", "node_modules")), + ).rejects.toMatchObject({ + code: "ENOENT", + }); + await execFileAsync( + "node", + [join(native, "proof-policy-windows.mjs"), "build"], + { + cwd: native, + env: { ...process.env, NODE_PATH: "" }, + }, + ); + const proof = join(native, "dist", nativeTarget, "policy-proof"); + expect( + await readFile( + join(proof, "native", nativeTarget, "windows.node"), + "utf8", + ), + ).toBe("native fixture"); + const helper = join(root, "helpers.cjs"); + await copyFile(join(proof, "helpers.cjs"), helper); + await execFileAsync("node", [ + "--eval", + "require('node:fs').unlinkSync(process.argv[1])", + join(sdk, "node_modules"), + ]); + await expect(stat(join(sdk, "node_modules"))).rejects.toMatchObject({ + code: "ENOENT", + }); + const result = await execFileAsync( + "node", + [ + "--eval", + ` + const assert = require("node:assert/strict"); + const helper = require(process.argv.pop()); + const input = { + scanId: "synthetic-scan", + manifest: { scan: {} }, + findings: { findings: [] }, + coverage: { + completeness: "complete", surfaces: [], explicitExclusions: [], deferred: [], + }, + }; + assert.equal(helper.parseCanonicalScanDraft(input).scanId, input.scanId); + assert.throws(() => helper.parseCanonicalScanDraft({ + ...input, coverage: { ...input.coverage, completeness: "invalid" }, + })); + console.log("Bundled parser accepted valid input and rejected invalid coverage."); + `, + helper, + ], + { cwd: root, env: { ...process.env, NODE_PATH: "" } }, + ); + expect(result.stdout).toBe( + "Bundled parser accepted valid input and rejected invalid coverage.\n", + ); + expect(result.stderr).toBe(""); + }); + + test("builds the MCP runtime through a directory alias with only MCP dependencies and no npm launcher", async () => { + const root = await temporaryDirectory(); + const plugin = join(root, "plugins", "codex-security"); + const mcp = join(plugin, "mcp-app"); + const sdk = join(root, "sdk", "typescript"); + const source = new URL("../../../plugins/codex-security/", import.meta.url); + await mkdir(mcp, { recursive: true }); + await mkdir(sdk, { recursive: true }); + for (const name of [ + "package.json", + "tsconfig.json", + "main.ts", + "artifact-writer-main.ts", + "helpers-main.ts", + "server.ts", + "src", + "scripts", + "templates", + ]) { + await cp(new URL(`mcp-app/${name}`, source), join(mcp, name), { + recursive: true, + }); + } + for (const name of [ + "schemas", + "native/prebuilt", + "plugin-files.json", + "scripts/reserved_artifact_paths.json", + ]) { + await cp(new URL(name, source), join(plugin, name), { recursive: true }); + } + for (const name of await readdir(new URL("native/", source))) { + if (/\.(?:mjs|mts)$/.test(name)) { + await copyFile( + new URL(`native/${name}`, source), + join(plugin, "native", name), + ); + } + } + for (const name of ["src", "package.json", "tsconfig.json"]) { + await cp(new URL(`../${name}`, import.meta.url), join(sdk, name), { + recursive: true, + }); + } + await symlink( + fileURLToPath(new URL("mcp-app/node_modules", source)), + join(mcp, "node_modules"), + process.platform === "win32" ? "junction" : "dir", + ); + await expect(stat(join(sdk, "node_modules"))).rejects.toMatchObject({ + code: "ENOENT", + }); const bin = join(root, "bin"); const launcher = process.platform === "win32" ? "npm.cmd" : "npm"; await writeFixture( @@ -80,15 +239,23 @@ describe("bundled plugin build", () => { ); if (process.platform !== "win32") await chmod(join(bin, launcher), 0o755); + const alias = join(await temporaryDirectory(), "plugin link"); + await symlink( + root, + alias, + process.platform === "win32" ? "junction" : "dir", + ); const destination = join(root, "mcp"); await execFileAsync( "node", [ - fileURLToPath( - new URL( - "../../../plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs", - import.meta.url, - ), + join( + alias, + "plugins", + "codex-security", + "mcp-app", + "scripts", + "build_mcp_app.mjs", ), "--output", destination, @@ -96,11 +263,20 @@ describe("bundled plugin build", () => { { env: { ...process.env, + NODE_PATH: "", PATH: [bin, process.env["PATH"]].filter(Boolean).join(delimiter), }, }, ); + await execFileAsync("node", [ + "--eval", + "require('node:fs').unlinkSync(process.argv[1])", + join(mcp, "node_modules"), + ]); + await expect(stat(join(mcp, "node_modules"))).rejects.toMatchObject({ + code: "ENOENT", + }); const contract = JSON.parse( await readFile( new URL( @@ -125,6 +301,55 @@ describe("bundled plugin build", () => { ]); expect(helper.stdout).toBe("[]\n"); expect(helper.stderr).toBe(""); + + const repository = await temporaryDirectory(); + const policy = "Preserve this synthetic inherited security policy."; + await writeFixture( + repository, + "SECURITY.md", + `# Synthetic policy\n${policy}\n`, + ); + await mkdir(join(root, "scripts"), { recursive: true }); + await copyFile( + new URL("scripts/launch_codex_security_mcp", source), + join(root, "scripts", "launch_codex_security_mcp"), + ); + const node = ( + await execFileAsync("node", ["--print", "process.execPath"]) + ).stdout.trim(); + for (const pluginPath of [root, alias]) { + const linkedHelper = join(pluginPath, "mcp", "helpers.mjs"); + const list = await execFileAsync( + process.platform === "win32" ? node : "/bin/sh", + [ + ...(process.platform === "win32" + ? [linkedHelper] + : [ + join(pluginPath, "scripts", "launch_codex_security_mcp"), + "--helper", + ]), + "resolve-security-md", + "--repo", + repository, + "--list", + ], + { env: { ...process.env, CODEX_MCP_NODE_PATH: node, NODE_PATH: "" } }, + ); + expect(list.stdout).toBe('["SECURITY.md"]\n'); + expect(list.stderr).toBe(""); + const guidance = await execFileAsync(node, [ + linkedHelper, + "resolve-security-md", + "--repo", + repository, + "--scope", + repository, + "--out", + "-", + ]); + expect(guidance.stdout).toContain(policy); + expect(guidance.stderr).toBe(""); + } }); test("builds from a source snapshot without Git metadata", async () => { diff --git a/sdk/typescript/tests-ts/cost-context.test.ts b/sdk/typescript/tests-ts/cost-context.test.ts index dfdd69277..437478cb4 100644 --- a/sdk/typescript/tests-ts/cost-context.test.ts +++ b/sdk/typescript/tests-ts/cost-context.test.ts @@ -129,3 +129,42 @@ test("component totals preserve uncertainty and label legacy records without rep ]), ).toContain("at least $8.00"); }); + +test.each([false, true])( + "attributed model totals retain context bounds and partial coverage (%s)", + (unknownUpper) => { + const first = { + model: "gpt-5.6-sol", + input_tokens: 1_000_000, + cached_input_tokens: 0, + cache_write_input_tokens: 0, + output_tokens: 0, + }; + const second = { + ...first, + model: unknownUpper ? "gpt-daybreak-red-latest" : "gpt-5.6-terra", + }; + const cost = estimateScanCost("gpt-6-astra", { + input_tokens: 2_000_000, + cached_input_tokens: 0, + cache_write_input_tokens: 0, + output_tokens: 0, + modelUsage: [first, second], + coverage: "partial", + })!; + expect(cost.coverage).toBe("partial"); + expect(cost.modelCosts?.map((part) => part.model)).toEqual([ + first.model, + second.model, + ]); + expect(cost.estimatedUsd).toBe(unknownUpper ? 16.5 : 6); + expect(cost.estimatedUsdRange).toEqual({ + min: cost.estimatedUsd, + max: unknownUpper ? null : 12, + context: "unknown", + }); + expect(formatScanCost(cost)).toContain( + unknownUpper ? "upper estimate unavailable" : "context unknown", + ); + }, +); diff --git a/sdk/typescript/tests-ts/cost.test.ts b/sdk/typescript/tests-ts/cost.test.ts index aaa2f09e2..d04056615 100644 --- a/sdk/typescript/tests-ts/cost.test.ts +++ b/sdk/typescript/tests-ts/cost.test.ts @@ -1,10 +1,13 @@ import { spawnSync } from "node:child_process"; import { appendFile, + cp, mkdir, mkdtemp, realpath, + readFile, rm, + symlink, writeFile, } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -15,9 +18,14 @@ import { estimateScanCost, ScanCostTracker, type ScanSessionEvent, + type ScanCost, } from "../src/cost.js"; import type { ScanActivity } from "../src/scan-activity.js"; -import { formatTokenUsage, tokenUsage } from "../src/cost-model.js"; +import { + estimateScanCostLowerBound, + formatTokenUsage, + tokenUsage, +} from "../src/cost-model.js"; import { readScanLogs } from "../src/scan-logs.js"; import { sessionParentThreadId } from "../src/scan-sessions.js"; import type { ScanProgress } from "../src/worker-progress.js"; @@ -2266,3 +2274,479 @@ describe("live scan cost tracking", () => { }, ); }); + +describe("recorded Deep worker homes", () => { + test("only enforces a priced subtotal from a valid attributed usage partition", () => { + const known = { + model: "gpt-5.6-sol", + input_tokens: 1_000, + output_tokens: 0, + }; + const unknown = { model: null, input_tokens: 100, output_tokens: 0 }; + const usage = { + input_tokens: 1_100, + output_tokens: 0, + modelUsage: [known, unknown], + }; + expect(estimateScanCostLowerBound("gpt-5.6-sol", usage)?.estimatedUsd).toBe( + 0.004, + ); + expect(estimateScanCost("gpt-5.6-sol", usage)).toBeNull(); + for (const invalid of [ + { ...usage, input_tokens: 999 }, + { ...usage, modelUsage: [known, known, unknown] }, + { ...usage, modelUsage: [known, { ...unknown, input_tokens: -1 }] }, + { ...usage, modelUsage: [{ ...known, model: null }, unknown] }, + ]) + expect(estimateScanCostLowerBound("gpt-5.6-sol", invalid)).toBeNull(); + }); + + test.each([null, "synthetic-unpriced-model"])( + "reports an internal priced lower bound with model %p without inventing a total", + async (unknownModel) => { + const home = await codexHome(); + const at = "2026-09-01T00:00:02Z"; + const known = await writeSession(home, "owner", {}); + await appendFile( + known, + JSON.stringify({ + type: "token_usage_record", + timestamp: at, + payload: { + thread_id: "owner", + turn_id: "turn", + response_id: "known-response", + model: "gpt-5.6-sol", + usage: { input_tokens: 1_000, output_tokens: 0 }, + }, + }) + "\n", + ); + const unknown = await writeSession(home, "worker", {}); + await appendFile( + unknown, + JSON.stringify({ + type: "turn_context", + timestamp: at, + payload: { + turn_id: "worker-turn", + ...(unknownModel === null ? {} : { model: unknownModel }), + }, + }) + + "\n" + + JSON.stringify({ + type: "event_msg", + timestamp: at, + payload: { + type: "token_count", + info: { + total_token_usage: { input_tokens: 100, output_tokens: 0 }, + }, + }, + }) + + "\n", + ); + const lowerBounds: Readonly[] = []; + const publicCosts: Readonly[] = []; + const options = { + codexHome: home, + model: "gpt-5.6-sol", + maxCostUsd: 0.003, + onCost: (cost: Readonly) => publicCosts.push(cost), + onCostLowerBound: (cost: Readonly) => lowerBounds.push(cost), + }; + const tracker = new ScanCostTracker(options); + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + executionThreadIds: ["worker"], + owner: { threadId: "owner", turnId: "turn", startedAt: at }, + startedAt: at, + completedAt: null, + })); + tracker.start("owner"); + try { + const snapshot = await tracker.refresh(); + expect(tokenUsage(snapshot.usage)?.input_tokens).toBe(1_100); + expect(snapshot.cost).toBeNull(); + expect(publicCosts).toEqual([]); + expect(lowerBounds).toHaveLength(1); + expect(lowerBounds[0]).toMatchObject({ + inputTokens: 1_000, + estimatedUsd: 0.004, + coverage: "partial", + }); + expect(lowerBounds[0]!.estimatedUsd).toBeGreaterThan( + options.maxCostUsd, + ); + await tracker.refresh(); + expect(lowerBounds).toHaveLength(1); + } finally { + await tracker.stop(); + } + }, + ); + + test.each(["identical", "prefix-first", "prefix-last"] as const)( + "forwards each event occurrence once from copied logs: %s", + async (copy) => { + const home = await codexHome(); + const recordedHome = await codexHome(); + const scanDirectory = join(home, "scan"); + const settings = join(scanDirectory, "artifacts", "deep_discovery"); + await mkdir(settings, { recursive: true }); + await writeFile( + join(settings, "execution-settings.json"), + JSON.stringify({ + version: 1, + settings: { codexHome: recordedHome }, + }), + ); + await mkdir(join(home, "sessions")); + await mkdir(join(recordedHome, "sessions")); + const first = join(home, "sessions", "worker.jsonl"); + const second = join(recordedHome, "sessions", "worker-copy.jsonl"); + const repeated = { + timestamp: "2026-09-01T00:00:02Z", + type: "event_msg", + payload: { type: "agent_message", message: "Reviewing source." }, + }; + const expected = [ + { + timestamp: "2026-09-01T00:00:00Z", + type: "session_meta", + payload: { id: "worker", model: "gpt-5.6-sol" }, + }, + repeated, + repeated, + { + timestamp: "2026-09-01T00:00:03Z", + type: "token_usage_record", + payload: { + thread_id: "worker", + turn_id: "turn", + response_id: "response", + model: "gpt-5.6-sol", + usage: { input_tokens: 100, output_tokens: 0 }, + }, + }, + ]; + const contents = expected.map((event) => JSON.stringify(event) + "\n"); + await writeFile( + first, + contents.slice(0, copy === "prefix-first" ? 2 : 4).join(""), + ); + await writeFile( + second, + contents.slice(0, copy === "prefix-last" ? 2 : 4).join(""), + ); + const events: ScanSessionEvent[] = []; + const options = { + codexHome: home, + scanDirectory, + model: "gpt-5.6-sol", + onSessionEvent: (event: ScanSessionEvent) => events.push(event), + }; + const tracker = new ScanCostTracker(options); + tracker.start("worker"); + try { + expect((await tracker.refresh()).cost?.inputTokens).toBe(100); + expect(events.map((event) => event.event)).toEqual(expected); + await tracker.refresh(); + expect(events).toHaveLength(expected.length); + // Both logs catch up, then a genuine repeated occurrence is copied later. + await writeFile(first, contents.join("")); + await writeFile(second, contents.join("")); + await appendFile(first, JSON.stringify(repeated) + "\n"); + await tracker.refresh(); + expect(events.map((event) => event.event)).toEqual([ + ...expected, + repeated, + ]); + await appendFile(second, JSON.stringify(repeated) + "\n"); + await tracker.stop(); + expect(events.map((event) => event.event)).toEqual([ + ...expected, + repeated, + ]); + // Re-reading after the owner interval becomes available filters early + // events, but must not renumber the surviving source occurrences. + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + executionThreadIds: ["worker"], + owner: { + threadId: "worker", + turnId: "turn", + startedAt: "2026-09-01T00:00:03Z", + }, + startedAt: "2026-09-01T00:00:03Z", + completedAt: "2026-09-01T00:00:04Z", + })); + await tracker.refresh(); + expect(events.map((event) => event.event)).toEqual([ + ...expected, + repeated, + ]); + events.length = 0; + const reconstructed = new ScanCostTracker(options); + reconstructed.start("worker"); + await reconstructed.stop(); + expect(events.map((event) => event.event)).toEqual([ + ...expected, + repeated, + ]); + } finally { + await tracker.stop(); + } + }, + ); + + test("keeps resumed worker usage and current parent usage isolated per scan", async () => { + const currentHome = await codexHome(); + const firstHome = await codexHome(); + const secondHome = await codexHome(); + const at = "2026-09-01T00:00:02Z"; + const trackers: ScanCostTracker[] = []; + const fixture = async (home: string, id: string, count: number) => { + const path = await writeSession(home, id, {}); + await appendFile( + path, + [ + { + type: "turn_context", + timestamp: at, + payload: { turn_id: "scan-turn", model: "gpt-5.6-sol" }, + }, + { + type: "token_usage_record", + timestamp: at, + payload: { + thread_id: id, + turn_id: "scan-turn", + response_id: `${id}-response`, + model: "gpt-5.6-sol", + usage: { input_tokens: count, output_tokens: 0 }, + }, + }, + ] + .map((event) => JSON.stringify(event)) + .join("\n") + "\n", + ); + return path; + }; + try { + const cases = [ + { id: "one", home: firstHome, parent: 10, discovery: 20, reducer: 30 }, + { id: "two", home: secondHome, parent: 11, discovery: 21, reducer: 31 }, + ]; + for (const row of cases) { + const scanDirectory = join(currentHome, "scans", row.id); + const settingsDirectory = join( + scanDirectory, + "artifacts", + "deep_discovery", + ); + await mkdir(settingsDirectory, { recursive: true }); + await writeFile( + join(settingsDirectory, "execution-settings.json"), + JSON.stringify({ + version: 1, + settings: { codexHome: row.home }, + }), + ); + await fixture(currentHome, `${row.id}-parent`, row.parent); + const discovery = await fixture( + row.home, + `${row.id}-discovery`, + row.discovery, + ); + await fixture(row.home, `${row.id}-reducer`, row.reducer); + await fixture(row.home, `${row.id}-unrelated`, 10_000); + // Repeated receipt identity after reconnect must remain one charge. + const duplicate = (await readFile(discovery, "utf8")) + .trim() + .split("\n") + .at(-1)!; + await appendFile(discovery, duplicate + "\n"); + const attribution = { + formatVersion: 1 as const, + executionThreadIds: [`${row.id}-discovery`, `${row.id}-reducer`], + owner: { + threadId: `${row.id}-parent`, + turnId: "scan-turn", + startedAt: at, + }, + startedAt: at, + completedAt: null, + }; + const tracker = new ScanCostTracker({ + codexHome: currentHome, + scanDirectory, + model: "gpt-5.6-sol", + }); + tracker.setAttributionReader(async () => attribution); + tracker.start(`${row.id}-parent`); + trackers.push(tracker); + } + const initial = await Promise.all( + trackers.map((tracker) => tracker.refresh()), + ); + expect( + initial.map((snapshot) => tokenUsage(snapshot.usage)?.input_tokens), + ).toEqual([60, 63]); + expect(initial.map((snapshot) => snapshot.cost?.inputTokens)).toEqual([ + 60, 63, + ]); + const firstDirectory = join(currentHome, "scans", "one"); + const rebuilt = new ScanCostTracker({ + codexHome: currentHome, + scanDirectory: firstDirectory, + model: "gpt-5.6-sol", + }); + rebuilt.setAttributionReader(async () => ({ + formatVersion: 1, + executionThreadIds: [ + "one-discovery", + "one-reducer", + "one-missing-attempt", + ], + owner: { threadId: "one-parent", turnId: "scan-turn", startedAt: at }, + startedAt: at, + completedAt: null, + })); + rebuilt.start("one-parent"); + trackers.push(rebuilt); + expect((await rebuilt.refresh()).usage).toMatchObject({ + input_tokens: 60, + coverage: "partial", + }); + await fixture(firstHome, "one-missing-attempt", 7); + expect((await rebuilt.refresh()).cost?.inputTokens).toBe(67); + expect((await trackers[1]!.refresh()).cost?.inputTokens).toBe(63); + } finally { + await Promise.all(trackers.map((tracker) => tracker.stop())); + } + }); + + test("reads a recorded directory alias only once", async () => { + const home = await codexHome(); + const alias = join(await codexHome(), "recorded-home"); + await symlink( + home, + alias, + process.platform === "win32" ? "junction" : "dir", + ); + const scanDirectory = join(home, "scan"); + const directory = join(scanDirectory, "artifacts", "deep_discovery"); + await mkdir(directory, { recursive: true }); + await writeFile( + join(directory, "execution-settings.json"), + JSON.stringify({ version: 1, settings: { codexHome: alias } }), + ); + await writeSession(home, "worker", { input_tokens: 100, output_tokens: 0 }); + const events: ScanSessionEvent[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + scanDirectory, + model: "gpt-5.6-sol", + onSessionEvent: (event) => events.push(event), + }); + tracker.start("worker"); + try { + expect((await tracker.stop()).cost?.inputTokens).toBe(100); + expect(events).toHaveLength(2); + } finally { + await tracker.stop(); + } + }); + + test.each([ + ["identical", false], + ["identical", true], + ["prefix-first", false], + ["prefix-first", true], + ["prefix-last", false], + ["prefix-last", true], + ] as const)( + "prices copied response records (%s, attribution: %s)", + async (copy, attributed) => { + const home = await codexHome(); + const recordedHome = await codexHome(); + const scanDirectory = join(home, "scan"); + const directory = join(scanDirectory, "artifacts", "deep_discovery"); + await mkdir(directory, { recursive: true }); + await writeFile( + join(directory, "execution-settings.json"), + JSON.stringify({ version: 1, settings: { codexHome: recordedHome } }), + ); + const path = await writeSession(home, "worker", {}); + for (const [id, model, input, output] of [ + ["response-one", "gpt-5.6-sol", 100, 10], + ["response-two", "gpt-6-astra", 50, 5], + ] as const) { + await appendFile( + path, + JSON.stringify({ + type: "token_usage_record", + timestamp: "2026-09-01T00:00:02Z", + payload: { + thread_id: "worker", + turn_id: "turn", + response_id: id, + model, + usage: { input_tokens: input, output_tokens: output }, + }, + }) + "\n", + ); + } + await mkdir(join(recordedHome, "sessions")); + const copiedPath = join(recordedHome, "sessions", "copied-worker.jsonl"); + await cp(path, copiedPath); + const prefix = + (await readFile(path, "utf8")) + .trimEnd() + .split("\n") + .slice(0, -1) + .join("\n") + "\n"; + if (copy === "prefix-first") await writeFile(path, prefix); + if (copy === "prefix-last") await writeFile(copiedPath, prefix); + const tracker = new ScanCostTracker({ + codexHome: home, + scanDirectory, + model: "gpt-5.6-sol", + }); + if (attributed) + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + executionThreadIds: ["worker"], + owner: { + threadId: null, + turnId: null, + startedAt: "2026-09-01T00:00:00Z", + }, + startedAt: "2026-09-01T00:00:00Z", + completedAt: null, + })); + tracker.start("worker"); + try { + const snapshot = await tracker.stop(); + expect(snapshot.usage).toMatchObject({ + input_tokens: 150, + output_tokens: 15, + total_tokens: 165, + }); + expect( + Object.fromEntries( + snapshot.cost!.modelCosts!.map((part) => [ + part.model, + [part.inputTokens, part.outputTokens], + ]), + ), + ).toEqual({ + "gpt-5.6-sol": [100, 10], + "gpt-6-astra": [50, 5], + }); + } finally { + await tracker.stop(); + } + }, + ); +}); diff --git a/sdk/typescript/tests-ts/deep-finalization.test.ts b/sdk/typescript/tests-ts/deep-finalization.test.ts new file mode 100644 index 000000000..b9cd6055f --- /dev/null +++ b/sdk/typescript/tests-ts/deep-finalization.test.ts @@ -0,0 +1,982 @@ +import { execFileSync } from "node:child_process"; +import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, expect, test } from "bun:test"; +import type { ThreadEvent } from "@openai/codex-sdk"; +import { + ScanCostLimitExceededError, + ScanInterruptedError, +} from "../src/errors.js"; +import { + prepareScanArtifactRestorer, + runWorkbench, + type WorkbenchCommandOptions, +} from "../src/runtime.js"; +import { TestClient } from "./support/api-client.js"; +import { + completedEvents, + createApiTestFixtures, + preparedRuntime, +} from "./support/api-events.js"; +import { PLUGIN_ROOT } from "./plugin-root.js"; + +const { temporaryDirectory, cleanup } = createApiTestFixtures(); +afterEach(cleanup); +const threadId = "1af317a1-c9ed-4c73-b428-cb0d160cf8e8"; +const followUp = "Explain the selected finding."; + +for (const boundary of ["registration", "stream-start"] as const) { + test(`SDK cancels registered Deep Scan before first thread event: ${boundary}`, async () => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const scanDir = join(root, "scan"); + const codexHome = join(root, "codex-home"); + await Promise.all([ + mkdir(repository), + mkdir(scanDir, { mode: 0o700 }), + mkdir(codexHome), + ]); + await writeFile(join(repository, "source.py"), "# Synthetic source\n"); + const environment = { + ...process.env, + CODEX_HOME: codexHome, + CODEX_SECURITY_STATE_DIR: join(root, "state"), + }; + const cancellation = new AbortController(); + const reason = new Error( + "Synthetic cancellation before first thread event", + ); + const commands: string[][] = []; + let scanId = ""; + let savedOptions: WorkbenchCommandOptions; + let startedTurns = 0; + const client = new TestClient( + {}, + { + environment, + prepareRuntime: async () => ({ + ...preparedRuntime(codexHome), + environment, + }), + resolvePluginPython: async () => "python3", + prepareOutputDir: async () => scanDir, + runWorkbench: async (options, args, input) => { + savedOptions = options; + commands.push([...args]); + const result = await runWorkbench(options, args, input); + if (args[0] === "register-cli-scan") { + scanId = result["scanId"] as string; + if (boundary === "registration") cancellation.abort(reason); + } + return result; + }, + createCodex: () => ({ + startThread: () => ({ + id: null, + runStreamed: async () => { + startedTurns++; + cancellation.abort(reason); + throw reason; + }, + }), + }), + }, + ); + try { + const error = await client + .run(repository, { + mode: "deep", + signal: cancellation.signal, + postScanPrompt: followUp, + }) + .catch((error: unknown) => error); + expect(error).toBeInstanceOf(ScanInterruptedError); + expect((error as ScanInterruptedError).cause).toBe(reason); + const stopped = await runWorkbench( + { ...savedOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + expect(stopped["scan"]).toMatchObject({ + progress: { status: "canceled" }, + }); + expect(commands.filter((args) => args[0] === "cancel-scan")).toEqual([ + ["cancel-scan", "--scan-id", scanId], + ]); + expect(commands.some((args) => args[0] === "fail-scan")).toBe(false); + expect(commands.some((args) => args[0] === "set-scan-thread")).toBe( + false, + ); + expect(startedTurns).toBe(boundary === "registration" ? 0 : 1); + } finally { + await client.close(); + } + }, 30_000); +} + +const outcomes = [ + "failed", + "completed", + "restart", + "canceled-before-publication", + "canceled-during-publication", + "canceled-during-resumed-publication", + "published-before-cancellation", + "closed-during-publication", + "budget-during-publication", + "budget-after-deep-finish", + "budget-during-resumed-publication", + "closed-during-resumed-publication", + "lost-completion-response", + "lost-completion-response-followup-canceled", + "completion-before-commit-fails", + "followup-canceled", +] as const; +type BudgetCompletionFault = "lost" | "before-commit" | "lost-and-canceled"; +const cases: { + outcome: (typeof outcomes)[number]; + budgetCompletionFault?: BudgetCompletionFault; + initialResumeUsage?: boolean; + unpricedUsage?: boolean; + cancellationFault?: "status-read" | "deep-state-read" | "cancel-response"; +}[] = [ + ...outcomes.map((outcome) => ({ outcome })), + ...( + [ + "budget-during-publication", + "budget-after-deep-finish", + "budget-during-resumed-publication", + ] as const + ).flatMap((outcome) => [ + { outcome, unpricedUsage: true }, + { outcome, unpricedUsage: true, budgetCompletionFault: "lost" as const }, + ]), + ...( + [ + "budget-during-publication", + "budget-after-deep-finish", + "budget-during-resumed-publication", + ] as const + ).map((outcome) => ({ outcome, budgetCompletionFault: "lost" as const })), + { + outcome: "budget-after-deep-finish", + budgetCompletionFault: "before-commit", + }, + { + outcome: "budget-after-deep-finish", + budgetCompletionFault: "lost-and-canceled", + }, + { + outcome: "canceled-during-publication", + cancellationFault: "status-read", + }, + { + outcome: "canceled-during-publication", + cancellationFault: "cancel-response", + }, + { + outcome: "canceled-during-publication", + cancellationFault: "deep-state-read", + }, + { + outcome: "canceled-during-resumed-publication", + cancellationFault: "deep-state-read", + }, + { + outcome: "canceled-before-publication", + cancellationFault: "deep-state-read", + }, + { + outcome: "canceled-during-resumed-publication", + initialResumeUsage: true, + }, +]; +for (const { + outcome, + budgetCompletionFault, + cancellationFault, + initialResumeUsage, + unpricedUsage, +} of cases) { + const resumedStop = outcome.includes("-resumed-"); + const restart = outcome === "restart" || resumedStop; + const closed = outcome.startsWith("closed-"); + const budgeted = outcome.startsWith("budget-"); + const canceledFollowUp = outcome.endsWith("followup-canceled"); + const loseCompletionResponse = outcome.startsWith("lost-completion-response"); + const name = + outcome === "followup-canceled" + ? "SDK preserves a selected aggregate when its follow-up is canceled" + : `SDK handles selected aggregate: ${outcome}${budgetCompletionFault ? ` (budget completion ${budgetCompletionFault})` : ""}${cancellationFault ? ` (cancellation ${cancellationFault})` : ""}${initialResumeUsage ? " (initial resume cost)" : ""}${unpricedUsage ? " (unpriced remainder)" : ""}`; + const runCase = async () => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const scanDir = join(root, "scan"); + const codexHome = join(root, "codex-home"); + const stateDir = join(root, "state"); + await Promise.all([ + mkdir(repository), + mkdir(scanDir, { mode: 0o700 }), + mkdir(codexHome), + ]); + await writeFile(join(repository, "extract.py"), "# Synthetic source\n"); + const environment = { + ...process.env, + CODEX_HOME: codexHome, + CODEX_SECURITY_STATE_DIR: stateDir, + }; + const cancellation = new AbortController(); + const parentError = new Error( + "Parent turn ended before its final completion tool call", + ); + let scanId = ""; + let workbenchOptions: WorkbenchCommandOptions; + let publicationFails = restart; + let completionReceiptLost = loseCompletionResponse; + let budgetReceiptLost = + budgetCompletionFault === "lost" || + budgetCompletionFault === "lost-and-canceled"; + let budgetTriggered = false; + let cancellationReadLost = false; + let lostCancellationDeepState: unknown; + let originalFinalizationInput: unknown; + let selectedPath = ""; + let selectedBytes: Buffer; + let originalResumeSignal: AbortSignal | undefined; + let acceptedReport = ""; + let completedArtifacts: Buffer[] = []; + const modelInputs: string[] = []; + const commands: string[] = []; + const reportedCosts: number[] = []; + const usagePath = join( + codexHome, + "sessions", + "2026", + "01", + "01", + `rollout-${threadId}.jsonl`, + ); + const recordBudgetUsage = () => + appendFile( + usagePath, + JSON.stringify({ + timestamp: new Date().toISOString(), + type: "turn_context", + payload: { + turn_id: "synthetic-scan-turn", + model: "gpt-5.6-sol", + }, + }) + + "\n" + + JSON.stringify({ + timestamp: new Date().toISOString(), + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: 1_250, + cached_input_tokens: 200, + output_tokens: 30, + }, + }, + }, + }) + + "\n" + + (unpricedUsage + ? [ + JSON.stringify({ + timestamp: new Date().toISOString(), + type: "turn_context", + payload: { + turn_id: "synthetic-scan-turn", + model: "synthetic-unpriced-model", + }, + }), + JSON.stringify({ + timestamp: new Date().toISOString(), + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: 1_350, + cached_input_tokens: 200, + output_tokens: 30, + }, + }, + }, + }), + "", + ].join("\n") + : ""), + ); + let closePromise: Promise | undefined; + const makeClient = () => + new TestClient( + {}, + { + environment, + prepareScanArtifactRestorer, + prepareRuntime: async () => { + const runtime = preparedRuntime(codexHome); + const manifest = JSON.parse( + await readFile( + join(PLUGIN_ROOT, ".codex-plugin/plugin.json"), + "utf8", + ), + ); + return { + ...runtime, + environment, + persistentCredentialHome: true, + plugin: { ...runtime.plugin, version: manifest.version }, + }; + }, + resolvePluginPython: async () => "python3", + prepareOutputDir: async () => scanDir, + runWorkbench: async (options, args, input) => { + workbenchOptions = options; + if (args[0] === "get-cli-scan-resume") + originalResumeSignal = options.signal; + commands.push(args[0]!); + if ( + args[0] === "get-scan" && + cancellation.signal.aborted && + cancellationFault === "status-read" + ) { + throw new Error("Synthetic lost cancellation status response"); + } + if (args[0] === "write-scan-draft" && publicationFails) { + publicationFails = false; + throw new Error("Synthetic publication write failure"); + } + if ( + args[0] === "complete-scan" && + outcome === "completion-before-commit-fails" + ) + throw new Error("Synthetic completion failure before commit"); + if ( + args[0] === "complete-budget-exhausted-scan" && + budgetCompletionFault === "before-commit" + ) + throw new Error( + "Synthetic budget completion failure before commit", + ); + const result = await runWorkbench(options, args, input); + if ( + args[0] === "get-deep-scan" && + cancellation.signal.aborted && + cancellationFault === "deep-state-read" && + !cancellationReadLost + ) { + cancellationReadLost = true; + lostCancellationDeepState = result["deepScan"]; + throw new Error( + "Synthetic lost cancellation Deep state response", + ); + } + if ( + args[0] === "cancel-scan" && + cancellationFault === "cancel-response" + ) { + completedArtifacts = await Promise.all( + ["report.md", "scan-manifest.json"].map((name) => + readFile(join(scanDir, name)), + ), + ); + throw new Error("Synthetic lost cancellation response"); + } + if ( + args[0] === "complete-budget-exhausted-scan" && + budgetReceiptLost + ) { + budgetReceiptLost = false; + if (budgetCompletionFault === "lost-and-canceled") { + completedArtifacts = await Promise.all( + ["report.md", "scan-manifest.json"].map((name) => + readFile(join(scanDir, name)), + ), + ); + cancellation.abort( + "Synthetic cancellation after budget completion", + ); + } + throw Object.assign( + new Error("Synthetic lost budget completion response"), + { code: "ETIMEDOUT" }, + ); + } + if (args[0] === "complete-scan" && completionReceiptLost) { + completionReceiptLost = false; + throw new Error("Synthetic lost completion response"); + } + if ( + !budgetTriggered && + ((args[0] === "write-scan-draft" && + (outcome === "budget-during-publication" || + outcome === "budget-during-resumed-publication")) || + (args[0] === "finish-deep-scan" && + outcome === "budget-after-deep-finish")) + ) { + budgetTriggered = true; + await recordBudgetUsage(); + await new Promise((resolve) => { + if (options.signal?.aborted) resolve(); + else + options.signal!.addEventListener("abort", () => resolve(), { + once: true, + }); + }); + } + if (args[0] === "write-scan-draft" && closed) { + closePromise = client.close(); + } + if ( + args[0] === "write-scan-draft" && + (outcome === "canceled-during-publication" || + outcome === "canceled-during-resumed-publication") + ) { + if (initialResumeUsage) { + expect(originalResumeSignal?.reason).toBeInstanceOf( + ScanCostLimitExceededError, + ); + expect(options.signal?.aborted).toBe(false); + } + cancellation.abort("Synthetic user cancellation"); + } + if ( + args[0] === "complete-scan" && + outcome === "published-before-cancellation" + ) { + cancellation.abort( + "Synthetic user cancellation after completion", + ); + } + if (args[0] === "register-cli-scan") + scanId = result["scanId"] as string; + return result; + }, + createCodex: () => { + const thread = { + id: threadId, + async runStreamed(input: string) { + modelInputs.push(input); + if (input === followUp) { + if (canceledFollowUp) { + const reportPath = join(scanDir, "report.md"); + acceptedReport = await readFile(reportPath, "utf8"); + expect(acceptedReport).toContain( + "Validate the resolved destination", + ); + await writeFile( + reportPath, + "Incomplete follow-up report.\n", + ); + cancellation.abort( + "Synthetic cancellation during follow-up", + ); + } + return { events: completedEvents(threadId) }; + } + expect(modelInputs.length).toBe(1); + async function* events(): AsyncGenerator { + yield { type: "thread.started", thread_id: threadId }; + await runWorkbench(workbenchOptions, [ + "begin-deep-scan", + "--scan-id", + scanId, + "--thread-id", + threadId, + ]); + const draft = { + scanId, + complete: true, + findings: [ + { + ruleId: "path-traversal.archive", + title: "Unsafe archive extraction", + summary: + "An untrusted entry reaches a filesystem write.", + severity: { level: "high" }, + confidence: { + level: "high", + rationale: "Source evidence.", + }, + taxonomy: { + category: "path-traversal", + cwe: ["CWE-22"], + }, + locations: [{ path: "extract.py", startLine: 1 }], + remediation: + "Validate the resolved destination before writing.", + provenance: { + source: "local_plugin", + candidateId: "archive-entry", + }, + }, + ], + coverage: { + completeness: "partial", + surfaces: [], + explicitExclusions: [], + deferred: [ + { + id: "dependency", + reason: "A dependency remains unreviewed.", + }, + ], + }, + }; + const seeded = JSON.parse( + execFileSync( + "python3", + [ + fileURLToPath( + new URL( + "./fixtures/selected-deep-scan.py", + import.meta.url, + ), + ), + ], + { + input: JSON.stringify({ + scanId, + scanDir, + database: join(stateDir, "workbench.sqlite3"), + draft, + terminalReason: loseCompletionResponse + ? "capped" + : "saturated", + }), + encoding: "utf8", + env: environment, + }, + ), + ); + // Exercise the dedicated function bridge, without extending CLI arguments. + const selectionOutput = execFileSync( + "python3", + [ + "-c", + "import runpy, sys; script = sys.argv.pop(1); runpy.run_path(script)['main'](select_finalization=True)", + join(PLUGIN_ROOT, "scripts/workbench_db.py"), + "finish-deep-scan", + "--scan-id", + scanId, + "--coordinator-generation", + "2", + "--terminal-reason", + loseCompletionResponse ? "capped" : "saturated", + "--manifest-path", + join(scanDir, "scan-manifest.json"), + ], + { + input: JSON.stringify({ + resultPath: seeded.resultPath, + }), + encoding: "utf8", + env: environment, + }, + ); + originalFinalizationInput = + JSON.parse(selectionOutput).deepScan.finalizationInput; + selectedPath = join( + scanDir, + (originalFinalizationInput as { resultPath: string }) + .resultPath, + ); + selectedBytes = await readFile(selectedPath); + const sessions = join( + codexHome, + "sessions", + "2026", + "01", + "01", + ); + await mkdir(sessions, { recursive: true }); + await writeFile( + join(sessions, `rollout-${threadId}.jsonl`), + JSON.stringify({ + timestamp: new Date().toISOString(), + type: "session_meta", + payload: { id: threadId, cwd: scanDir }, + }) + "\n", + ); + if (outcome === "canceled-before-publication") + cancellation.abort("Synthetic user cancellation"); + if (outcome === "completed") { + yield { + type: "turn.completed", + usage: { + input_tokens: 0, + cached_input_tokens: 0, + cache_write_input_tokens: 0, + reasoning_output_tokens: 0, + output_tokens: 0, + }, + }; + } else { + throw parentError; + } + } + return { events: events() }; + }, + }; + return { + startThread: () => thread, + resumeThread: (id: string) => { + expect(id).toBe(threadId); + return thread; + }, + }; + }, + }, + ); + let client = makeClient(); + // Native usage polling is unref'ed; the transport double has no child process. + const keepAlive = setTimeout(() => {}, 30_000); + try { + if (restart) { + await expect( + client.run(repository, { + mode: "deep", + postScanPrompt: resumedStop ? undefined : followUp, + ...(budgeted ? { maxCostUsd: 0.004 } : {}), + }), + ).rejects.toThrow("Synthetic publication write failure"); + const pending = await runWorkbench(workbenchOptions!, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + threadId, + ]); + expect(pending["deepScan"]).toMatchObject({ + status: "running", + terminalReason: "saturated", + }); + expect(commands).not.toContain("fail-scan"); + await client.close(); + client = makeClient(); + if (initialResumeUsage) await recordBudgetUsage(); + } + if (budgeted || closed) { + const running = client.run(repository, { + mode: "deep", + signal: cancellation.signal, + ...(budgeted ? { maxCostUsd: 0.004 } : {}), + ...(resumedStop ? { resumeScanId: scanId, outputDir: scanDir } : {}), + ...(unpricedUsage + ? { onCost: (cost) => reportedCosts.push(cost.estimatedUsd) } + : {}), + postScanPrompt: followUp, + }); + if (budgeted) { + if ( + budgetCompletionFault === "before-commit" || + budgetCompletionFault === "lost-and-canceled" + ) { + await expect(running).rejects.toBeInstanceOf( + ScanCostLimitExceededError, + ); + expect( + commands.filter( + (command) => command === "complete-budget-exhausted-scan", + ), + ).toHaveLength(1); + expect(commands).not.toContain("complete-scan"); + expect(modelInputs).toHaveLength(1); + const saved = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + expect(saved["scan"]).toMatchObject({ + progress: { + status: + budgetCompletionFault === "before-commit" + ? "failed" + : "complete", + }, + findingCount: 1, + reportAvailable: true, + }); + if (budgetCompletionFault === "lost-and-canceled") { + expect( + await Promise.all( + ["report.md", "scan-manifest.json"].map((name) => + readFile(join(scanDir, name)), + ), + ), + ).toEqual(completedArtifacts); + } + return; + } + const result = await running; + expect(result.coverage.completeness).toBe("partial"); + expect(JSON.stringify(result.coverage)).toContain("cost limit"); + if (unpricedUsage) { + expect(result.cost).toBeNull(); + expect(reportedCosts).toEqual([]); + const saved = await runWorkbench(workbenchOptions!, [ + "get-scan", + "--scan-id", + scanId, + ]); + expect( + (saved["scan"] as { cost?: unknown }).cost ?? null, + ).toBeNull(); + } else expect(result.cost?.estimatedUsd).toBeGreaterThan(0.004); + expect(result.threadId).toBe(threadId); + expect(modelInputs).toHaveLength(1); + expect(commands).toContain("complete-budget-exhausted-scan"); + expect(commands).not.toContain("fail-scan"); + expect( + commands.filter( + (command) => command === "complete-budget-exhausted-scan", + ), + ).toHaveLength(1); + if (budgetCompletionFault === "lost") { + expect( + commands.filter((command) => command === "complete-scan"), + ).toHaveLength(1); + expect(result.findings.findings[0]?.remediation).toBe( + "Validate the resolved destination before writing.", + ); + const saved = await runWorkbench(workbenchOptions!, [ + "get-scan", + "--scan-id", + scanId, + ]); + expect(saved["scan"]).toMatchObject({ + progress: { status: "complete" }, + findingCount: 1, + reportAvailable: true, + }); + } + const completed = await runWorkbench(workbenchOptions!, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + threadId, + ]); + expect(completed["deepScan"]).toMatchObject({ + status: "succeeded", + finalizationInput: { terminalReason: "saturated" }, + }); + return; + } + await expect(running).rejects.toThrow(/closed/); + await closePromise; + const stopped = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + const deep = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-deep-scan", "--scan-id", scanId, "--thread-id", threadId], + ); + expect(stopped["scan"]).toMatchObject({ + progress: { status: "failed" }, + findingCount: 1, + reportAvailable: true, + }); + expect(deep["deepScan"]).toMatchObject({ + status: "failed", + finalizationInput: { terminalReason: "saturated" }, + }); + expect(await readFile(join(scanDir, "report.md"), "utf8")).toContain( + "Validate the resolved destination", + ); + expect( + JSON.parse(await readFile(join(scanDir, "coverage.json"), "utf8")) + .completeness, + ).toBe("partial"); + expect(commands).toContain("fail-scan"); + expect(modelInputs).toHaveLength(1); + await client.close(); + client = makeClient(); + await expect( + client.run(repository, { + mode: "deep", + resumeScanId: scanId, + outputDir: scanDir, + }), + ).rejects.toThrow(); + expect(modelInputs).toHaveLength(1); + return; + } + if (outcome.startsWith("canceled-")) { + const error = await client + .run(repository, { + mode: "deep", + signal: cancellation.signal, + ...(resumedStop + ? { resumeScanId: scanId, outputDir: scanDir } + : {}), + postScanPrompt: followUp, + ...(initialResumeUsage ? { maxCostUsd: 0.004 } : {}), + }) + .catch((error: unknown) => error); + if (initialResumeUsage) { + expect(error).toBeInstanceOf(ScanCostLimitExceededError); + expect(error).toBe(originalResumeSignal?.reason); + } else { + expect(error).toBeInstanceOf(ScanInterruptedError); + expect((error as ScanInterruptedError).cause).toBe( + outcome === "canceled-before-publication" + ? parentError + : cancellation.signal.reason, + ); + } + expect(await readFile(selectedPath)).toEqual(selectedBytes!); + if (cancellationReadLost) { + expect(lostCancellationDeepState).toMatchObject({ + status: "running", + finalizationInput: { terminalReason: "saturated" }, + }); + } + expect(originalFinalizationInput).toMatchObject({ + terminalReason: "saturated", + }); + const stopped = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + const deep = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-deep-scan", "--scan-id", scanId, "--thread-id", threadId], + ); + expect(stopped["scan"]).toMatchObject({ + progress: { status: "canceled" }, + findingCount: 1, + reportAvailable: true, + }); + expect( + JSON.parse(await readFile(join(scanDir, "coverage.json"), "utf8")) + .completeness, + ).toBe("partial"); + expect(await readFile(join(scanDir, "report.md"), "utf8")).toContain( + "Validate the resolved destination", + ); + expect(deep["deepScan"]).toMatchObject({ + status: "canceled", + }); + expect( + (deep["deepScan"] as Record)["finalizationInput"], + ).toEqual(originalFinalizationInput); + expect( + commands.filter((command) => command === "cancel-scan"), + ).toHaveLength(1); + expect(commands).not.toContain("fail-scan"); + expect(modelInputs.length).toBe(1); + if (cancellationFault === "cancel-response") { + expect( + await Promise.all( + ["report.md", "scan-manifest.json"].map((name) => + readFile(join(scanDir, name)), + ), + ), + ).toEqual(completedArtifacts); + } + await expect( + client.run(repository, { + mode: "deep", + resumeScanId: scanId, + outputDir: scanDir, + }), + ).rejects.toThrow(); + expect(modelInputs.length).toBe(1); + return; + } + if (canceledFollowUp) { + await expect( + client.run(repository, { + mode: "deep", + signal: cancellation.signal, + postScanPrompt: followUp, + }), + ).rejects.toThrow(/interrupted/); + const completed = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + expect(completed["scan"]).toMatchObject({ + progress: { status: "complete" }, + findingCount: 1, + reportAvailable: true, + }); + expect(modelInputs.length).toBe(2); + expect(modelInputs[1]).toBe(followUp); + expect( + commands.filter((command) => command === "complete-scan"), + ).toHaveLength(loseCompletionResponse ? 2 : 1); + expect(commands).not.toContain("cancel-scan"); + expect(commands).not.toContain("fail-scan"); + expect(await readFile(join(scanDir, "report.md"), "utf8")).toBe( + acceptedReport, + ); + return; + } + if (outcome === "completion-before-commit-fails") { + await expect(client.run(repository, { mode: "deep" })).rejects.toThrow( + "Synthetic completion failure before commit", + ); + expect( + commands.filter((command) => command === "complete-scan"), + ).toHaveLength(1); + expect(commands).not.toContain("fail-scan"); + expect(modelInputs).toHaveLength(1); + const saved = await runWorkbench(workbenchOptions!, [ + "get-scan", + "--scan-id", + scanId, + ]); + expect(saved["scan"]).toMatchObject({ + progress: { status: "running" }, + }); + return; + } + const result = await client.run(repository, { + mode: "deep", + signal: cancellation.signal, + postScanPrompt: + outcome === "published-before-cancellation" ? undefined : followUp, + ...(restart ? { resumeScanId: scanId, outputDir: scanDir } : {}), + }); + expect(result.threadId).toBe(threadId); + if (outcome === "lost-completion-response") { + expect( + commands.filter((command) => command === "complete-scan"), + ).toHaveLength(2); + } + // The synthetic accepted workers have no native usage receipts. + expect(result.cost).toBeNull(); + expect(result.coverage.completeness).toBe("partial"); + expect(result.findings.findings[0]?.remediation).toBe( + "Validate the resolved destination before writing.", + ); + // postScanPrompt retains its existing behavior on each caller invocation. + expect(modelInputs.filter((input) => input !== followUp).length).toBe(1); + expect(modelInputs.filter((input) => input === followUp).length).toBe( + outcome === "published-before-cancellation" ? 0 : restart ? 2 : 1, + ); + const completed = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + expect(completed["scan"]).toMatchObject({ + progress: { status: "complete" }, + }); + expect(await readFile(join(scanDir, "report.md"), "utf8")).toContain( + "Validate the resolved destination", + ); + expect(commands).not.toContain("fail-scan"); + } finally { + clearTimeout(keepAlive); + await client.close(); + } + }; + test(name, runCase, 30_000); +} diff --git a/sdk/typescript/tests-ts/deep-scan-coverage.test.ts b/sdk/typescript/tests-ts/deep-scan-coverage.test.ts new file mode 100644 index 000000000..4d4996bad --- /dev/null +++ b/sdk/typescript/tests-ts/deep-scan-coverage.test.ts @@ -0,0 +1,114 @@ +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { expect, test } from "bun:test"; +import { main } from "../src/cli.js"; +import { loadContract } from "../src/contract.js"; +import { ScanResult } from "../src/result.js"; +import { capture, dependencies } from "./cli-fixtures.js"; + +const fixtureUrl = new URL( + "../../../plugins/codex-security/mcp-app/tests/deep_scan_coverage_fixture.mjs", + import.meta.url, +); + +test.each([ + ["partial", false, false], + ["unknown", false, false], + ["complete", false, false], + ["partial", true, false], + ["partial", true, true], +] as const)( + "publishes %s source coverage through CLI results (resume: %p, continued: %p)", + async (completeness, resume, continueAfterResume) => { + const root = await mkdtemp(join(tmpdir(), "deep-coverage-publication-")); + try { + await mkdir(join(root, "fixture"), { mode: 0o700 }); + // Keep the real workbench outside other suites' persistent module mocks. + const child = Bun.spawn( + [ + Bun.which("node")!, + fileURLToPath(fixtureUrl), + join(root, "fixture"), + completeness, + String(resume), + String(continueAfterResume), + ], + { stdout: "pipe", stderr: "pipe" }, + ); + const [output, errors, exitCode] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + expect(exitCode, errors).toBe(0); + const { scanDir, threadId, terminal } = JSON.parse(output); + const contract = await loadContract(scanDir, { + pluginRoot: fileURLToPath( + new URL("../../../plugins/codex-security/", import.meta.url), + ), + }); + const result = new ScanResult({ + ...contract, + scanDir, + threadId, + turnResult: { status: "completed" }, + }); + expect(result.coverage.completeness).toBe(completeness); + const coverage = JSON.parse( + await readFile(join(scanDir, "coverage.json"), "utf8"), + ); + expect(coverage.reviews[0].attempt).toBe(2); + const report = await readFile(join(scanDir, "report.md"), "utf8"); + expect(report).toContain(`| Coverage | ${completeness} |`); + expect(coverage.explicitExclusions).toHaveLength(coverage.reviews.length); + for (const review of coverage.reviews) + expect(report).toContain(review.workerId); + if (completeness === "partial") { + expect( + coverage.deferred.map((item: { reason: string }) => item.reason), + ).toEqual(["Verify entry boundaries.", "Verify symbolic links."]); + expect( + new Set(coverage.deferred.map((item: { id: string }) => item.id)) + .size, + ).toBe(2); + expect( + coverage.reviews.map( + (review: { completeness: string }) => review.completeness, + ), + ).toEqual(["partial", "complete", "unknown"]); + for (const item of coverage.deferred) { + expect(item.provenance.candidateId).toBe("candidate-1"); + expect(report).toContain(item.reason); + expect( + coverage.surfaces.some((surface: { id: string }) => + item.surfaceIds.includes(surface.id), + ), + ).toBe(true); + } + } + for (const surface of coverage.surfaces) { + expect( + await readFile(join(scanDir, surface.receiptRefs[0]), "utf8"), + ).toContain("Synthetic review evidence."); + } + const stdout = capture(); + const stderr = capture(); + expect( + await main( + ["scan", "--mode", "deep", "--json"], + stdout.stream, + stderr.stream, + dependencies({ result, onWorkbench: () => ({ deepScan: terminal }) }), + ), + ).toBe(completeness === "complete" ? 0 : 2); + expect(JSON.parse(stdout.text()).coverage).toEqual(coverage); + if (completeness !== "complete") + expect(stderr.text()).toContain("STOPPED"); + } finally { + await rm(root, { recursive: true, force: true }); + } + }, + 60_000, +); diff --git a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts index 1df7f704a..75c7427e3 100644 --- a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts +++ b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts @@ -34,12 +34,13 @@ const deepScanOwnershipProbe = [ "connection.executescript('''", "CREATE TABLE workspaces (id TEXT PRIMARY KEY, thread_id TEXT, updated_at TEXT);", "CREATE TABLE scans (id TEXT PRIMARY KEY, workspace_id TEXT, mode TEXT, status TEXT, recipe_json TEXT, handoff_status TEXT, handoff_claim_token TEXT, deep_scan_owner_thread_id TEXT, updated_at TEXT);", - "CREATE TABLE deep_scan_runs (scan_id TEXT PRIMARY KEY);", + "CREATE TABLE deep_scan_runs (scan_id TEXT PRIMARY KEY, schema_version INTEGER NOT NULL DEFAULT 1, workflow_version TEXT NOT NULL DEFAULT 'deep-scan-mcp/v1');", + "CREATE TABLE deep_scan_attempts (scan_id TEXT NOT NULL);", "''')", "scan_id = '11111111-1111-4111-8111-111111111111'", "connection.execute(\"INSERT INTO workspaces VALUES ('workspace', NULL, 'before')\")", "connection.execute(\"INSERT INTO scans VALUES (?, 'workspace', 'deep', 'running', '{}', 'delivered', ?, NULL, 'before')\", (scan_id, case['storedToken']))", - "connection.execute('INSERT INTO deep_scan_runs VALUES (?)', (scan_id,))", + "connection.execute('INSERT INTO deep_scan_runs (scan_id) VALUES (?)', (scan_id,))", "connection.commit()", "if case.get('mutation') == 'rotate':", " connection.executescript(\"CREATE TRIGGER rotate_claim BEFORE UPDATE OF thread_id ON workspaces BEGIN UPDATE scans SET handoff_claim_token = '33333333-3333-4333-8333-333333333333' WHERE workspace_id = NEW.id; END\")", @@ -642,6 +643,14 @@ describe("deep scan workbench ownership", () => { ]); const scanId = registration["scanId"] as string; const targetId = registration["targetId"] as string; + const snapshotDigest = ( + registration["contract"] as { + target: { requiredSnapshotDigest: string }; + } + ).target.requiredSnapshotDigest; + expect(snapshotDigest).toMatch( + /^codex-security-snapshot\/v1:sha256:[0-9a-f]{64}$/, + ); command([ "begin-deep-scan", "--scan-id", @@ -753,6 +762,7 @@ describe("deep scan workbench ownership", () => { kind: "directory_snapshot", targetId, displayName: "repository", + snapshotDigest, }, scope: { limitations: [], validationMode: "incomplete" }, }, @@ -812,6 +822,10 @@ describe("deep scan workbench ownership", () => { }; expect(scan.progress.status).toBe("complete"); expect(scan.warnings).toContain(warning); + const manifest = JSON.parse( + await readFile(join(scanDir, "scan-manifest.json"), "utf8"), + ) as { scan: { target: { snapshotDigest: string } } }; + expect(manifest.scan.target.snapshotDigest).toBe(snapshotDigest); const findings = JSON.parse( await readFile(join(scanDir, "findings.json"), "utf8"), ) as { findings: unknown[] }; diff --git a/sdk/typescript/tests-ts/deep-scan-worker-shutdown.test.ts b/sdk/typescript/tests-ts/deep-scan-worker-shutdown.test.ts index c23adfb4d..a2a74d36d 100644 --- a/sdk/typescript/tests-ts/deep-scan-worker-shutdown.test.ts +++ b/sdk/typescript/tests-ts/deep-scan-worker-shutdown.test.ts @@ -31,6 +31,18 @@ async function bundledWorkerExecutor( if (source === undefined) { throw new Error("Bundled Deep Scan worker executor was not found."); } + const sessionSource = + /\n\/\/ [^\n]*\/codex-session\.ts\n([\s\S]*?)(?=\n\/\/)/u.exec( + runtime, + )?.[1]; + expect(sessionSource).toBeDefined(); + const recordFunction = /\b(isRecord\d*)\(/u.exec(source)?.[1]; + expect(recordFunction).toBeDefined(); + const recordSource = new RegExp( + `function ${recordFunction}\\([^\\n]*\\) \\{[\\s\\S]*?\\n\\}`, + "u", + ).exec(runtime)?.[0]; + expect(recordSource).toBeDefined(); const fileSystemImport = /\b(import_node_fs\d*)\.promises\.readFile\(/u.exec( source, )?.[1]; @@ -54,7 +66,8 @@ async function bundledWorkerExecutor( "workerPermissionProfile", "workerPermissionProfileConfigOverrides", "snapshotWorkerEnvironment", - "workerReasoningSummary", + "workerModelConfig", + "workerModelSelection", "environmentVariable", "preflightDeepScanWorkerPermissionProfile", "DEEP_SCAN_WORKER_PERMISSION_PROFILE_ID", @@ -64,14 +77,15 @@ async function bundledWorkerExecutor( "workerSubagentConfig", "appendSafeItemDiagnostic", "classifyCodexWorkerError", - `${source}\nreturn CodexSdkWorkerExecutor;`, + `${sessionSource}\n${recordSource}\n${source}\nreturn CodexSdkWorkerExecutor;`, )( FakeCodex, { promises: { readFile: async () => "fixture worker prompt" } }, () => ({}), () => [], async () => ({}), - async () => undefined, + async () => ({}), + () => ({}), () => undefined, preflight, "codex_security_deep_scan_worker", diff --git a/sdk/typescript/tests-ts/fixtures/selected-deep-scan.py b/sdk/typescript/tests-ts/fixtures/selected-deep-scan.py new file mode 100644 index 000000000..6f9497f28 --- /dev/null +++ b/sdk/typescript/tests-ts/fixtures/selected-deep-scan.py @@ -0,0 +1,80 @@ +"""Synthetic accepted workers for the installed finalization/SDK contract test.""" +import hashlib +import json +import sqlite3 +import sys +import uuid +from pathlib import Path + +payload = json.load(sys.stdin) +scan_id = payload["scanId"] +scan_dir = Path(payload["scanDir"]) +with sqlite3.connect(payload["database"]) as connection: + connection.execute("PRAGMA foreign_keys = ON") + timestamp = connection.execute( + "SELECT created_at FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) + ).fetchone()[0] + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "coordinator_generation = 2, phase = 'reducing', " + "discovery_runs_dispatched = 2, completion_sequence = 2, " + "consecutive_no_new = 2, stop_after_no_new = 2, max_discovery_runs = 2 " + "WHERE scan_id = ?", (scan_id,), + ) + discoveries = [] + for kind, label in [("discovery", "review-1"), ("discovery", "review-2"), ("dedup", "merge-1")]: + draft = dict(payload["draft"]) + if kind == "dedup": + draft["sourceCoverage"] = draft.pop("coverage") + encoded = json.dumps(draft).encode() + digest = hashlib.sha256(encoded).hexdigest() + worker_id = str(uuid.uuid4()) + output = scan_dir / "artifacts" / "deep_discovery" / label / "output" + output.mkdir(parents=True) + prompt = output.parent / "prompt.md" + prompt.write_text("Synthetic accepted audit\n") + accepted = output / "checkpoints" / f"{digest}.json" + accepted.parent.mkdir() + accepted.write_bytes(encoded) + result = output / "result.json" + result.write_bytes(encoded) + sequence = len(discoveries) + 1 if kind == "discovery" else None + connection.execute( + "INSERT INTO deep_scan_workers " + "(id, scan_id, kind, status, merge_state, prompt_path, artifact_dir, " + "result_manifest_path, attempt, completion_sequence, created_at, updated_at, completed_at) " + "VALUES (?, ?, ?, 'succeeded', ?, ?, ?, ?, 1, ?, ?, ?, ?)", + (worker_id, scan_id, kind, "merged" if kind == "discovery" else "none", + str(prompt), str(output), str(result), sequence, timestamp, timestamp, timestamp), + ) + connection.execute( + "INSERT INTO deep_scan_attempts " + "(scan_id, worker_id, attempt, status, started_at, completed_at, accepted_result_path, accepted_result_sha256) " + "VALUES (?, ?, 1, 'succeeded', ?, ?, ?, ?)", + (scan_id, worker_id, timestamp, timestamp, str(accepted), digest), + ) + if kind == "discovery": + discoveries.append(worker_id) + else: + for order, discovery in enumerate(discoveries): + connection.execute( + "INSERT INTO deep_scan_dedup_inputs " + "(scan_id, dedup_worker_id, discovery_worker_id, input_order) VALUES (?, ?, ?, ?)", + (scan_id, worker_id, discovery, order), + ) + # The committed immutable reference survives loss of the replaceable output. + result.unlink() + # A prior writer selected this aggregate before the public reader resumed it. + selection = { + "version": 1, + "resultPath": accepted.relative_to(scan_dir).as_posix(), + "resultSha256": digest, + "terminalReason": payload["terminalReason"], + "omittedWorkerIds": [], + "selectedAt": timestamp, + } + connection.execute( + "UPDATE deep_scan_runs SET finalization_input_json = ?, terminal_reason = ? WHERE scan_id = ?", + (json.dumps(selection), payload["terminalReason"], scan_id), + ) +print(json.dumps({"resultPath": str(result), "acceptedPath": str(accepted)})) diff --git a/sdk/typescript/tests-ts/reasoning-summary.test.ts b/sdk/typescript/tests-ts/reasoning-summary.test.ts new file mode 100644 index 000000000..854ee85ea --- /dev/null +++ b/sdk/typescript/tests-ts/reasoning-summary.test.ts @@ -0,0 +1,82 @@ +import { execFileSync } from "node:child_process"; +import { + mkdir, + mkdtemp, + readFile, + realpath, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { expect, test } from "bun:test"; +import { captureOriginalReasoningSummary } from "../src/reasoning-summary.js"; +import type { JsonObject } from "../src/config.js"; + +test.each([ + { model_reasoning_summary: "none" }, + { model_reasoning_summary: "concise" }, + { model_reasoning_summary: null }, + { model_reasoning_summary: "" }, + { + profile: "selected", + profiles: { selected: { model_reasoning_summary: "auto" } }, + }, +] as JsonObject[])( + "preserves explicit summary without a native lookup: %j", + async (config) => { + expect( + await captureOriginalReasoningSummary({ + config, + command: { command: "unused-native-executable" }, + cwd: tmpdir(), + environment: {}, + signal: new AbortController().signal, + }), + ).toBeUndefined(); + }, +); + +test.each(["unknown-model", "unsupported-command", "missing-metadata"])( + "keeps an unavailable original model default unknown: %s", + async (scenario) => { + const root = await mkdtemp(join(tmpdir(), "summary-selection-")); + try { + const cwd = join(root, "original output"); + const script = join(root, "native.mjs"); + const receipt = join(root, "receipt.json"); + await mkdir(cwd); + await writeFile( + script, + [ + 'import { writeFileSync } from "node:fs";', + `writeFileSync(${JSON.stringify(receipt)}, JSON.stringify({cwd:process.cwd(),args:process.argv,home:process.env.CODEX_HOME}));`, + `console.log(JSON.stringify({models:[{slug:${JSON.stringify(scenario === "unknown-model" ? "another-model" : "selected-model")}${scenario === "missing-metadata" ? "" : ',default_reasoning_summary:"none"'}}]}));`, + `process.exit(${scenario === "unsupported-command" ? 1 : 0});`, + ].join("\n"), + ); + const command = execFileSync("node", ["-p", "process.execPath"], { + encoding: "utf8", + }).trim(); + const value = await captureOriginalReasoningSummary({ + config: { model: "selected-model", model_reasoning_effort: "high" }, + command: { command }, + cwd, + environment: { + CODEX_HOME: root, + NODE_OPTIONS: `--import=${pathToFileURL(script).href}`, + }, + signal: new AbortController().signal, + }); + expect(value).toBeUndefined(); + const recorded = JSON.parse(await readFile(receipt, "utf8")); + expect(await realpath(recorded.cwd)).toBe(await realpath(cwd)); + expect(recorded.home).toBe(root); + expect(recorded.args).toContain('model="selected-model"'); + expect(recorded.args).toContain('model_reasoning_effort="high"'); + } finally { + await rm(root, { recursive: true, force: true }); + } + }, +); diff --git a/sdk/typescript/tests-ts/scan-resume.test.ts b/sdk/typescript/tests-ts/scan-resume.test.ts index 876d0b2ed..741cced07 100644 --- a/sdk/typescript/tests-ts/scan-resume.test.ts +++ b/sdk/typescript/tests-ts/scan-resume.test.ts @@ -158,10 +158,16 @@ async function interruptedScan( const sessionPath = join(codexHome, "sessions", `rollout-${threadId}.jsonl`); await writeFile( sessionPath, - JSON.stringify({ - type: "session_meta", - payload: { id: threadId, cwd: scanDir }, - }) + "\n", + [ + { type: "session_meta", payload: { id: threadId, cwd: scanDir } }, + { + type: "turn_context", + timestamp: new Date().toISOString(), + payload: { turn_id: "synthetic-scan-turn", model: "gpt-5.6-sol" }, + }, + ] + .map((event) => JSON.stringify(event)) + .join("\n") + "\n", ); if (mode === "deep") { await command([ @@ -488,6 +494,7 @@ test.each([ f.sessionPath, JSON.stringify({ type: "event_msg", + timestamp: new Date().toISOString(), payload: { type: "token_count", info: { @@ -613,6 +620,83 @@ test.each([ }, ); +test("reader resumes a saved budget selection without model work", async () => { + const f = await interruptedScan(); + await finishDiscovery(f); + const selection = JSON.stringify({ + version: 1, + resultPath: null, + resultSha256: null, + terminalReason: "capped", + omittedWorkerIds: [], + selectedAt: "2000-01-01T00:00:00Z", + }); + // These committed facts came from the later writer before process loss. + const prepared = Bun.spawnSync( + [ + f.python, + "-I", + "-B", + "-c", + "import sqlite3,sys; c=sqlite3.connect(sys.argv[1]); c.execute(\"UPDATE deep_scan_runs SET workflow_version='deep-security-scan/v2', cancel_requested=1, finalization_input_json=? WHERE scan_id=?\", (sys.argv[3],sys.argv[2])); c.commit()", + join(f.environment.CODEX_SECURITY_STATE_DIR, "workbench.sqlite3"), + f.scanId, + selection, + ], + { stdout: "pipe", stderr: "pipe" }, + ); + expect(prepared.exitCode, new TextDecoder().decode(prepared.stderr)).toBe(0); + const stdout = capture(); + const stderr = capture(); + const code = await main( + ["scans", "resume", f.scanId, "--json"], + stdout.stream, + stderr.stream, + { + ...dependencies({ environment: f.environment, currentDirectory: f.root }), + runWorkbench: f.command, + createSecurity: resumeClient(f, () => ({ + startThread() { + throw new Error("Saved publication must not start a new session"); + }, + resumeThread(threadId) { + expect(threadId).toBe(f.threadId); + return { + id: threadId, + async runStreamed() { + throw new Error( + "Saved publication must not run another model turn", + ); + }, + }; + }, + })), + }, + ); + expect(code, stderr.text()).toBe(2); + expect(stdout.text(), stderr.text()).not.toBe(""); + const result = JSON.parse(stdout.text()); + expect(result.manifest.scan.id).toBe(f.scanId); + expect(result.manifest.scan.sealedAt).toBeString(); + expect(result.coverage.completeness).toBe("partial"); + const conserved = Bun.spawnSync( + [ + f.python, + "-I", + "-B", + "-c", + "import sqlite3,sys; c=sqlite3.connect(sys.argv[1]); assert c.execute('SELECT finalization_input_json FROM deep_scan_runs WHERE scan_id=?',(sys.argv[2],)).fetchone()==(sys.argv[3],); assert c.execute('SELECT COUNT(*) FROM deep_scan_attempts').fetchone()==(0,); assert c.execute('SELECT COUNT(*) FROM deep_scan_attempt_sessions').fetchone()==(0,)", + join(f.environment.CODEX_SECURITY_STATE_DIR, "workbench.sqlite3"), + f.scanId, + selection, + ], + { stdout: "pipe", stderr: "pipe" }, + ); + expect(conserved.exitCode, new TextDecoder().decode(conserved.stderr)).toBe( + 0, + ); +}); + test.each([ "single", "bulk", diff --git a/sdk/typescript/tests-ts/scan-usage-reconciliation.test.ts b/sdk/typescript/tests-ts/scan-usage-reconciliation.test.ts new file mode 100644 index 000000000..6265ebf0b --- /dev/null +++ b/sdk/typescript/tests-ts/scan-usage-reconciliation.test.ts @@ -0,0 +1,601 @@ +import { describe, expect, test } from "bun:test"; +import { appendFile, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { estimateScanCost } from "../src/cost-model.js"; +import { ScanCostTracker } from "../src/cost.js"; +import { readScanLogs } from "../src/scan-logs.js"; +import type { ScanExecutionAttribution } from "../src/scan-sessions.js"; + +describe("scan usage reconciliation", () => { + test("SDK usage and logs share attempt membership and the original owner turn", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-attribution-")); + const observed: unknown[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + onSessionEvent: (event) => observed.push(event), + }); + const at = (second: number) => + `2026-09-01T00:00:${String(second).padStart(2, "0")}Z`; + const attribution: ScanExecutionAttribution = { + formatVersion: 1, + executionThreadIds: ["old-worker", "replacement-worker"], + owner: { threadId: "parent", turnId: "scan-turn", startedAt: at(1) }, + startedAt: at(1), + completedAt: at(10), + }; + const token = (second: number, count: number) => ({ + timestamp: at(second), + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: { input_tokens: count, output_tokens: 0 } }, + }, + }); + const context = (second: number, turn: string) => ({ + timestamp: at(second), + type: "turn_context", + payload: { turn_id: turn, model: "gpt-5.6-sol" }, + }); + try { + await mkdir(join(home, "sessions")); + for (const [id, parent, events] of [ + [ + "parent", + null, + [ + context(0, "prior-turn"), + token(0, 100), + context(1, "scan-turn"), + token(2, 110), + context(3, "unrelated-turn"), + token(4, 1010), + ], + ], + [ + "old-worker", + null, + [context(1, "worker-turn"), token(2, 20), token(11, 120)], + ], + ["replacement-worker", null, [context(3, "worker-turn"), token(4, 30)]], + ["worker-child", "old-worker", [context(3, "child-turn"), token(4, 5)]], + ["unrelated-child", "parent", [context(3, "side-turn"), token(4, 900)]], + ] as const) { + const records = [ + { + type: "session_meta", + payload: { id, ...(parent ? { parent_thread_id: parent } : {}) }, + }, + ...events, + ]; + await writeFile( + join(home, "sessions", `${id}.jsonl`), + records.map((value) => JSON.stringify(value)).join("\n") + "\n", + ); + } + tracker.setAttributionReader(async () => attribution); + tracker.start("parent"); + const snapshot = await tracker.stop(); + expect(snapshot.cost?.inputTokens).toBe(65); + expect(JSON.stringify(observed)).not.toContain("unrelated-turn"); + expect(JSON.stringify(observed)).not.toContain(at(11)); + const logs = await readScanLogs({ + scanId: "scan", + threadId: "parent", + codexHome: home, + executionAttribution: attribution, + }); + expect(logs.sessions.map((session) => session.threadId).sort()).toEqual([ + "old-worker", + "parent", + "replacement-worker", + "worker-child", + ]); + expect( + logs.events.some(({ event }) => + JSON.stringify(event).includes("unrelated-turn"), + ), + ).toBe(false); + expect( + logs.events.some(({ event }) => JSON.stringify(event).includes(at(11))), + ).toBe(false); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); + + test("waits for attribution and retains uncertainty until missing attempt usage arrives", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-delayed-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + let attribution: ScanExecutionAttribution | null = null; + tracker.setAttributionReader(async () => attribution); + const at = "2026-09-01T00:00:02Z"; + const records = (id: string, count: number) => + [ + { type: "session_meta", payload: { id } }, + { + timestamp: at, + type: "turn_context", + payload: { model: "gpt-5.6-sol", turn_id: "own" }, + }, + { + timestamp: at, + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { input_tokens: count, output_tokens: 0 }, + }, + }, + }, + ] + .map((event) => JSON.stringify(event)) + .join("\n") + "\n"; + try { + await mkdir(join(home, "sessions")); + await writeFile( + join(home, "sessions", "worker.jsonl"), + records("worker", 20), + ); + tracker.start("worker"); + tracker.recordUsage({ input_tokens: 20, output_tokens: 0 }); + expect((await tracker.refresh()).cost).toBeNull(); + attribution = { + formatVersion: 1, + executionThreadIds: ["worker", "failed-attempt"], + owner: { threadId: "worker", turnId: "own", startedAt: at }, + startedAt: "2026-09-01T00:00:01Z", + completedAt: "2026-09-01T00:00:10Z", + }; + expect((await tracker.refresh()).cost).toMatchObject({ + inputTokens: 20, + coverage: "partial", + }); + await writeFile( + join(home, "sessions", "failed.jsonl"), + records("failed-attempt", 5), + ); + const final = await tracker.stop(); + expect(final.cost?.inputTokens).toBe(25); + expect(final.cost?.coverage).toBeUndefined(); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); + + test("preserves receipt accounting for a resumed legacy Deep scan", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-legacy-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + legacy: true, + executionThreadIds: ["legacy-parent"], + owner: { + threadId: "legacy-parent", + turnId: null, + startedAt: "2026-09-01T00:00:00Z", + }, + startedAt: "2026-09-01T00:00:00Z", + completedAt: null, + })); + try { + tracker.start("legacy-parent"); + const snapshot = await tracker.stop({ + input_tokens: 10000, + output_tokens: 100, + }); + expect(snapshot.cost?.inputTokens).toBe(10000); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); + + test("prices each observed model instead of repricing the sum with the parent", () => { + const usage = { + input_tokens: 200, + output_tokens: 20, + modelUsage: [ + { model: "gpt-5.6-sol", input_tokens: 100, output_tokens: 10 }, + { model: "gpt-6-astra", input_tokens: 100, output_tokens: 10 }, + ], + }; + const expected = + estimateScanCost("gpt-5.6-sol", usage.modelUsage[0])!.estimatedUsd + + estimateScanCost("gpt-6-astra", usage.modelUsage[1])!.estimatedUsd; + expect(estimateScanCost("gpt-5.6-sol", usage)?.estimatedUsd).toBe(expected); + }); + + test("keeps incomplete model attribution unpriced", () => { + expect( + estimateScanCost("gpt-5.6-sol", { + input_tokens: 200, + output_tokens: 20, + modelUsage: [ + { model: "gpt-5.6-sol", input_tokens: 100, output_tokens: 10 }, + { model: null, input_tokens: 100, output_tokens: 10 }, + ], + }), + ).toBeNull(); + }); + + test("reconciles stale and missing cumulative receipts without reducing usage", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-receipts-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + try { + tracker.start("worker"); + tracker.recordUsage({ input_tokens: 160, output_tokens: 0 }); + tracker.recordUsage({ input_tokens: 100, output_tokens: 0 }); + tracker.recordUsage(null); + expect((await tracker.stop()).cost?.inputTokens).toBe(160); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); + + test("tracks per-model deltas within one resumed session", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-models-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + try { + await mkdir(join(home, "sessions")); + const records = [ + { type: "session_meta", payload: { id: "worker" } }, + { + type: "turn_context", + payload: { model: "gpt-5.6-sol", turn_id: "turn-1" }, + }, + { + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { input_tokens: 100, output_tokens: 10 }, + }, + }, + }, + { + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: { input_tokens: 60, output_tokens: 6 } }, + }, + }, + { + type: "turn_context", + payload: { model: "gpt-6-astra", turn_id: "turn-2" }, + }, + { + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { input_tokens: 200, output_tokens: 20 }, + }, + }, + }, + ]; + await writeFile( + join(home, "sessions", "worker.jsonl"), + records.map((record) => JSON.stringify(record)).join("\n") + "\n", + ); + tracker.start("worker"); + tracker.recordUsage({ input_tokens: 200, output_tokens: 20 }); + const snapshot = await tracker.stop(); + expect(snapshot.cost?.inputTokens).toBe(200); + expect(snapshot.cost?.estimatedUsd).toBeCloseTo(0.0021, 12); + expect(snapshot.cost?.modelCosts?.map((cost) => cost.model)).toEqual([ + "gpt-5.6-sol", + "gpt-6-astra", + ]); + tracker.recordUsage({ input_tokens: 250, output_tokens: 25 }); + expect((await tracker.refresh()).cost).toBeNull(); + await appendFile( + join(home, "sessions", "worker.jsonl"), + JSON.stringify({ + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { input_tokens: 250, output_tokens: 25 }, + }, + }, + }) + "\n", + ); + expect((await tracker.refresh()).cost?.estimatedUsd).toBeCloseTo( + 0.00285, + 12, + ); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); +}); + +describe("charged response receipts", () => { + for (const exactOnly of [false, true]) { + test(`counts compaction and deduplicates responses across counter resets (exact only: ${exactOnly})`, async () => { + const home = await mkdtemp(join(tmpdir(), "usage-response-receipts-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + const usage = (input: number, cached: number, output: number) => ({ + input_tokens: input, + cached_input_tokens: cached, + cache_write_input_tokens: 0, + output_tokens: output, + reasoning_output_tokens: 0, + total_tokens: input + output, + }); + const record = ( + id: string, + count: unknown, + cumulative: unknown, + model = "gpt-5.6-sol", + ) => ({ + type: "token_usage_record", + payload: { + thread_id: "worker", + turn_id: "turn", + response_id: id, + model, + usage: count, + thread_token_usage: cumulative, + }, + }); + const first = record("normal-1", usage(100, 80, 10), usage(100, 80, 10)); + const compact = record( + "compaction", + usage(50, 40, 5), + usage(150, 120, 15), + "gpt-6-astra", + ); + const second = record("normal-2", usage(120, 90, 12), usage(120, 90, 12)); + const counter = (count: unknown) => ({ + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: count }, + }, + }); + const events = [ + { type: "session_meta", payload: { id: "worker" } }, + first, + ...(!exactOnly ? [counter(usage(100, 80, 10))] : []), + compact, + { + type: "compacted", + payload: { message: "Synthetic context summary" }, + }, + compact, + second, + ...(!exactOnly ? [counter(usage(220, 170, 22))] : []), + first, + ]; + try { + await mkdir(join(home, "sessions")); + await writeFile( + join(home, "sessions", "worker.jsonl"), + events.map((e) => JSON.stringify(e)).join("\n") + "\n", + ); + tracker.start("worker"); + tracker.recordUsage(usage(220, 170, 22)); + const result = await tracker.stop(); + expect(result.usage).toMatchObject(usage(270, 210, 27)); + expect( + result.cost?.modelCosts?.map((part) => [ + part.model, + part.inputTokens + part.outputTokens, + ]), + ).toEqual([ + ["gpt-5.6-sol", 242], + ["gpt-6-astra", 55], + ]); + expect((await tracker.refresh()).usage).toEqual(result.usage); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); + } + + test("uses receipt turn identity for shared-parent usage and logs", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-response-owner-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + const attribution: ScanExecutionAttribution = { + formatVersion: 1, + executionThreadIds: [], + owner: { + threadId: "parent", + turnId: "scan-turn", + startedAt: "2026-09-01T00:00:01Z", + }, + startedAt: "2026-09-01T00:00:01Z", + completedAt: "2026-09-01T00:00:10Z", + }; + const receipt = ( + id: string, + turn: string, + second: string, + input: number, + ) => ({ + type: "token_usage_record", + timestamp: `2026-09-01T00:00:${second}Z`, + payload: { + response_id: id, + thread_id: "parent", + turn_id: turn, + model: "gpt-5.6-sol", + usage: { input_tokens: input, output_tokens: 0 }, + }, + }); + try { + await mkdir(join(home, "sessions")); + await writeFile( + join(home, "sessions", "parent.jsonl"), + [ + { type: "session_meta", payload: { id: "parent" } }, + receipt("prior", "prior-turn", "00", 900), + receipt("owned", "scan-turn", "02", 20), + receipt("side", "other-turn", "03", 800), + receipt("post", "scan-turn", "11", 700), + ] + .map((e) => JSON.stringify(e)) + .join("\n") + "\n", + ); + tracker.setAttributionReader(async () => attribution); + tracker.start("parent"); + expect((await tracker.stop()).cost?.inputTokens).toBe(20); + const logs = await readScanLogs({ + scanId: "scan", + threadId: "parent", + codexHome: home, + executionAttribution: attribution, + }); + const ids = logs.events + .map( + ({ event }) => + (event as { payload?: Record })["payload"]?.[ + "response_id" + ], + ) + .filter(Boolean); + expect(ids).toEqual(["owned"]); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); +}); + +test("delayed response receipts resolve cumulative gaps without treating smaller counters as stale", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-delayed-response-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + const record = (id: string, tokens: number, cumulative: number) => + JSON.stringify({ + type: "token_usage_record", + payload: { + response_id: id, + thread_id: "worker", + model: "gpt-5.6-sol", + usage: { input_tokens: tokens, output_tokens: 0 }, + thread_token_usage: { input_tokens: cumulative, output_tokens: 0 }, + }, + }) + "\n"; + try { + await mkdir(join(home, "sessions")); + const file = join(home, "sessions", "worker.jsonl"); + await writeFile( + file, + JSON.stringify({ type: "session_meta", payload: { id: "worker" } }) + + "\n" + + record("first", 100, 100) + + record("third", 50, 180), + ); + tracker.start("worker"); + expect((await tracker.refresh()).cost).toMatchObject({ + inputTokens: 150, + coverage: "partial", + }); + await appendFile(file, record("second", 30, 130)); + const result = await tracker.stop(); + expect(result.cost?.inputTokens).toBe(180); + expect(result.cost?.coverage).toBeUndefined(); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } +}); + +test("a reader installed before the native attribution writer preserves legacy receipts", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-reader-first-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + try { + tracker.setAttributionReader(async () => undefined); + tracker.start("parent"); + tracker.recordUsage({ input_tokens: 100, output_tokens: 0 }); + expect((await tracker.stop()).cost?.inputTokens).toBe(100); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } +}); + +test("late exact receipts replace an overlapping legacy counter without adding it twice", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-overlap-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + const usage = (count: number) => ({ input_tokens: count, output_tokens: 0 }); + const receipt = (id: string, count: number, cumulative: number) => ({ + type: "token_usage_record", + payload: { + thread_id: "worker", + response_id: id, + model: "gpt-5.6-sol", + usage: usage(count), + thread_token_usage: usage(cumulative), + }, + }); + try { + await mkdir(join(home, "sessions")); + await writeFile( + join(home, "sessions", "worker.jsonl"), + [ + { type: "session_meta", payload: { id: "worker" } }, + { + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: usage(100) }, + }, + }, + receipt("new", 10, 110), + receipt("old", 100, 100), + { + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: usage(10) }, + }, + }, + ] + .map((e) => JSON.stringify(e)) + .join("\n") + "\n", + ); + tracker.start("worker"); + const result = await tracker.stop(); + expect(result.cost?.inputTokens).toBe(110); + expect(result.cost?.coverage).toBeUndefined(); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } +});