diff --git a/src/core/tools/execute-tool-call.ts b/src/core/tools/execute-tool-call.ts index 0189cdcd..30f056c0 100644 --- a/src/core/tools/execute-tool-call.ts +++ b/src/core/tools/execute-tool-call.ts @@ -108,6 +108,8 @@ export function isStringToolFailureResult(text: string | undefined): boolean { // Prefix-only failure encodings (start of the tool result). if (/^\s*Error:/i.test(head)) return true; if (/^\s*Command failed\b/i.test(head)) return true; + // exec timeout / signal kill are marked structurally via ctx.markResultError + // (builtin exec catch branch) — never by matching stdout text (O96). if (/^\s*Command blocked:/i.test(head)) return true; if (/^\s*Patch rejected\b/i.test(head)) return true; // exec non-zero exit is emitted as the first line: "exit_code: N\n..." @@ -465,7 +467,14 @@ async function executeOneToolCallInner( const attemptSignal = combineAbortSignals(effectiveAbortSignal, timeoutAbortCtrl.signal) ?? effectiveAbortSignal; - const attemptCtx: ToolContext = { ...callToolCtx, abortSignal: attemptSignal }; + let structuralError = false; + const attemptCtx: ToolContext = { + ...callToolCtx, + abortSignal: attemptSignal, + markResultError: () => { + structuralError = true; + }, + }; if (!skipAgentHeartbeat) { const heartbeatIntervalMs = Math.max(1, deps.heartbeatIntervalMs); const maxMissed = resolveMaxMissedHeartbeats( @@ -519,7 +528,7 @@ async function executeOneToolCallInner( // Many tools (exec, edit_file, …) return error as a string instead of // throwing. Detect those so is_error / failure gates / loop-guard // see a real failure (Claude/Codex: non-zero shell is not success). - if (!attemptErrFlag && isStringToolFailureResult(attemptText)) { + if (!attemptErrFlag && (structuralError || isStringToolFailureResult(attemptText))) { attemptErrFlag = true; attemptError = new MossError({ code: ErrorCode.TOOL_EXECUTION_FAILED, diff --git a/src/core/tools/tool-loop-guard.ts b/src/core/tools/tool-loop-guard.ts index 7f42be9f..9c524a38 100644 --- a/src/core/tools/tool-loop-guard.ts +++ b/src/core/tools/tool-loop-guard.ts @@ -93,6 +93,23 @@ const SURGICAL_EDIT_TOOLS = new Set(['edit_file', 'multi_edit', 'apply_patch']); /** Max failed surgical edits per path before forcing re-read (identical input limit still applies). */ const DEFAULT_EDIT_PATH_FAILURE_LIMIT = 3; +/** + * O96: exec / run_tests that finish normally with a non-zero exit (lint/test fail) + * must not burn the global byToolFailure budget. Same-command identical-output + * thrash still hints to change approach via a separate counter. + */ +const VERIFY_SHELL_TOOLS = new Set(['exec', 'run_tests']); +/** Consecutive same-command failures with identical output before hinting to change approach. */ +const DEFAULT_SAME_COMMAND_FAILURE_LIMIT = 3; + +export type ShellFaultKind = 'timeout' | 'start_failure' | 'killed' | 'unusable'; + +export type SameCommandFailureStreak = { + count: number; + outputKey: string; + displayCommand: string; +}; + export type ToolLoopGuardState = { bySignature: Map; byTool: Map; @@ -109,6 +126,18 @@ export type ToolLoopGuardState = { * go stale — Claude Code "read then edit" discipline at the loop layer. */ byEditPathFailure: Map; + /** + * O96: per normalized command signature — consecutive normal non-zero exits + * with substantially identical output. Does not feed byToolFailure. + */ + bySameCommandFailure: Map; + /** O96: last real shell fault kind for exec/run_tests (timeout / start / …). */ + lastShellFaultKind: Map; + /** + * O96: missing command name parsed from a 126/127 start failure's output. + * Copy-only — never used for classification. + */ + lastMissingCommand?: Map; /** * O97: paths whose edit/write was just refused for read scope (partial * window, file content not attached). While locked, an exec that would write @@ -272,6 +301,9 @@ export function createToolLoopGuardState( byToolFailure: new Map(), byWebFetchUrlFailure: new Map(), byEditPathFailure: new Map(), + bySameCommandFailure: new Map(), + lastShellFaultKind: new Map(), + lastMissingCommand: new Map(), readScopeLockedPaths: new Set(), webSearchQueries: new Set(), freshNewsSearchRequested: false, @@ -305,6 +337,145 @@ function normalizeWebFetchUrlKey(input: unknown): string | null { } } +/** + * O96: normalize exec / run_tests command for same-command streak keys. + * @internal exported for tests + */ +export function normalizeVerifyCommandSignature( + toolName: string, + input?: Record +): { key: string; display: string } | null { + if (toolName === 'exec') { + const cmd = typeof input?.command === 'string' ? input.command.trim().replace(/\s+/g, ' ') : ''; + if (!cmd) return null; + return { key: `exec:${cmd.toLowerCase()}`, display: cmd }; + } + if (toolName === 'run_tests') { + const file = typeof input?.file === 'string' && input.file.trim() ? input.file.trim() : ''; + const cmd = + typeof input?.command === 'string' && input.command.trim() ? input.command.trim() : ''; + const display = file ? `node --test ${file}` : cmd || 'npm test'; + return { key: `run_tests:${display.toLowerCase().replace(/\s+/g, ' ')}`, display }; + } + return null; +} + +/** @internal exported for tests */ +export function normalizeShellOutputFingerprint(resultText: string): string { + return resultText + .trim() + .replace(/\r\n/g, '\n') + .replace(/[ \t]+/g, ' ') + .replace(/\n{3,}/g, '\n\n'); +} + +/** + * O96: classify an exec / run_tests error result. + * - `normal_nonzero`: process started and finished; lint/test simply failed. + * - fault kinds: timeout / start failure / killed / unusable empty output. + * - `other`: policy deny, blocked, or unrecognized — keep existing byToolFailure path. + * @internal exported for tests + */ +export function classifyShellToolFailure( + resultText: string | undefined +): 'normal_nonzero' | ShellFaultKind | 'other' { + const text = resultText ?? ''; + const head = text.slice(0, 600); + // Fault markers are only trusted on the tool-generated first line. Child + // stdout (e.g. a test named "... timed out after 5000ms") must never be + // scanned for fault text, or a normal run is misclassified as a tool fault. + const firstLine = (head.replace(/^\s+/, '').split(/\r?\n/, 1)[0] ?? '').trim(); + + // Normal completion header: process ran to completion; stdout is untrusted. + const exitHeader = /^exit_code:\s*(\d+)\b/i.exec(firstLine); + // 126 (not executable) / 127 (command not found) from the exec first line + // mean the process never started — a tool fault, not a lint/test failure. + if (exitHeader) { + const code = Number(exitHeader[1]); + if (code === 126 || code === 127) return 'start_failure'; + return code > 0 ? 'normal_nonzero' : 'other'; + } + if (/^Command failed \(exit\s+12[67]\)/i.test(firstLine)) return 'start_failure'; + if (/^Command failed \(exit\s+[1-9]\d*\)/i.test(firstLine)) return 'normal_nonzero'; + + if ( + /^timed out after\b/i.test(firstLine) || + /^❌\s*timed out\b/i.test(firstLine) || + /^Execution error:.*\btimed out\b/i.test(firstLine) || + /^Tool \S+ timed out\b/i.test(firstLine) + ) { + return 'timeout'; + } + if (/^Command killed by\b/i.test(firstLine)) { + return 'killed'; + } + // "Command failed:" without "(exit N)" — spawn/start failure style from builtin.ts + if (/^Command failed:\s*$/i.test(firstLine)) { + return 'start_failure'; + } + if ( + /^(?:Execution error:|Error:)?.*\bspawn(?:Sync)?\b.*\bENOENT\b/i.test(firstLine) || + /^(?:Execution error:|Error:)\s.*\bENOENT\b/i.test(firstLine) || + /^(?:\S+:\s*)?(?:\S+:\s*)?command not found\b/i.test(firstLine) || + /^.*\bnot found in PATH\b/i.test(firstLine) || + /^(?:Error:\s*)?Failed to (?:start|launch|spawn)\b/i.test(firstLine) || + /^未能启动/.test(firstLine) + ) { + return 'start_failure'; + } + if (/^Execution error:/i.test(firstLine)) { + return 'start_failure'; + } + + // Policy / user deny — not a normal lint non-zero (QA guardrail). + if ( + /^\s*Command blocked:/i.test(head) || + /^\s*已拦截/u.test(head) || + /^\s*Blocked\b/i.test(head) || + /^\s*Tool execution denied\b/i.test(head) || + /^\s*Operation blocked by workspace policy\b/i.test(head) + ) { + return 'other'; + } + + // Normal completed non-zero (lint / test / check failed). + if (/Test Results:\s*❌\s+\d+\s+FAILED\b/i.test(head)) return 'normal_nonzero'; + if (/Test Results:\s*❌\s+exit\s+[1-9]/i.test(head)) return 'normal_nonzero'; + if (/Test Results:\s*⚠️\s*NO TESTS EXECUTED/i.test(head)) return 'normal_nonzero'; + if (/Verify Fix:\s*❌/i.test(head)) return 'normal_nonzero'; + if (/❌\s+\d+\s+FAILED\b/i.test(head)) return 'normal_nonzero'; + + const trimmed = text.trim(); + if (!trimmed || trimmed === '(no output)') { + return 'unusable'; + } + return 'other'; +} + +/** + * O96: best-effort missing command name from shell "not found" output + * (`sh: eslint: not found`, `sh: 1: eslint: not found`, + * `bash: foo: command not found`, `zsh: command not found: foo`, + * `bash: /no/such/bin: No such file or directory`). Copy-only. + * @internal exported for tests + */ +export function parseMissingCommandName(resultText: string | undefined): string | undefined { + const text = (resultText ?? '').slice(0, 4000); + const patterns = [ + /^(?:\S*\/)?(?:sh|bash|dash|ash|ksh)(?::\s*(?:line\s+)?\d+)?:\s*([^\s:]+):\s*(?:command\s+)?not found\b/im, + /^(?:\S*\/)?zsh(?::\d+)?:\s*command not found:\s*(\S+)/im, + /^(?:\S*\/)?(?:sh|bash|dash|ash|ksh)(?::\s*(?:line\s+)?\d+)?:\s*(\/[^\s:]+):\s*No such file or directory\b/im, + ]; + for (const re of patterns) { + const m = re.exec(text); + const name = m?.[1]; + // Display-only: strict whitelist, else fall back to generic copy. + if (name && /^[A-Za-z0-9._/-]{1,64}$/.test(name)) return name; + if (name) return undefined; + } + return undefined; +} + export function isSoftToolFailureResult(resultText: string | undefined): boolean { if (!resultText) return false; if (/^\s*(web_fetch_error|web_search_error)\b/i.test(resultText)) return true; @@ -335,7 +506,56 @@ export function recordToolLoopOutcome( state.hasSufficientRssNewsEvidence = true; } trackReadScopeLocks(state, toolName, isError, resultText, input); - if (!isError && !isSoftToolFailureResult(resultText)) return; + if (!isError && !isSoftToolFailureResult(resultText)) { + // O96: trust is_error only (set by execute-tool-call anchored + // isStringToolFailureResult). Never text-scan successful output. + if (VERIFY_SHELL_TOOLS.has(toolName)) { + const sig = normalizeVerifyCommandSignature(toolName, input); + if (sig) state.bySameCommandFailure.delete(sig.key); + } + return; + } + // O96: normal completed non-zero for exec/run_tests — independent streak only. + if (VERIFY_SHELL_TOOLS.has(toolName)) { + const kind = classifyShellToolFailure(resultText); + if (kind === 'normal_nonzero') { + const sig = normalizeVerifyCommandSignature(toolName, input); + if (sig) { + const outputKey = normalizeShellOutputFingerprint(resultText ?? ''); + const prev = state.bySameCommandFailure.get(sig.key); + if (prev && prev.outputKey === outputKey) { + state.bySameCommandFailure.set(sig.key, { + count: prev.count + 1, + outputKey, + displayCommand: sig.display, + }); + } else { + state.bySameCommandFailure.set(sig.key, { + count: 1, + outputKey, + displayCommand: sig.display, + }); + } + } + // Do not bump byToolFailure — lint/test non-zero is not a broken tool. + return; + } + if ( + kind === 'timeout' || + kind === 'start_failure' || + kind === 'killed' || + kind === 'unusable' + ) { + state.lastShellFaultKind.set(toolName, kind); + const missing = kind === 'start_failure' ? parseMissingCommandName(resultText) : undefined; + state.lastMissingCommand ??= new Map(); + if (missing) state.lastMissingCommand.set(toolName, missing); + else state.lastMissingCommand.delete(toolName); + const sig = normalizeVerifyCommandSignature(toolName, input); + if (sig) state.bySameCommandFailure.delete(sig.key); + } + // Policy deny / other: fall through to byToolFailure (existing path). + } // web_fetch failures are tracked per-URL (see ToolLoopGuardState) so that a // failed host doesn't poison the tool's reputation for unrelated hosts. if (toolName === 'web_fetch') { @@ -364,6 +584,61 @@ export function recordToolLoopOutcome( } export function formatToolLoopGuardMessage(reason: string, toolName: string): string { + // O96: same-command identical-output streak — never say STOP / tool broken. + if (/^same check failed repeatedly/i.test(reason)) { + const cmd = + reason.replace(/^same check failed repeatedly\s*\(\d+×\):\s*/i, '').trim() || toolName; + return [ + `同一检查连续失败:\`${cmd}\` 输出实质相同。请改代码或改命令/断言,不要原样重跑。`, + `Same check failed repeatedly for \`${cmd}\` with substantially identical output. Change the code or the command/assertions — do not re-run the same check unchanged.`, + '优先修代码;不要换工具绕开同一检查。Prefer fixing the code; do not switch tools merely to bypass the same check.', + ].join(' '); + } + // O96: real exec/run_tests faults — say 超时 / 未能启动, not "test failed". + if (VERIFY_SHELL_TOOLS.has(toolName)) { + if (/timed out \d+ time/i.test(reason)) { + return [ + `已拦截:${toolName} 连续超时(${reason})。`, + `Blocked: ${toolName} timed out repeatedly (${reason}).`, + '请提高 timeout_ms、简化命令,或改用 exec_background;不要原样重试。', + 'Raise timeout_ms, simplify the command, or use exec_background — do not retry unchanged.', + ].join(' '); + } + const missingMatch = /\(missing command: ([^\s)]+)\)/.exec(reason); + if (/failed to start \d+ time/i.test(reason) && missingMatch) { + const name = missingMatch[1]; + return [ + `已拦截:${toolName} 未能启动:找不到命令 \`${name}\`(${reason})。`, + `Blocked: ${toolName} failed to start: command \`${name}\` not found (${reason}).`, + `下一步:装依赖或改命令(例如安装提供 \`${name}\` 的包,或换成已安装的命令);不要原样重试。`, + `Next step: install the dependency or change the command (e.g. install the package that provides \`${name}\`, or use a command that is installed) — do not retry unchanged.`, + ].join(' '); + } + if (/failed to start \d+ time/i.test(reason)) { + return [ + `已拦截:${toolName} 未能启动(命令不存在或起不来;${reason})。`, + `Blocked: ${toolName} failed to start (command missing or could not launch; ${reason}).`, + '请检查命令是否在 PATH 中、拼写是否正确,再换一条能启动的命令。', + 'Check PATH and spelling, then try a command that can actually launch.', + ].join(' '); + } + if (/was killed \d+ time/i.test(reason)) { + return [ + `已拦截:${toolName} 连续被信号杀掉(${reason})。`, + `Blocked: ${toolName} was killed repeatedly (${reason}).`, + '请检查是否超时杀进程或外部信号;不要原样重试。', + 'Check for timeout kills or external signals — do not retry unchanged.', + ].join(' '); + } + if (/returned unusable results \d+ time/i.test(reason)) { + return [ + `已拦截:${toolName} 连续返回不可用输出(${reason})。`, + `Blocked: ${toolName} returned unusable output repeatedly (${reason}).`, + '请换一条能产生可读输出的命令;不要原样重试。', + 'Try a different command that produces readable output — do not retry unchanged.', + ].join(' '); + } + } if (/fresh-news search is already in progress/i.test(reason)) { return [ `[moss-agent] Tool loop guard stopped another ${toolName} call: ${reason}.`, @@ -728,6 +1003,21 @@ export function shouldShortCircuitToolCall( return 'a fresh-news search is already in progress in this assistant step'; } + // O96: same-command identical-output streak (independent of byToolFailure). + if (VERIFY_SHELL_TOOLS.has(toolName)) { + const sameCmdLimit = resolveOptionalPositiveIntEnv( + 'MOSS_TOOL_LOOP_SAME_COMMAND_LIMIT', + DEFAULT_SAME_COMMAND_FAILURE_LIMIT + ); + const sig = normalizeVerifyCommandSignature(toolName, input); + if (sig && sameCmdLimit !== undefined) { + const streak = state.bySameCommandFailure.get(sig.key); + if (streak && streak.count >= sameCmdLimit) { + return `same check failed repeatedly (${streak.count}×): ${streak.displayCommand}`; + } + } + } + // web_fetch failures are tracked per-URL (see ToolLoopGuardState), so the // block decision for a given web_fetch call keys off its own URL, not the // aggregate tool-level counter. A different URL that has never failed stays @@ -745,6 +1035,23 @@ export function shouldShortCircuitToolCall( // against, and the identical-input / single-tool / total guards below // still bound it. } else if (effectiveFailureLimit !== undefined && failureCount >= effectiveFailureLimit) { + // O96: encode fault kind so the message says 超时 / 未能启动, not "test failed". + if (VERIFY_SHELL_TOOLS.has(toolName)) { + const kind = state.lastShellFaultKind.get(toolName) ?? 'unusable'; + if (kind === 'timeout') { + return `${toolName} timed out ${failureCount} time(s) in this user turn`; + } + if (kind === 'start_failure') { + const missing = state.lastMissingCommand?.get(toolName); + return `${toolName} failed to start ${failureCount} time(s) in this user turn${ + missing ? ` (missing command: ${missing})` : '' + }`; + } + if (kind === 'killed') { + return `${toolName} was killed ${failureCount} time(s) in this user turn`; + } + return `${toolName} returned unusable results ${failureCount} time(s) in this user turn`; + } return `${toolName} has failed ${failureCount} time(s) in this user turn`; } if (identicalLimit !== undefined && sameSignatureCount >= identicalLimit) { diff --git a/src/core/tools/tool-types.ts b/src/core/tools/tool-types.ts index 050aadc0..119f6e05 100644 --- a/src/core/tools/tool-types.ts +++ b/src/core/tools/tool-types.ts @@ -20,6 +20,11 @@ export interface SubagentRunProgress { export interface ToolContext { workspaceDir: string; + /** + * O96: lets a string-returning tool mark its result as an error structurally + * (e.g. exec timeout / signal kill) instead of relying on output text. + */ + markResultError?: () => void; bootstrapDir?: string; runId?: string; sessionKey: string; diff --git a/src/tools/builtin.ts b/src/tools/builtin.ts index d32cd6e5..892505f2 100644 --- a/src/tools/builtin.ts +++ b/src/tools/builtin.ts @@ -250,6 +250,7 @@ export const execTool: Tool = { if (err instanceof ProcessError) { const output = [err.stdout.trim(), err.stderr.trim()].filter(Boolean).join('\n'); if (err.timedOut) { + ctx.markResultError?.(); const waited = err.timeoutMs && err.timeoutMs > 0 ? err.timeoutMs : timeoutMs; return ( `timed out after ${waited}ms (raise timeout_ms or use exec_background)${output ? `\n${output}` : ''}${writebackWarning}` + @@ -258,6 +259,7 @@ export const execTool: Tool = { ); } if (err.signal) { + ctx.markResultError?.(); return ( `Command killed by ${err.signal}:\n${output || err.message}${writebackWarning}` + footnote + diff --git a/test/o96-loop-guard-exec-exit.spec.mjs b/test/o96-loop-guard-exec-exit.spec.mjs new file mode 100644 index 00000000..4bd0edaf --- /dev/null +++ b/test/o96-loop-guard-exec-exit.spec.mjs @@ -0,0 +1,586 @@ +#!/usr/bin/env node +/** + * O96 — loop guard must not treat normal lint/test non-zero exits as a broken + * exec tool. Real faults (timeout / start failure) still block; same-command + * identical-output streaks hint to change code/command without STOP wording. + */ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { + createToolLoopGuardState, + recordToolLoopOutcome, + shouldShortCircuitToolCall, + formatToolLoopGuardMessage, + classifyShellToolFailure, + normalizeVerifyCommandSignature, + parseMissingCommandName, +} from '../dist/core/tools/tool-loop-guard.js'; +import { + executeOneToolCall, + isStringToolFailureResult, + outcomeToResult, +} from '../dist/core/tools/execute-tool-call.js'; +import { execTool } from '../dist/tools/builtin.js'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +const LINT_FAIL = 'Command failed (exit 1):\neslint: 3 errors\n'; +const LINT_FAIL_ALT = 'Command failed (exit 1):\neslint: 5 errors (different)\n'; +const TEST_FAIL = + 'Test Results: ❌ 2 FAILED\nCommand: npm test\nTests: 2 total, 0 passed, 2 failed\n'; +const EXIT_CODE_FAIL = 'exit_code: 1\nnpm run check failed\n'; +const TIMEOUT = 'timed out after 120000ms (raise timeout_ms or use exec_background)\n'; +const START_FAIL = 'Execution error: spawn Sync foo ENOENT\n'; +const START_FAIL_CMD = 'Command failed:\n/bin/sh: npmx: command not found\n'; +const BLOCKED = 'Command blocked: dangerous rm -rf /\n'; +const KILLED = 'Command killed by SIGKILL:\npartial\n'; + +const STOP_RE = /STOP calling|is not returning usable results|工具不可用|不要再调用/; + +test('classifyShellToolFailure: normal non-zero vs real faults vs deny', () => { + assert.equal(classifyShellToolFailure(LINT_FAIL), 'normal_nonzero'); + assert.equal(classifyShellToolFailure(EXIT_CODE_FAIL), 'normal_nonzero'); + assert.equal(classifyShellToolFailure(TEST_FAIL), 'normal_nonzero'); + assert.equal(classifyShellToolFailure(TIMEOUT), 'timeout'); + assert.equal(classifyShellToolFailure(START_FAIL), 'start_failure'); + assert.equal(classifyShellToolFailure(START_FAIL_CMD), 'start_failure'); + assert.equal(classifyShellToolFailure(KILLED), 'killed'); + assert.equal(classifyShellToolFailure(BLOCKED), 'other'); + assert.equal(classifyShellToolFailure(''), 'unusable'); + assert.equal(classifyShellToolFailure('(no output)'), 'unusable'); +}); + +test('A: three normal lint/test non-zero execs do NOT trip global byToolFailure / STOP', () => { + const state = createToolLoopGuardState(); + // Three different verification commands, each a normal non-zero exit. + const cmds = [ + { command: 'npm run check' }, + { command: 'npm test' }, + { command: 'npx eslint src' }, + ]; + const results = [LINT_FAIL, TEST_FAIL, EXIT_CODE_FAIL]; + for (let i = 0; i < 3; i++) { + recordToolLoopOutcome(state, 'exec', true, results[i], cmds[i]); + } + assert.equal( + state.byToolFailure.get('exec') ?? 0, + 0, + 'normal non-zero must not increment byToolFailure' + ); + // Still allowed to call exec again (e.g. another check). + const next = shouldShortCircuitToolCall(state, 'exec', { command: 'npm run lint' }); + assert.equal(next, null, 'must still allow further exec after 3 lint/test non-zeros'); + // Guard: no STOP wording could have been produced from a non-block. + assert.equal(state.byToolFailure.get('exec'), undefined); +}); + +test('A-run_tests: harness FAIL banner also skips byToolFailure', () => { + const state = createToolLoopGuardState(); + for (let i = 0; i < 3; i++) { + recordToolLoopOutcome(state, 'run_tests', true, TEST_FAIL, { command: 'npm test' }); + } + assert.equal(state.byToolFailure.get('run_tests') ?? 0, 0); + // Same command identical output → same-command streak, not global failure. + const blocked = shouldShortCircuitToolCall(state, 'run_tests', { command: 'npm test' }); + assert.ok(blocked, 'identical same-command streak should hint'); + assert.match(blocked, /same check failed repeatedly/i); + assert.equal(state.byToolFailure.get('run_tests') ?? 0, 0); + const msg = formatToolLoopGuardMessage(blocked, 'run_tests'); + assert.doesNotMatch(msg, STOP_RE); + assert.match(msg, /同一检查连续失败|Same check failed repeatedly/i); + assert.match(msg, /改代码|Change the code/i); +}); + +test('B: three timeouts still block; message says 超时 not test-failed', () => { + const state = createToolLoopGuardState(); + for (let i = 0; i < 3; i++) { + recordToolLoopOutcome(state, 'exec', true, TIMEOUT, { command: `sleep ${i}` }); + } + assert.equal(state.byToolFailure.get('exec'), 3); + const blocked = shouldShortCircuitToolCall(state, 'exec', { command: 'sleep 99' }); + assert.ok(blocked); + assert.match(blocked, /timed out 3 time/i); + assert.doesNotMatch(blocked, /has failed 3 time/i); + const msg = formatToolLoopGuardMessage(blocked, 'exec'); + assert.match(msg, /超时|timed out/i); + assert.doesNotMatch(msg, /测试没过|lint failed/i); +}); + +test('B: three start failures still block; message says 未能启动', () => { + const state = createToolLoopGuardState(); + for (let i = 0; i < 3; i++) { + recordToolLoopOutcome(state, 'exec', true, START_FAIL, { command: `missing-bin-${i}` }); + } + assert.equal(state.byToolFailure.get('exec'), 3); + const blocked = shouldShortCircuitToolCall(state, 'exec', { command: 'missing-bin-x' }); + assert.ok(blocked); + assert.match(blocked, /failed to start 3 time/i); + const msg = formatToolLoopGuardMessage(blocked, 'exec'); + assert.match(msg, /未能启动|failed to start/i); +}); + +test('C: same command identical output ≥3 → change-approach hint, no STOP', () => { + const state = createToolLoopGuardState(); + const input = { command: 'npm run check' }; + for (let i = 0; i < 3; i++) { + recordToolLoopOutcome(state, 'exec', true, LINT_FAIL, input); + } + assert.equal(state.byToolFailure.get('exec') ?? 0, 0, 'streak must not pollute byToolFailure'); + const blocked = shouldShortCircuitToolCall(state, 'exec', input); + assert.ok(blocked); + assert.match(blocked, /same check failed repeatedly/); + assert.match(blocked, /npm run check/); + const msg = formatToolLoopGuardMessage(blocked, 'exec'); + assert.doesNotMatch(msg, STOP_RE); + assert.match(msg, /同一检查连续失败/); + assert.match(msg, /Same check failed repeatedly/); + assert.match(msg, /改代码|Change the code/); + assert.match(msg, /不要换工具绕开|do not switch tools/i); +}); + +test('QA isolation: after 3× same npm run check, a different exec (npm test) still runs', () => { + const state = createToolLoopGuardState(); + for (let i = 0; i < 3; i++) { + recordToolLoopOutcome(state, 'exec', true, LINT_FAIL, { command: 'npm run check' }); + } + assert.ok(shouldShortCircuitToolCall(state, 'exec', { command: 'npm run check' })); + // Different command must still be allowed; no global STOP. + const other = shouldShortCircuitToolCall(state, 'exec', { command: 'npm test' }); + assert.equal(other, null, 'different command must not be blocked by same-command streak'); + assert.equal(state.byToolFailure.get('exec') ?? 0, 0); +}); + +test('C: different output on same command resets streak (no false block)', () => { + const state = createToolLoopGuardState(); + const input = { command: 'npm run check' }; + recordToolLoopOutcome(state, 'exec', true, LINT_FAIL, input); + recordToolLoopOutcome(state, 'exec', true, LINT_FAIL, input); + recordToolLoopOutcome(state, 'exec', true, LINT_FAIL_ALT, input); // different output + assert.equal(state.bySameCommandFailure.get('exec:npm run check')?.count, 1); + assert.equal(shouldShortCircuitToolCall(state, 'exec', input), null); +}); + +test('exit 0 success clears same-command streak', () => { + const state = createToolLoopGuardState(); + const input = { command: 'npm run check' }; + recordToolLoopOutcome(state, 'exec', true, LINT_FAIL, input); + recordToolLoopOutcome(state, 'exec', true, LINT_FAIL, input); + recordToolLoopOutcome(state, 'exec', false, 'ok\n', input); + assert.equal(state.bySameCommandFailure.has('exec:npm run check'), false); +}); + +test('QA guardrail: Command blocked is NOT normal_nonzero (still counts as fault path)', () => { + const state = createToolLoopGuardState(); + for (let i = 0; i < 3; i++) { + recordToolLoopOutcome(state, 'exec', true, BLOCKED, { command: `rm -rf /tmp/x${i}` }); + } + assert.equal(state.byToolFailure.get('exec'), 3, 'policy deny must not be treated as lint miss'); + const blocked = shouldShortCircuitToolCall(state, 'exec', { command: 'rm -rf /tmp/z' }); + assert.ok(blocked, 'three policy denies still short-circuit via byToolFailure'); +}); + +test('normalizeVerifyCommandSignature collapses whitespace', () => { + const a = normalizeVerifyCommandSignature('exec', { command: 'npm run check' }); + const b = normalizeVerifyCommandSignature('exec', { command: 'npm run check' }); + assert.equal(a?.key, b?.key); + assert.equal(a?.display, 'npm run check'); +}); + +test('UX: normal non-zero path never produces STOP / 工具不可用 wording', () => { + const state = createToolLoopGuardState(); + for (let i = 0; i < 3; i++) { + recordToolLoopOutcome(state, 'exec', true, LINT_FAIL, { command: 'npm run check' }); + } + const reason = shouldShortCircuitToolCall(state, 'exec', { command: 'npm run check' }); + assert.ok(reason); + const msg = formatToolLoopGuardMessage(reason, 'exec'); + assert.doesNotMatch(msg, STOP_RE); + assert.doesNotMatch(msg, /is not returning usable results/); +}); + +function execCallDeps(tools) { + const abortController = new AbortController(); + return { + toolsForRun: tools, + toolCtx: { workspaceDir: process.cwd(), sessionKey: 'o96-e2e' }, + sessionKey: 'o96-e2e', + abortSignal: abortController.signal, + toolTimeoutMs: 30_000, + enableHeartbeat: false, + heartbeatIntervalMs: 60_000, + skipHeartbeatToolNames: new Set(['exec']), + push() {}, + }; +} + +test('e2e: production-shaped timed out after → is_error, byToolFailure, 超时 copy', async () => { + // Same shape as builtin.ts ProcessError timeout (historically is_error=false). + const PROD_TIMEOUT = 'timed out after 200ms (raise timeout_ms or use exec_background)'; + // O96: timeout is marked structurally (ctx.markResultError), not by text. + assert.equal(isStringToolFailureResult(PROD_TIMEOUT), false); + + const mockExec = { + name: 'exec', + description: 'test', + inputSchema: { + type: 'object', + properties: { command: { type: 'string' }, timeout_ms: { type: 'number' } }, + required: ['command'], + }, + async execute(_input, ctx) { + ctx.markResultError?.(); + return PROD_TIMEOUT; + }, + }; + const deps = execCallDeps([mockExec]); + const state = createToolLoopGuardState(); + + for (let i = 0; i < 3; i++) { + const outcome = await executeOneToolCall( + { + id: `t${i}`, + name: 'exec', + input: { command: 'sleep 5', timeout_ms: 200 }, + }, + deps + ); + assert.equal(outcome.kind, 'completed'); + const { text, isError } = outcomeToResult(outcome); + assert.equal(isError, true, `attempt ${i}: is_error must be true for timed out after`); + assert.match(text, /timed out after 200ms/); + // Feed the REAL isError from executeOneToolCall (the e2e wiring UX failed on). + recordToolLoopOutcome(state, 'exec', isError, text, { + command: 'sleep 5', + timeout_ms: 200, + }); + } + + assert.equal(state.byToolFailure.get('exec'), 3, 'byToolFailure must count real timeouts'); + assert.equal(state.lastShellFaultKind.get('exec'), 'timeout'); + + const blocked = shouldShortCircuitToolCall(state, 'exec', { + command: 'sleep 5', + timeout_ms: 200, + }); + assert.ok(blocked, 'must short-circuit after 3 timeouts'); + assert.match(blocked, /timed out 3 time/i); + assert.doesNotMatch(blocked, /identical input was already requested/i); + assert.doesNotMatch(blocked, /same check failed repeatedly/i); + const msg = formatToolLoopGuardMessage(blocked, 'exec'); + assert.match(msg, /超时|timed out/i); + assert.doesNotMatch(msg, STOP_RE); +}); + +test('e2e: real execTool sleep+timeout_ms through executeOneToolCall counts as timeout', async () => { + const workspaceDir = fs.mkdtempSync(path.join(os.tmpdir(), 'moss-o96-e2e-')); + const deps = execCallDeps([execTool]); + deps.toolCtx = { workspaceDir, sessionKey: 'o96-real' }; + deps.sessionKey = 'o96-real'; + const state = createToolLoopGuardState(); + + for (let i = 0; i < 3; i++) { + const outcome = await executeOneToolCall( + { + id: `r${i}`, + name: 'exec', + input: { command: 'sleep 2', timeout_ms: 200 }, + }, + deps + ); + assert.equal(outcome.kind, 'completed'); + const { text, isError } = outcomeToResult(outcome); + assert.match(text, /timed out after 200ms/); + assert.equal(isError, true, `real exec timeout ${i} must be is_error`); + recordToolLoopOutcome(state, 'exec', isError, text, { + command: 'sleep 2', + timeout_ms: 200, + }); + } + + assert.equal(state.byToolFailure.get('exec'), 3); + const blocked = shouldShortCircuitToolCall(state, 'exec', { + command: 'sleep 2', + timeout_ms: 200, + }); + assert.ok(blocked); + assert.match(blocked, /timed out 3 time/i); + const msg = formatToolLoopGuardMessage(blocked, 'exec'); + assert.match(msg, /超时/); + assert.match(msg, /timed out/i); + try { + fs.rmSync(workspaceDir, { recursive: true, force: true }); + } catch { + /* ignore */ + } +}); + +// QA #97 913354c4: fault-looking text in child stdout must never count as a tool fault. +// is_error comes from execute-tool-call (anchored isStringToolFailureResult). +const NEG_CASES = [ + { + name: '(a) exit 1, node:test prints "test timed out after 100ms"', + text: "exit_code: 1\nnot ok 1 - slow case\n error: 'test timed out after 100ms'\n# fail 1\n", + isError: true, + kind: 'normal_nonzero', + }, + { + name: '(b) exit 1, jest test name contains "timed out after"', + text: 'exit_code: 1\nFAIL src/net.test.ts\n ✕ retries when socket timed out after 3 attempts (12 ms)\nTests: 1 failed, 4 passed\n', + isError: true, + kind: 'normal_nonzero', + }, + { + name: '(c) exit 0, log "connection timed out after 5s"', + text: 'exit_code: 0\n[warn] connection timed out after 5s, retrying\nok\n', + isError: false, + kind: 'other', + }, + { + name: '(d) exit 0, warning line containing ENOENT', + text: "exit_code: 0\nnpm warn optional: ENOENT: no such file or directory, open '.npmrc'\ndone\n", + isError: false, + kind: 'other', + }, +]; + +for (const c of NEG_CASES) { + test(`negative ${c.name} → no byToolFailure across 3 runs, no STOP`, () => { + assert.equal( + isStringToolFailureResult(c.text), + c.isError, + 'is_error as execute-tool-call sets it' + ); + assert.equal(classifyShellToolFailure(c.text), c.kind); + const state = createToolLoopGuardState(); + for (let i = 0; i < 3; i++) { + recordToolLoopOutcome(state, 'exec', c.isError, c.text, { command: 'npm test' }); + } + assert.equal(state.byToolFailure.get('exec') ?? 0, 0); + const blocked = shouldShortCircuitToolCall(state, 'exec', { command: 'npm run lint' }); + if (blocked) { + assert.doesNotMatch(formatToolLoopGuardMessage(blocked, 'exec'), /STOP calling|工具不可用/i); + } + }); +} + +test('fault patterns anchored to first line (ENOENT / command not found / timeout / killed)', () => { + assert.equal(classifyShellToolFailure('timed out after 200ms (raise timeout_ms)\np'), 'timeout'); + assert.equal(classifyShellToolFailure('Command killed by SIGKILL:\np'), 'killed'); + assert.equal( + classifyShellToolFailure('Execution error: spawnSync foo ENOENT\n'), + 'start_failure' + ); + assert.equal(classifyShellToolFailure('bash: foo: command not found\n'), 'start_failure'); + assert.equal( + classifyShellToolFailure('Command failed (exit 1):\nENOENT foo\n'), + 'normal_nonzero' + ); + assert.equal( + classifyShellToolFailure('Command failed (exit 1):\nsh: x: command not found\n'), + 'normal_nonzero' + ); + assert.equal( + classifyShellToolFailure('Command failed (exit 2):\nkilled by SIGTERM\n'), + 'normal_nonzero' + ); + assert.equal(classifyShellToolFailure('exit_code: 3\ntimed out after 1ms\n'), 'normal_nonzero'); +}); + +// PM (#97 88b62bdf): exec first-line exit 126/127 = start failure (未能启动). +for (const cmd of ['nosuchcmd_o96_xyz', '/no/such/bin']) { + test(`e2e: real exec \`${cmd}\` x3 → blocked on 4th with 未能启动`, async () => { + const workspaceDir = fs.mkdtempSync(path.join(os.tmpdir(), 'moss-o96-127-')); + const deps = execCallDeps([execTool]); + deps.toolCtx = { workspaceDir, sessionKey: 'o96-127' }; + deps.sessionKey = 'o96-127'; + const state = createToolLoopGuardState(); + for (let i = 0; i < 3; i++) { + assert.equal(shouldShortCircuitToolCall(state, 'exec', { command: cmd }) ?? null, null); + const outcome = await executeOneToolCall( + { id: `n${i}`, name: 'exec', input: { command: cmd } }, + deps + ); + const { text, isError } = outcomeToResult(outcome); + assert.equal(isError, true, `attempt ${i}: ${text.slice(0, 80)}`); + assert.equal(classifyShellToolFailure(text), 'start_failure', text.slice(0, 80)); + recordToolLoopOutcome(state, 'exec', isError, text, { command: cmd }); + } + assert.equal(state.byToolFailure.get('exec'), 3); + const blocked = shouldShortCircuitToolCall(state, 'exec', { command: cmd }); + assert.ok(blocked, '4th call must be blocked'); + const msg = formatToolLoopGuardMessage(blocked, 'exec'); + assert.match(msg, /未能启动/); + assert.doesNotMatch(msg, /STOP calling/); + fs.rmSync(workspaceDir, { recursive: true, force: true }); + }); +} + +test('first-line 126/127 = start_failure; script-internal exit 1 and stdout 127 stay normal', () => { + assert.equal( + classifyShellToolFailure('exit_code: 127\nsh: nosuch: not found\n'), + 'start_failure' + ); + assert.equal( + classifyShellToolFailure('exit_code: 126\nsh: /x: Permission denied\n'), + 'start_failure' + ); + assert.equal(classifyShellToolFailure('Command failed (exit 127):\n'), 'start_failure'); + assert.equal(classifyShellToolFailure('Command failed (exit 126):\n'), 'start_failure'); + assert.equal( + classifyShellToolFailure('exit_code: 1\nchild exited with exit_code: 127\n'), + 'normal_nonzero' + ); + assert.equal(classifyShellToolFailure('exit_code: 1\nscript failed\n'), 'normal_nonzero'); + const state = createToolLoopGuardState(); + for (let i = 0; i < 3; i++) { + recordToolLoopOutcome(state, 'exec', true, 'exit_code: 1\nscript failed\n', { + command: 'bash ci.sh', + }); + } + assert.equal(state.byToolFailure.get('exec') ?? 0, 0); +}); + +test('parseMissingCommandName: common shells; unparseable → undefined', () => { + assert.equal(parseMissingCommandName('exit_code: 127\nsh: eslint: not found\n'), 'eslint'); + assert.equal(parseMissingCommandName('exit_code: 127\nsh: 1: eslint: not found\n'), 'eslint'); + assert.equal(parseMissingCommandName('exit_code: 127\nbash: foo: command not found\n'), 'foo'); + assert.equal(parseMissingCommandName('exit_code: 127\nzsh: command not found: foo\n'), 'foo'); + assert.equal( + parseMissingCommandName('exit_code: 127\nbash: line 1: foo: command not found\n'), + 'foo' + ); + assert.equal( + parseMissingCommandName('exit_code: 127\n/bin/sh: /no/such/bin: No such file or directory\n'), + '/no/such/bin' + ); + assert.equal(parseMissingCommandName('exit_code: 127\n'), undefined); +}); + +test('127 without a parseable name → generic 未能启动 copy (no 装依赖或改命令 name)', () => { + const state = createToolLoopGuardState(); + for (let i = 0; i < 3; i++) + recordToolLoopOutcome(state, 'exec', true, 'exit_code: 127\n', { command: 'x' }); + const blocked = shouldShortCircuitToolCall(state, 'exec', { command: 'x' }); + assert.ok(blocked); + assert.doesNotMatch(blocked, /missing command/); + const msg = formatToolLoopGuardMessage(blocked, 'exec'); + assert.match(msg, /未能启动/); + assert.doesNotMatch(msg, /STOP calling|工具不可用/); +}); + +test('e2e: npm run script calling a missing binary x3 → 4th blocked with name + 装依赖或改命令', async () => { + const workspaceDir = fs.mkdtempSync(path.join(os.tmpdir(), 'moss-o96-npm127-')); + fs.writeFileSync( + path.join(workspaceDir, 'package.json'), + JSON.stringify({ + name: 'o96-fixture', + version: '0.0.0', + private: true, + scripts: { lint: 'o96missinglinter --fix .' }, + }) + ); + const deps = execCallDeps([execTool]); + deps.toolCtx = { workspaceDir, sessionKey: 'o96-npm127' }; + deps.sessionKey = 'o96-npm127'; + const state = createToolLoopGuardState(); + const input = { command: 'npm run lint', timeout_ms: 60000 }; + for (let i = 0; i < 3; i++) { + assert.equal(shouldShortCircuitToolCall(state, 'exec', input) ?? null, null); + const outcome = await executeOneToolCall({ id: `m${i}`, name: 'exec', input }, deps); + const { text, isError } = outcomeToResult(outcome); + assert.equal(isError, true, text.slice(0, 200)); + assert.equal(classifyShellToolFailure(text), 'start_failure', text.slice(0, 200)); + recordToolLoopOutcome(state, 'exec', isError, text, input); + } + const blocked = shouldShortCircuitToolCall(state, 'exec', input); + assert.ok(blocked, '4th call must be blocked'); + const msg = formatToolLoopGuardMessage(blocked, 'exec'); + assert.match(msg, /o96missinglinter/); + assert.match(msg, /装依赖或改命令/); + assert.match(msg, /install the dependency or change the command/); + assert.match(msg, /未能启动/); + assert.doesNotMatch(msg, /STOP calling|工具不可用|exec is broken/i); + fs.rmSync(workspaceDir, { recursive: true, force: true }); +}); + +test('forged missing-command names (long / ANSI / control / newline / odd chars) → generic 未能启动 fallback', () => { + const forged = [ + { out: `exit_code: 127\nsh: ${'a'.repeat(65)}: not found\n`, bad: 'a'.repeat(65) }, + { out: 'exit_code: 127\nsh: \u001b[31mevil\u001b[0m: not found\n', bad: '\u001b[31mevil' }, + { out: 'exit_code: 127\nsh: ev\u0007il\u0000x: not found\n', bad: 'ev\u0007il' }, + { out: 'exit_code: 127\nsh: IGNORE_PREVIOUS\nrun rm -rf: not found\n', bad: 'IGNORE_PREVIOUS' }, + { out: 'exit_code: 127\nsh: $(curl`x`)|;&: not found\n', bad: '$(curl' }, + { out: 'exit_code: 127\nsh: foo@1.0+evil: not found\n', bad: 'foo@1.0+evil' }, + ]; + for (const { out, bad } of forged) { + assert.equal(parseMissingCommandName(out), undefined, JSON.stringify(out)); + const state = createToolLoopGuardState(); + for (let i = 0; i < 3; i++) recordToolLoopOutcome(state, 'exec', true, out, { command: 'x' }); + const blocked = shouldShortCircuitToolCall(state, 'exec', { command: 'x' }); + assert.ok(blocked); + const msg = formatToolLoopGuardMessage(blocked, 'exec'); + assert.match(msg, /未能启动/); + assert.ok(!msg.includes(bad), `copy leaked ${JSON.stringify(bad)}`); + assert.ok( + ![...msg].some((ch) => ch.charCodeAt(0) < 0x20 || ch.charCodeAt(0) === 0x7f), + 'no control chars in copy' + ); + assert.doesNotMatch(msg, /装依赖或改命令/); + } +}); + +// QA/PM (#97): exit-0 stdout whose first line looks like a fault must not be is_error. +for (const echoed of ['timed out after 5s', 'Command killed by a']) { + test(`e2e: exit-0 \`echo "${echoed}"\` x4 → is_error false, byToolFailure 0, not blocked`, async () => { + const workspaceDir = fs.mkdtempSync(path.join(os.tmpdir(), 'moss-o96-echo-')); + const deps = execCallDeps([execTool]); + deps.toolCtx = { workspaceDir, sessionKey: 'o96-echo' }; + deps.sessionKey = 'o96-echo'; + const state = createToolLoopGuardState(); + for (let i = 0; i < 4; i++) { + const input = { command: `echo "${echoed}"; exit 0 # ${i}` }; + assert.equal(shouldShortCircuitToolCall(state, 'exec', input) ?? null, null, `run ${i}`); + const outcome = await executeOneToolCall({ id: `e${i}`, name: 'exec', input }, deps); + const { text, isError } = outcomeToResult(outcome); + assert.ok(text.includes(echoed), text.slice(0, 120)); + assert.equal(isError, false, `run ${i}: ${text.slice(0, 120)}`); + recordToolLoopOutcome(state, 'exec', isError, text, input); + } + assert.equal(state.byToolFailure.get('exec') ?? 0, 0); + assert.equal( + shouldShortCircuitToolCall(state, 'exec', { command: 'echo ok # 5' }) ?? null, + null + ); + fs.rmSync(workspaceDir, { recursive: true, force: true }); + }); +} + +test('isStringToolFailureResult no longer text-matches timed out / killed prefixes', () => { + assert.equal(isStringToolFailureResult('timed out after 5s\n'), false); + assert.equal(isStringToolFailureResult('Command killed by a\n'), false); +}); + +test( + 'e2e: real signal kill x3 → is_error true (structural), 4th blocked', + { skip: process.platform === 'win32' }, + async () => { + const workspaceDir = fs.mkdtempSync(path.join(os.tmpdir(), 'moss-o96-kill-')); + const deps = execCallDeps([execTool]); + deps.toolCtx = { workspaceDir, sessionKey: 'o96-kill' }; + deps.sessionKey = 'o96-kill'; + const state = createToolLoopGuardState(); + for (let i = 0; i < 3; i++) { + const input = { command: `kill -KILL $$ # ${i}` }; + const outcome = await executeOneToolCall({ id: `k${i}`, name: 'exec', input }, deps); + const { text, isError } = outcomeToResult(outcome); + assert.match(text, /^Command killed by SIG/, text.slice(0, 120)); + assert.equal(isError, true); + recordToolLoopOutcome(state, 'exec', isError, text, input); + } + assert.equal(state.byToolFailure.get('exec'), 3); + assert.ok(shouldShortCircuitToolCall(state, 'exec', { command: 'kill -KILL $$ # 9' })); + fs.rmSync(workspaceDir, { recursive: true, force: true }); + } +); diff --git a/test/string-tool-failure.spec.mjs b/test/string-tool-failure.spec.mjs index dd87a739..0be16b40 100644 --- a/test/string-tool-failure.spec.mjs +++ b/test/string-tool-failure.spec.mjs @@ -20,6 +20,16 @@ test('isStringToolFailureResult detects common failure encodings', () => { assert.equal(isStringToolFailureResult('exit_code: 1\nFAIL'), true); assert.equal(isStringToolFailureResult('exit_code: 0\nok'), false); assert.equal(isStringToolFailureResult('Command blocked: dangerous'), true); + assert.equal( + isStringToolFailureResult('timed out after 200ms (raise timeout_ms or use exec_background)\n'), + false, + 'O96: timeout is marked structurally by exec (ctx.markResultError), not by text' + ); + assert.equal( + isStringToolFailureResult('Command killed by SIGTERM:\npartial\n'), + false, + 'O96: signal kill is marked structurally by exec (ctx.markResultError), not by text' + ); assert.equal(isStringToolFailureResult('Patch rejected:\nbad hunk'), true); assert.equal(isStringToolFailureResult('Test Results: ❌ 2 FAILED\nCommand: npm test\n'), true); assert.equal(