From 149e046811112d2f920965125ceb34e7610eec76 Mon Sep 17 00:00:00 2001 From: QiaolongLi1201 Date: Mon, 12 Oct 2026 00:58:02 +0800 Subject: [PATCH 1/3] fix(o98): no false success at end of --print runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - new src/cli/run-completion.ts: track checks actually run + file writes, detect unverified success claims, assess completion, format 未完成 line - oneshot text mode: incomplete runs exit non-zero (ExitCode.INCOMPLETE=9 or existing error code) and end stdout with 未完成：…；卡在：… - test/run-completion-o98.spec.mjs --- src/cli/exit-codes.ts | 2 + src/cli/oneshot.ts | 30 ++++ src/cli/run-completion.ts | 276 +++++++++++++++++++++++++++++++ test/run-completion-o98.spec.mjs | 203 +++++++++++++++++++++++ 4 files changed, 511 insertions(+) create mode 100644 src/cli/run-completion.ts create mode 100644 test/run-completion-o98.spec.mjs diff --git a/src/cli/exit-codes.ts b/src/cli/exit-codes.ts index b12c31b9..e124e80c 100644 --- a/src/cli/exit-codes.ts +++ b/src/cli/exit-codes.ts @@ -10,6 +10,8 @@ export enum ExitCode { PROVIDER_UPSTREAM = 6, SESSION = 7, TOOL_EXECUTION = 8, + /** O98: run ended without finishing the task (or with an unverified success claim). */ + INCOMPLETE = 9, USER_ABORTED = 11, INTERNAL = 12, } diff --git a/src/cli/oneshot.ts b/src/cli/oneshot.ts index c2528131..ef8fb68b 100644 --- a/src/cli/oneshot.ts +++ b/src/cli/oneshot.ts @@ -29,6 +29,12 @@ import { buildGitStatusSnapshot } from '../context/git-status-snapshot.js'; import { acceptanceShell } from '../core/task/acceptance-command.js'; import { runStopHooks, type StopHookResult } from './hooks.js'; import { runProcess } from '../utils/run-process.js'; +import { + assessRunCompletion, + createRunCompletionTracker, + formatIncompleteLine, + noteRunCompletionEvent, +} from './run-completion.js'; /** Format the oneshot exit warning when background work outlives the wait window. */ export function formatOneshotStillRunningBackgroundNotice( @@ -422,6 +428,7 @@ export async function runOneShot( }); let finalResult: HeadlessResultEvent | undefined; let runError: unknown = undefined; + const completionTracker = createRunCompletionTracker(); const routedToolFilter = oneShotToolFilterForMessage(message); const focusedInspection = focusedInspectionRunOptions(message); const fastNews = fastNewsRunPolicy(message); @@ -529,6 +536,7 @@ export async function runOneShot( }; for await (const event of agent.streamChat(sessionKey, message, streamOptions)) { options.onAgentEvent?.(event); + noteRunCompletionEvent(completionTracker, event); const structuredEvents = formatHeadlessStreamEvent(state, event); if (outputFormat === 'text') { renderer?.handle(event); @@ -690,6 +698,28 @@ export async function runOneShot( } } + // O98: never end a text `--print` run on a false success. If the final + // answer claims checks that never (successfully) ran, or the task needed + // edits and none happened, exit non-zero and make the LAST stdout line say + // what is unfinished and where it got stuck. + if (outputFormat === 'text') { + const zh = isZhLocale(); + const verdict = assessRunCompletion({ + tracker: completionTracker, + finalText: typeof finalResult?.result === 'string' ? finalResult.result : state.finalText, + request: message, + ...(finalResult && isHeadlessResultError(finalResult) + ? { runError: finalResult.error ?? String(finalResult.subtype) } + : {}), + ...(typeof process.exitCode === 'number' ? { errorExitCode: process.exitCode } : {}), + zh, + }); + if (verdict.status === 'incomplete') { + process.exitCode = verdict.exitCode; + stdout.write(`\n${formatIncompleteLine(verdict, { zh })}\n`); + } + } + // Stop lifecycle hook: fires after every completed run. A blocking non-zero // exit vetoes the stop — the REPL turns that into one forced continuation // turn; headless reports it on stderr. diff --git a/src/cli/run-completion.ts b/src/cli/run-completion.ts new file mode 100644 index 00000000..8daefd97 --- /dev/null +++ b/src/cli/run-completion.ts @@ -0,0 +1,276 @@ +/** + * O98: no false success at the end of a headless run. + * + * O14 run2 ended with "All checks are green" although `format:check` never + * ran and `npm run check` was red; kimi-k3 r2 changed zero files and still + * exited 0. This module is the single place that decides whether a run is + * really finished: + * + * - {@link createRunCompletionTracker} / {@link noteRunCompletionEvent} + * record which verification commands actually ran (and whether they passed) + * and whether any file was written, from the agent event stream. + * - {@link findUnverifiedCheckClaims} lists success claims in the final + * answer that no run in this session backs up. + * - {@link assessRunCompletion} turns that into a verdict with a reason, the + * step it got stuck on, and the exit code. + * - {@link formatIncompleteLine} renders the mandatory last line + * (`未完成：… ；卡在：…` / `Incomplete: …; stuck at: …`). + * + * O99 (progress) reuses the verdict/exit code instead of re-deriving it. + */ +import type { MossAgentEvent } from '../core/index.js'; +import { ExitCode } from './exit-codes.js'; + +export type VerificationCheck = 'format' | 'lint' | 'typecheck' | 'test' | 'check'; + +export interface CheckRun { + command: string; + passed: boolean; +} + +export interface RunCompletionTracker { + /** Latest run of each check kind in this run (later runs overwrite earlier ones). */ + checks: Map; + /** True once a file-writing tool (or a write-like exec) succeeded. */ + wroteFiles: boolean; + /** Last failed tool step, e.g. `exec: npm run check`. */ + lastFailedStep?: string; + /** Last tool step that started, e.g. `multi_edit: src/cli/hooks.ts`. */ + lastStep?: string; + /** @internal pending tool_start inputs keyed by toolCallId. */ + pending: Map }>; +} + +export type RunCompletionStatus = 'complete' | 'incomplete'; + +export interface RunCompletionVerdict { + status: RunCompletionStatus; + /** Human reasons (already localized); empty when complete. */ + reasons: string[]; + /** Where the run got stuck, when known. */ + stuckAt?: string; + /** Claims in the final answer that were not backed by a real passing run. */ + unverifiedClaims: VerificationCheck[]; + /** 0 when complete, otherwise non-zero (existing error code or {@link ExitCode.INCOMPLETE}). */ + exitCode: number; +} + +export interface AssessRunCompletionInput { + tracker: RunCompletionTracker; + /** Final assistant answer. */ + finalText: string; + /** Original user request; used to decide whether file changes were expected. */ + request?: string; + /** The run already ended in error (max turns, budget, provider error …). */ + runError?: string; + /** Exit code already chosen for that error. */ + errorExitCode?: number; + zh?: boolean; +} + +const FILE_WRITE_TOOLS = new Set([ + 'write_file', + 'edit_file', + 'multi_edit', + 'apply_patch', + 'move_file', + 'delete_file', +]); +const EXEC_TOOLS = new Set(['exec', 'exec_background', 'device_exec']); +const WRITE_LIKE_EXEC = + /(?:\bnode\s+-e\b|\bsed\s+-i|\bperl\s+-[a-z]*i|>>?\s*[^\s&|]|\btee\b|\bgit\s+(?:apply|am|checkout|restore|mv|rm)\b|\bpatch\b|\b(?:mv|cp|rm|touch|mkdir)\s|\bprettier\b[^|;&]*--write|\beslint\b[^|;&]*--fix|\bnpm\s+run\s+(?:format|lint:fix)\b(?!:check))/; + +/** Map a shell command to the verification checks it really runs. */ +export function checksForCommand(command: string): VerificationCheck[] { + const out = new Set(); + if (/\bnpm\s+run\s+(?:check|verify)\b(?![:\w-])/.test(command)) { + out.add('check'); + out.add('format'); + out.add('lint'); + out.add('typecheck'); + } + if (/\bnpm\s+run\s+verify\b/.test(command)) out.add('test'); + if (/\bnpm\s+run\s+format:check\b|\bprettier\b[^|;&]*--check\b/.test(command)) out.add('format'); + if (/\bnpm\s+run\s+lint\b(?!:fix)|\beslint\b(?![^|;&]*--fix)/.test(command)) out.add('lint'); + if (/\bnpm\s+run\s+typecheck\b|\btsc\b[^|;&]*--noEmit\b/.test(command)) out.add('typecheck'); + if ( + /\b(?:npm\s+(?:run\s+)?test(?::\w+)?|npm\s+t|yarn\s+test|pnpm\s+test|node\s+--test|pytest|vitest|jest|mocha|go\s+test|cargo\s+test|make\s+test)\b/.test( + command + ) || + /\bnode\s+\S*\.spec\.[cm]?js\b/.test(command) + ) { + out.add('test'); + } + return [...out]; +} + +export function createRunCompletionTracker(): RunCompletionTracker { + return { checks: new Map(), wroteFiles: false, pending: new Map() }; +} + +function describeStep(toolName: string, input: Record): string { + const command = typeof input.command === 'string' ? input.command : undefined; + const filePath = typeof input.path === 'string' ? input.path : undefined; + const detail = (command ?? filePath ?? '').replace(/\s+/g, ' ').trim().slice(0, 120); + return detail ? `${toolName}: ${detail}` : toolName; +} + +function execFailed(result: string): boolean { + return /\bexit(?:ed)?(?:\s+code)?[\s:=]+[1-9]\d*\b/i.test(result); +} + +/** Feed one agent event into the tracker. Safe to call for every event. */ +export function noteRunCompletionEvent(tracker: RunCompletionTracker, event: MossAgentEvent): void { + if (event.type === 'tool_start') { + tracker.pending.set(event.toolCallId, { toolName: event.toolName, input: event.input ?? {} }); + tracker.lastStep = describeStep(event.toolName, event.input ?? {}); + return; + } + if (event.type !== 'tool_end') return; + const started = tracker.pending.get(event.toolCallId); + tracker.pending.delete(event.toolCallId); + const input = started?.input ?? {}; + const toolName = started?.toolName ?? event.toolName; + const failed = event.isError || (EXEC_TOOLS.has(toolName) && execFailed(event.result)); + if (failed) tracker.lastFailedStep = describeStep(toolName, input); + + if (FILE_WRITE_TOOLS.has(toolName) && !failed) tracker.wroteFiles = true; + if (toolName === 'run_tests') + tracker.checks.set('test', { command: 'run_tests', passed: !failed }); + if (EXEC_TOOLS.has(toolName) && typeof input.command === 'string') { + const command = input.command; + if (!failed && WRITE_LIKE_EXEC.test(command)) tracker.wroteFiles = true; + for (const check of checksForCommand(command)) { + tracker.checks.set(check, { command, passed: !failed }); + } + } +} + +const CLAIM_PATTERNS: Array<{ re: RegExp; requires: VerificationCheck[] }> = [ + { + re: /\ball\s+(?:the\s+)?checks?\s+(?:are\s+|is\s+)?(?:green|pass(?:ed|ing)?|clean|ok)\b|(?:全部|所有)(?:的)?检查(?:都|均|全部)?(?:已)?(?:通过|是绿的|绿了|变绿)|\bnpm\s+run\s+check\b[^.\n。]*\b(?:pass(?:es|ed)?|green|succeed(?:s|ed)?|clean)\b|npm\s+run\s+check[^。\n]*(?:通过|绿)/i, + requires: ['check'], + }, + { + re: /\b(?:prettier(?:\s+--check)?|format:check|formatting)\b[^.\n。]*\b(?:pass(?:es|ed)?|green|clean|ok)\b|(?:prettier|format:check|格式检查)[^。\n]*(?:通过|绿)/i, + requires: ['format'], + }, + { + re: /\b(?:eslint|lint(?:ing)?)\b[^.\n。]*\b(?:pass(?:es|ed)?|green|clean|ok)\b|(?:eslint|lint)[^。\n]*(?:通过|绿)/i, + requires: ['lint'], + }, + { + re: /\b(?:typecheck|tsc|type\s*check(?:ing)?)\b[^.\n。]*\b(?:pass(?:es|ed)?|green|clean|ok)\b|(?:typecheck|tsc|类型检查)[^。\n]*(?:通过|绿)/i, + requires: ['typecheck'], + }, + { + re: /\b(?:all\s+)?tests?\s+(?:are\s+|now\s+)?(?:pass(?:es|ed|ing)?|green)\b|\d+\s*\/\s*\d+\s+(?:tests?\s+)?passed\b|测试(?:都|均|全部)?(?:已)?(?:通过|绿了|变绿)/i, + requires: ['test'], + }, +]; + +/** + * Success claims in `text` that the run does not back up: the matching check + * never ran, or its latest run failed. + */ +export function findUnverifiedCheckClaims( + text: string, + tracker: RunCompletionTracker +): VerificationCheck[] { + const missing = new Set(); + const sentences = text.split(/(?<=[.!?。！？])\s+|\n+/); + for (const sentence of sentences) { + for (const { re, requires } of CLAIM_PATTERNS) { + const match = re.exec(sentence); + if (!match) continue; + if (/\b(?:not|n't|never)\b|未|没|尚未|无法/.test(match[0])) continue; + for (const check of requires) { + const run = tracker.checks.get(check); + if (!run || !run.passed) missing.add(check); + } + } + } + return [...missing]; +} + +const CHECK_LABEL: Record = { + check: 'npm run check', + format: 'format:check', + lint: 'lint', + typecheck: 'typecheck', + test: 'test', +}; + +const EXPECTS_CHANGES = + /(?:修复|修改|实现|改成|改为|改掉|新增|添加|增加|补上|补一|重构|删掉|删除|\bfix(?:es)?\b|\bimplement\b|\brefactor\b|\bchange\b|\bupdate\b|\bedit\b|\bwrite\s+(?:a\s+)?(?:test|file|function)|\badd\s+(?:a\s+|an\s+)?(?:test|option|flag|function|file|check|support))/i; +const QUESTION_ONLY = + /^(?:how|why|what|which|explain|describe|can you explain|为什么|怎么|如何|解释|说明一下|是什么)|[?？]\s*$/i; + +/** Heuristic: does this request expect the run to change files? */ +export function requestExpectsFileChanges(request: string): boolean { + const text = request.trim(); + if (!text || QUESTION_ONLY.test(text)) return false; + return EXPECTS_CHANGES.test(text); +} + +/** Decide whether the run is complete; never upgrades an error run to success. */ +export function assessRunCompletion(input: AssessRunCompletionInput): RunCompletionVerdict { + const zh = input.zh ?? false; + const { tracker } = input; + const reasons: string[] = []; + const unverifiedClaims = findUnverifiedCheckClaims(input.finalText, tracker); + + if (input.runError) reasons.push(input.runError); + if (unverifiedClaims.length > 0) { + const labels = unverifiedClaims.map((c) => CHECK_LABEL[c]).join(', '); + const failedRun = unverifiedClaims.some((c) => tracker.checks.get(c)); + reasons.push( + zh + ? `收尾声称检查已通过，但本次运行${failedRun ? '中这些检查未通过或' : ''}未实际跑过：${labels}` + : `final answer claims checks passed, but this run did not actually pass: ${labels}` + ); + } + if (input.request && requestExpectsFileChanges(input.request) && !tracker.wroteFiles) { + reasons.push( + zh + ? '任务需要改动文件，但本次运行没有修改任何文件' + : 'the task needed file changes, but no file was modified' + ); + } + + if (reasons.length === 0) { + return { status: 'complete', reasons, unverifiedClaims, exitCode: ExitCode.SUCCESS }; + } + const stuckAt = + tracker.lastFailedStep ?? + (unverifiedClaims.length > 0 + ? zh + ? `验证（未运行 ${unverifiedClaims.map((c) => CHECK_LABEL[c]).join(', ')}）` + : `verification (did not run ${unverifiedClaims.map((c) => CHECK_LABEL[c]).join(', ')})` + : tracker.lastStep); + const exitCode = + input.errorExitCode && input.errorExitCode !== 0 ? input.errorExitCode : ExitCode.INCOMPLETE; + return { + status: 'incomplete', + reasons, + ...(stuckAt ? { stuckAt } : {}), + unverifiedClaims, + exitCode, + }; +} + +/** The mandatory last line for an incomplete run; '' when complete. */ +export function formatIncompleteLine( + verdict: RunCompletionVerdict, + opts: { zh?: boolean } = {} +): string { + if (verdict.status === 'complete') return ''; + const reason = verdict.reasons + .join(opts.zh ? '；' : '; ') + .replace(/\s+/g, ' ') + .trim(); + const stuck = verdict.stuckAt ?? (opts.zh ? '未知' : 'unknown'); + return opts.zh + ? `未完成：${reason}；卡在：${stuck}` + : `Incomplete: ${reason}; stuck at: ${stuck}`; +} diff --git a/test/run-completion-o98.spec.mjs b/test/run-completion-o98.spec.mjs new file mode 100644 index 00000000..af34f839 --- /dev/null +++ b/test/run-completion-o98.spec.mjs @@ -0,0 +1,203 @@ +#!/usr/bin/env node +/** + * O98: a headless `moss -p` run must not end on a false success. + * O14 run2 said "All checks are green" without running format:check (npm run + * check was red); kimi-k3 r2 changed nothing and exited 0. + */ +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import { runOneShot } from '../dist/cli/oneshot.js'; +import { ExitCode } from '../dist/cli/exit-codes.js'; +import { + assessRunCompletion, + createRunCompletionTracker, + findUnverifiedCheckClaims, + formatIncompleteLine, + noteRunCompletionEvent, +} from '../dist/cli/run-completion.js'; +import { MossAgent } from '../dist/core/agent/moss-agent.js'; +import { InMemorySessionStore } from '../dist/core/session/session.js'; +import { createMockTranscriptProvider } from './e2e/mock-transcript-provider.mjs'; + +function execEvents(tracker, id, command, result, isError = false) { + noteRunCompletionEvent(tracker, { + type: 'tool_start', + toolName: 'exec', + toolCallId: id, + input: { command }, + }); + noteRunCompletionEvent(tracker, { + type: 'tool_end', + toolName: 'exec', + toolCallId: id, + result, + isError, + }); +} + +// --- Unit: claims vs. real runs ------------------------------------------- +{ + const t = createRunCompletionTracker(); + execEvents(t, '1', 'npm run lint', 'exit code 0'); + execEvents(t, '2', 'npm run typecheck', 'exit code 0'); + assert.deepEqual(findUnverifiedCheckClaims('All checks are green.', t), ['check']); + assert.deepEqual(findUnverifiedCheckClaims('Lint passes and typecheck is clean.', t), []); + assert.deepEqual(findUnverifiedCheckClaims('prettier --check passed', t), ['format']); + assert.deepEqual(findUnverifiedCheckClaims('所有检查都通过了', t), ['check']); + const v = assessRunCompletion({ tracker: t, finalText: 'All checks are green.', zh: true }); + assert.equal(v.status, 'incomplete'); + assert.equal(v.exitCode, ExitCode.INCOMPLETE); + assert.notEqual(v.exitCode, 0); + const line = formatIncompleteLine(v, { zh: true }); + assert.match(line, /^未完成：.*npm run check.*；卡在：/); +} +{ + // A failed `npm run check` does not back the claim either. + const t = createRunCompletionTracker(); + execEvents(t, '1', 'npm run check', 'format:check failed\nexit code 1'); + assert.deepEqual(findUnverifiedCheckClaims('npm run check passes.', t), ['check']); + const v = assessRunCompletion({ tracker: t, finalText: 'npm run check passes.', zh: true }); + assert.match(formatIncompleteLine(v, { zh: true }), /卡在：exec: npm run check/); +} +{ + // Really ran and passed: claim is fine, run is complete, exit 0. + const t = createRunCompletionTracker(); + execEvents(t, '1', 'npm run check', 'exit code 0'); + noteRunCompletionEvent(t, { + type: 'tool_start', + toolName: 'edit_file', + toolCallId: 'e', + input: { path: 'src/a.ts' }, + }); + noteRunCompletionEvent(t, { + type: 'tool_end', + toolName: 'edit_file', + toolCallId: 'e', + result: 'ok', + isError: false, + }); + const v = assessRunCompletion({ + tracker: t, + finalText: 'All checks are green.', + request: 'fix src/a.ts', + }); + assert.equal(v.status, 'complete'); + assert.equal(v.exitCode, 0); + assert.equal(formatIncompleteLine(v), ''); + // Negated statements are not claims. + assert.deepEqual( + findUnverifiedCheckClaims( + 'I did not run tests; checks not green yet.', + createRunCompletionTracker() + ), + [] + ); +} +{ + // Edit task, zero files changed → incomplete. + const v = assessRunCompletion({ + tracker: createRunCompletionTracker(), + finalText: 'Done.', + request: '修复 src/cli/hooks.ts 的超时问题', + zh: true, + }); + assert.equal(v.status, 'incomplete'); + assert.match( + formatIncompleteLine(v, { zh: true }), + /^未完成：任务需要改动文件，但本次运行没有修改任何文件/ + ); + // Pure questions do not need edits. + assert.equal( + assessRunCompletion({ + tracker: createRunCompletionTracker(), + finalText: 'x', + request: '怎么修复这个问题？', + }).status, + 'complete' + ); +} + +// --- Integration: runOneShot text mode (what `moss -p` uses) -------------- +async function runPrint(name, turns, prompt, registerTools) { + const ws = await fs.mkdtemp(path.join(os.tmpdir(), `moss-o98-${name}-`)); + const agent = new MossAgent({ + llmProvider: createMockTranscriptProvider(name, 'O98', turns), + sessionStore: new InMemorySessionStore(), + model: name, + workspaceDir: ws, + baseSystemPrompt: 'Test.', + domainPrompt: false, + includeAgentBehaviorPrompt: false, + enableSteering: false, + maxAgentTurns: 6, + }); + registerTools?.(agent); + let out = ''; + const previous = process.exitCode; + process.exitCode = undefined; + await runOneShot(agent, prompt, { + sessionKey: `o98-${name}`, + outputFormat: 'text', + headless: true, + cwd: ws, + stdout: { write: (s) => ((out += s), true) }, + }); + const code = process.exitCode; + process.exitCode = previous; + await fs.rm(ws, { recursive: true, force: true }); + return { out, code }; +} + +const prevLang = process.env.MOSS_LANG; +process.env.MOSS_LANG = 'zh'; +try { + // O14 run2 shape: lint ran, format:check never ran, answer says "All checks are green". + const r = await runPrint( + 'false-green', + [ + { toolCalls: [{ name: 'shell_probe', input: { command: 'npm run lint' } }] }, + { text: 'Updated the hook. All checks are green.' }, + ], + 'Check the repo lint status and report.', + (agent) => + agent.tools.register({ + name: 'shell_probe', + description: 'probe', + metadata: { sideEffectClass: 'readonly' }, + inputSchema: { type: 'object', properties: { command: { type: 'string' } } }, + async execute() { + return 'exit code 0'; + }, + }) + ); + assert.notEqual(r.code ?? 0, 0, `false "all green" must not exit 0 (got ${r.code})`); + const last = r.out.trimEnd().split('\n').at(-1); + assert.match(last, /^(未完成：|Incomplete: )/, `last line must say incomplete, got: ${last}`); + assert.match(last, /npm run check/); + + // kimi-k3 r2 shape: edit task, no tool calls, "done" → non-zero + 未完成 line. + const k = await runPrint( + 'no-edit', + [{ text: 'Done, the fix is in place.' }], + 'Fix the timeout clamp in src/cli/hooks.ts' + ); + assert.notEqual(k.code ?? 0, 0, 'edit task with zero changes must not exit 0'); + assert.match(k.out.trimEnd().split('\n').at(-1), /^(未完成：|Incomplete: )/); + + // Honest answer to a question: unchanged, exit 0, no incomplete line. + const ok = await runPrint( + 'honest', + [{ text: 'Lint was not run in this session.' }], + 'What does npm run check do?' + ); + assert.equal(ok.code ?? 0, 0); + assert.doesNotMatch(ok.out, /未完成：|Incomplete: /); +} finally { + if (prevLang === undefined) delete process.env.MOSS_LANG; + else process.env.MOSS_LANG = prevLang; +} + +console.log('run-completion-o98: ok'); From 9baccf9267f7d4c5ab9d2b7f563c61ac9f2eb5a0 Mon Sep 17 00:00:00 2001 From: QiaolongLi1201 Date: Mon, 12 Oct 2026 01:01:33 +0800 Subject: [PATCH 2/3] fix(o98): export stable RunOutcome for O99 - add src/cli/run-outcome.ts: resolveRunOutcome + formatOutcomeLine status completed | incomplete | unverified (false check claims) - oneshot -p path uses run-outcome for exitCode + last line - tracker stays in run-completion; O99 imports from run-outcome - tests cover unverified vs incomplete line prefixes --- src/cli/oneshot.ts | 25 ++---- src/cli/run-completion.ts | 4 +- src/cli/run-outcome.ts | 141 +++++++++++++++++++++++++++++++ test/run-completion-o98.spec.mjs | 62 +++++++------- 4 files changed, 186 insertions(+), 46 deletions(-) create mode 100644 src/cli/run-outcome.ts diff --git a/src/cli/oneshot.ts b/src/cli/oneshot.ts index ef8fb68b..6efdf8f4 100644 --- a/src/cli/oneshot.ts +++ b/src/cli/oneshot.ts @@ -29,12 +29,8 @@ import { buildGitStatusSnapshot } from '../context/git-status-snapshot.js'; import { acceptanceShell } from '../core/task/acceptance-command.js'; import { runStopHooks, type StopHookResult } from './hooks.js'; import { runProcess } from '../utils/run-process.js'; -import { - assessRunCompletion, - createRunCompletionTracker, - formatIncompleteLine, - noteRunCompletionEvent, -} from './run-completion.js'; +import { createRunCompletionTracker, noteRunCompletionEvent } from './run-completion.js'; +import { formatOutcomeLine, resolveRunOutcome } from './run-outcome.js'; /** Format the oneshot exit warning when background work outlives the wait window. */ export function formatOneshotStillRunningBackgroundNotice( @@ -698,13 +694,11 @@ export async function runOneShot( } } - // O98: never end a text `--print` run on a false success. If the final - // answer claims checks that never (successfully) ran, or the task needed - // edits and none happened, exit non-zero and make the LAST stdout line say - // what is unfinished and where it got stuck. + // O98: never end a text `--print` run on a false success. Resolve a stable + // RunOutcome (completed | incomplete | unverified); O99 reads the same API. if (outputFormat === 'text') { - const zh = isZhLocale(); - const verdict = assessRunCompletion({ + const lang = isZhLocale() ? 'zh' : 'en'; + const outcome = resolveRunOutcome({ tracker: completionTracker, finalText: typeof finalResult?.result === 'string' ? finalResult.result : state.finalText, request: message, @@ -712,11 +706,10 @@ export async function runOneShot( ? { runError: finalResult.error ?? String(finalResult.subtype) } : {}), ...(typeof process.exitCode === 'number' ? { errorExitCode: process.exitCode } : {}), - zh, }); - if (verdict.status === 'incomplete') { - process.exitCode = verdict.exitCode; - stdout.write(`\n${formatIncompleteLine(verdict, { zh })}\n`); + if (outcome.status !== 'completed') { + process.exitCode = outcome.exitCode; + stdout.write(`\n${formatOutcomeLine(outcome, lang)}\n`); } } diff --git a/src/cli/run-completion.ts b/src/cli/run-completion.ts index 8daefd97..27d0615a 100644 --- a/src/cli/run-completion.ts +++ b/src/cli/run-completion.ts @@ -16,7 +16,9 @@ * - {@link formatIncompleteLine} renders the mandatory last line * (`未完成：… ；卡在：…` / `Incomplete: …; stuck at: …`). * - * O99 (progress) reuses the verdict/exit code instead of re-deriving it. + * O99 (progress) must import the stable bilingual outcome from + * {@link ./run-outcome.js} (`resolveRunOutcome` / `formatOutcomeLine`) rather + * than calling these helpers directly. */ import type { MossAgentEvent } from '../core/index.js'; import { ExitCode } from './exit-codes.js'; diff --git a/src/cli/run-outcome.ts b/src/cli/run-outcome.ts new file mode 100644 index 00000000..6a7f1614 --- /dev/null +++ b/src/cli/run-outcome.ts @@ -0,0 +1,141 @@ +/** + * O98 → O99 stable run-outcome surface. + * + * Tracker / claim detection stay in {@link ./run-completion.js}; this module + * maps them to a single bilingual outcome O99 can read without re-deriving: + * + * { status: 'completed' | 'incomplete' | 'unverified', reason, reasonZh, exitCode } + * + * - `completed` — honest finish + * - `incomplete` — unfinished work (no edits when expected, run errors, …) + * - `unverified` — final answer claimed checks passed without a real pass + * (kept distinct from `incomplete` so O99 / UX can treat false-green specially) + */ +import { ExitCode } from './exit-codes.js'; +import { + createRunCompletionTracker, + findUnverifiedCheckClaims, + noteRunCompletionEvent, + requestExpectsFileChanges, + type RunCompletionTracker, + type VerificationCheck, +} from './run-completion.js'; + +export type { RunCompletionTracker, VerificationCheck }; +export { createRunCompletionTracker, noteRunCompletionEvent }; + +export type RunOutcomeStatus = 'completed' | 'incomplete' | 'unverified'; +export type OutcomeLang = 'zh' | 'en'; + +export interface RunOutcome { + status: RunOutcomeStatus; + /** English reason; empty when completed. */ + reason: string; + /** Chinese reason; empty when completed. */ + reasonZh: string; + exitCode: number; + /** Where the run got stuck (English), when known. */ + stuckAt?: string; + /** Where the run got stuck (Chinese), when known. */ + stuckAtZh?: string; +} + +export interface ResolveRunOutcomeInput { + tracker: RunCompletionTracker; + finalText: string; + request?: string; + runError?: string; + errorExitCode?: number; +} + +const CHECK_LABEL: Record = { + check: 'npm run check', + format: 'format:check', + lint: 'lint', + typecheck: 'typecheck', + test: 'test', +}; + +function joinReasons(parts: string[], zh: boolean): string { + return parts + .filter(Boolean) + .join(zh ? '；' : '; ') + .replace(/\s+/g, ' ') + .trim(); +} + +/** Resolve the stable O98/O99 outcome from a filled tracker + final answer. */ +export function resolveRunOutcome(input: ResolveRunOutcomeInput): RunOutcome { + const { tracker } = input; + const unverifiedClaims = findUnverifiedCheckClaims(input.finalText, tracker); + const reasonsEn: string[] = []; + const reasonsZh: string[] = []; + + if (input.runError) { + reasonsEn.push(input.runError); + reasonsZh.push(input.runError); + } + if (unverifiedClaims.length > 0) { + const labels = unverifiedClaims.map((c) => CHECK_LABEL[c]).join(', '); + const failedRun = unverifiedClaims.some((c) => tracker.checks.get(c)); + reasonsEn.push( + `final answer claims checks passed, but this run did not actually pass: ${labels}` + ); + reasonsZh.push( + `收尾声称检查已通过，但本次运行${failedRun ? '中这些检查未通过或' : ''}未实际跑过：${labels}` + ); + } + if (input.request && requestExpectsFileChanges(input.request) && !tracker.wroteFiles) { + reasonsEn.push('the task needed file changes, but no file was modified'); + reasonsZh.push('任务需要改动文件，但本次运行没有修改任何文件'); + } + + if (reasonsEn.length === 0) { + return { status: 'completed', reason: '', reasonZh: '', exitCode: ExitCode.SUCCESS }; + } + + // Prefer `unverified` when the distinctive O98 failure (false check claim) is + // present; otherwise `incomplete` (no edits / prior run error / …). + const status: RunOutcomeStatus = unverifiedClaims.length > 0 ? 'unverified' : 'incomplete'; + + let stuckAt: string | undefined; + let stuckAtZh: string | undefined; + if (tracker.lastFailedStep) { + stuckAt = tracker.lastFailedStep; + stuckAtZh = tracker.lastFailedStep; + } else if (unverifiedClaims.length > 0) { + const labels = unverifiedClaims.map((c) => CHECK_LABEL[c]).join(', '); + stuckAt = `verification (did not run ${labels})`; + stuckAtZh = `验证（未运行 ${labels}）`; + } else if (tracker.lastStep) { + stuckAt = tracker.lastStep; + stuckAtZh = tracker.lastStep; + } + + const exitCode = + input.errorExitCode && input.errorExitCode !== 0 ? input.errorExitCode : ExitCode.INCOMPLETE; + + return { + status, + reason: joinReasons(reasonsEn, false), + reasonZh: joinReasons(reasonsZh, true), + exitCode, + ...(stuckAt ? { stuckAt } : {}), + ...(stuckAtZh ? { stuckAtZh } : {}), + }; +} + +/** + * Mandatory last line for `-p` / O99 when the run is not honestly complete. + * Returns '' for `completed`. + */ +export function formatOutcomeLine(outcome: RunOutcome, lang: OutcomeLang): string { + if (outcome.status === 'completed') return ''; + const zh = lang === 'zh'; + const reason = (zh ? outcome.reasonZh : outcome.reason) || (zh ? '未知原因' : 'unknown reason'); + const stuck = (zh ? outcome.stuckAtZh : outcome.stuckAt) ?? (zh ? '未知' : 'unknown'); + if (outcome.status === 'unverified') { + return zh ? `未核实：${reason}；卡在：${stuck}` : `Unverified: ${reason}; stuck at: ${stuck}`; + } + return zh ? `未完成：${reason}；卡在：${stuck}` : `Incomplete: ${reason}; stuck at: ${stuck}`; +} diff --git a/test/run-completion-o98.spec.mjs b/test/run-completion-o98.spec.mjs index af34f839..01dbab9b 100644 --- a/test/run-completion-o98.spec.mjs +++ b/test/run-completion-o98.spec.mjs @@ -3,6 +3,9 @@ * O98: a headless `moss -p` run must not end on a false success. * O14 run2 said "All checks are green" without running format:check (npm run * check was red); kimi-k3 r2 changed nothing and exited 0. + * + * Stable O99 surface is `run-outcome.ts` (`resolveRunOutcome` / + * `formatOutcomeLine`); tracker logic stays in `run-completion.ts`. */ import assert from 'node:assert/strict'; import fs from 'node:fs/promises'; @@ -12,12 +15,11 @@ import path from 'node:path'; import { runOneShot } from '../dist/cli/oneshot.js'; import { ExitCode } from '../dist/cli/exit-codes.js'; import { - assessRunCompletion, createRunCompletionTracker, findUnverifiedCheckClaims, - formatIncompleteLine, noteRunCompletionEvent, } from '../dist/cli/run-completion.js'; +import { formatOutcomeLine, resolveRunOutcome } from '../dist/cli/run-outcome.js'; import { MossAgent } from '../dist/core/agent/moss-agent.js'; import { InMemorySessionStore } from '../dist/core/session/session.js'; import { createMockTranscriptProvider } from './e2e/mock-transcript-provider.mjs'; @@ -38,7 +40,7 @@ function execEvents(tracker, id, command, result, isError = false) { }); } -// --- Unit: claims vs. real runs ------------------------------------------- +// --- Unit: claims vs. real runs + stable RunOutcome ------------------------ { const t = createRunCompletionTracker(); execEvents(t, '1', 'npm run lint', 'exit code 0'); @@ -47,23 +49,27 @@ function execEvents(tracker, id, command, result, isError = false) { assert.deepEqual(findUnverifiedCheckClaims('Lint passes and typecheck is clean.', t), []); assert.deepEqual(findUnverifiedCheckClaims('prettier --check passed', t), ['format']); assert.deepEqual(findUnverifiedCheckClaims('所有检查都通过了', t), ['check']); - const v = assessRunCompletion({ tracker: t, finalText: 'All checks are green.', zh: true }); - assert.equal(v.status, 'incomplete'); - assert.equal(v.exitCode, ExitCode.INCOMPLETE); - assert.notEqual(v.exitCode, 0); - const line = formatIncompleteLine(v, { zh: true }); - assert.match(line, /^未完成：.*npm run check.*；卡在：/); + + const outcome = resolveRunOutcome({ tracker: t, finalText: 'All checks are green.' }); + assert.equal(outcome.status, 'unverified'); + assert.equal(outcome.exitCode, ExitCode.INCOMPLETE); + assert.notEqual(outcome.exitCode, 0); + assert.match(outcome.reason, /npm run check/); + assert.match(outcome.reasonZh, /npm run check/); + assert.match(formatOutcomeLine(outcome, 'zh'), /^未核实：.*npm run check.*；卡在：/); + assert.match(formatOutcomeLine(outcome, 'en'), /^Unverified: .*npm run check.*stuck at:/); } { - // A failed `npm run check` does not back the claim either. + // A failed `npm run check` does not back the claim either → unverified. const t = createRunCompletionTracker(); execEvents(t, '1', 'npm run check', 'format:check failed\nexit code 1'); assert.deepEqual(findUnverifiedCheckClaims('npm run check passes.', t), ['check']); - const v = assessRunCompletion({ tracker: t, finalText: 'npm run check passes.', zh: true }); - assert.match(formatIncompleteLine(v, { zh: true }), /卡在：exec: npm run check/); + const outcome = resolveRunOutcome({ tracker: t, finalText: 'npm run check passes.' }); + assert.equal(outcome.status, 'unverified'); + assert.match(formatOutcomeLine(outcome, 'zh'), /卡在：exec: npm run check/); } { - // Really ran and passed: claim is fine, run is complete, exit 0. + // Really ran and passed: completed, exit 0, empty line. const t = createRunCompletionTracker(); execEvents(t, '1', 'npm run check', 'exit code 0'); noteRunCompletionEvent(t, { @@ -79,15 +85,14 @@ function execEvents(tracker, id, command, result, isError = false) { result: 'ok', isError: false, }); - const v = assessRunCompletion({ + const outcome = resolveRunOutcome({ tracker: t, finalText: 'All checks are green.', request: 'fix src/a.ts', }); - assert.equal(v.status, 'complete'); - assert.equal(v.exitCode, 0); - assert.equal(formatIncompleteLine(v), ''); - // Negated statements are not claims. + assert.equal(outcome.status, 'completed'); + assert.equal(outcome.exitCode, 0); + assert.equal(formatOutcomeLine(outcome, 'zh'), ''); assert.deepEqual( findUnverifiedCheckClaims( 'I did not run tests; checks not green yet.', @@ -97,26 +102,25 @@ function execEvents(tracker, id, command, result, isError = false) { ); } { - // Edit task, zero files changed → incomplete. - const v = assessRunCompletion({ + // Edit task, zero files changed → incomplete (not unverified). + const outcome = resolveRunOutcome({ tracker: createRunCompletionTracker(), finalText: 'Done.', request: '修复 src/cli/hooks.ts 的超时问题', - zh: true, }); - assert.equal(v.status, 'incomplete'); + assert.equal(outcome.status, 'incomplete'); assert.match( - formatIncompleteLine(v, { zh: true }), + formatOutcomeLine(outcome, 'zh'), /^未完成：任务需要改动文件，但本次运行没有修改任何文件/ ); - // Pure questions do not need edits. + assert.match(formatOutcomeLine(outcome, 'en'), /^Incomplete: the task needed file changes/); assert.equal( - assessRunCompletion({ + resolveRunOutcome({ tracker: createRunCompletionTracker(), finalText: 'x', request: '怎么修复这个问题？', }).status, - 'complete' + 'completed' ); } @@ -175,7 +179,7 @@ try { ); assert.notEqual(r.code ?? 0, 0, `false "all green" must not exit 0 (got ${r.code})`); const last = r.out.trimEnd().split('\n').at(-1); - assert.match(last, /^(未完成：|Incomplete: )/, `last line must say incomplete, got: ${last}`); + assert.match(last, /^(未核实：|Unverified: )/, `last line must say unverified, got: ${last}`); assert.match(last, /npm run check/); // kimi-k3 r2 shape: edit task, no tool calls, "done" → non-zero + 未完成 line. @@ -187,14 +191,14 @@ try { assert.notEqual(k.code ?? 0, 0, 'edit task with zero changes must not exit 0'); assert.match(k.out.trimEnd().split('\n').at(-1), /^(未完成：|Incomplete: )/); - // Honest answer to a question: unchanged, exit 0, no incomplete line. + // Honest answer to a question: unchanged, exit 0, no incomplete/unverified line. const ok = await runPrint( 'honest', [{ text: 'Lint was not run in this session.' }], 'What does npm run check do?' ); assert.equal(ok.code ?? 0, 0); - assert.doesNotMatch(ok.out, /未完成：|Incomplete: /); + assert.doesNotMatch(ok.out, /未完成：|未核实：|Incomplete: |Unverified: /); } finally { if (prevLang === undefined) delete process.env.MOSS_LANG; else process.env.MOSS_LANG = prevLang; From 665c9ee9532b19d4cb837e72d7bdf64c42cd2455 Mon Sep 17 00:00:00 2001 From: QiaolongLi1201 Date: Mon, 12 Oct 2026 01:26:19 +0800 Subject: [PATCH 3/3] =?UTF-8?q?fix(o98):=20VERIFY=20re-check,=20=E5=AE=8C?= =?UTF-8?q?=E6=88=90=20line,=20=E6=9C=AA=E9=AA=8C=E8=AF=81,=20Stop=20non-z?= =?UTF-8?q?ero?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebase already on 58c9e3d6. Close QA/UX gaps on #94: - formatOutcomeLine always emits a conclusion (完成 / 未完成 / 未验证 / 验证失败 / 被拦); drop 未核实 wording - status: completed | incomplete | verify_failed | blocked | unverified (soft unverified = exit 0; VERIFY_CMD / Stop = non-zero) - VERIFY_CMD always recorded on tracker; continue once then re-verify; timeout/spawn not swallowed - headless Stop hook blocked → non-zero + 被拦 line - TUI shows outcome under the done stamp - tests cover A–E plus completed / blocked / TUI line --- src/cli/oneshot.ts | 156 +++++++++++------- src/cli/run-completion.ts | 40 ++++- src/cli/run-outcome.ts | 191 ++++++++++++++++----- src/cli/tui/app.ts | 41 ++++- src/cli/tui/transcript.ts | 11 +- test/run-completion-o98.spec.mjs | 275 +++++++++++++++++++++++++------ 6 files changed, 557 insertions(+), 157 deletions(-) diff --git a/src/cli/oneshot.ts b/src/cli/oneshot.ts index 6efdf8f4..8fea31ea 100644 --- a/src/cli/oneshot.ts +++ b/src/cli/oneshot.ts @@ -28,8 +28,12 @@ import { configuredBaseUrl, pricingOverridesFromConfig } from './model-pricing.j import { buildGitStatusSnapshot } from '../context/git-status-snapshot.js'; import { acceptanceShell } from '../core/task/acceptance-command.js'; import { runStopHooks, type StopHookResult } from './hooks.js'; -import { runProcess } from '../utils/run-process.js'; -import { createRunCompletionTracker, noteRunCompletionEvent } from './run-completion.js'; +import { ProcessError, runProcess } from '../utils/run-process.js'; +import { + createRunCompletionTracker, + noteGoalVerifyResult, + noteRunCompletionEvent, +} from './run-completion.js'; import { formatOutcomeLine, resolveRunOutcome } from './run-outcome.js'; /** Format the oneshot exit warning when background work outlives the wait window. */ @@ -630,17 +634,14 @@ export async function runOneShot( } // Headless goal-verify loop (MOSS_GOAL_VERIFY_LOOP=1): the headless - // counterpart of `/goal --accept`. After the primary run, execute - // MOSS_GOAL_VERIFY_CMD; a non-zero exit triggers ONE continuation turn - // carrying the failure evidence. + // counterpart of `/goal --accept`. After every run (including the one + // continuation turn), execute MOSS_GOAL_VERIFY_CMD and record the result on + // the tracker. A non-zero exit on the first pass triggers ONE continuation; + // the continuation always re-verifies. Spawn/timeout failures are + // verify_failed — never swallowed (O98). const verifyDepth = options.goalVerifyDepth ?? 0; - if ( - process.env.MOSS_GOAL_VERIFY_LOOP === '1' && - process.env.MOSS_GOAL_VERIFY_CMD && - verifyDepth < 1 - ) { - // Task OS M6: mirror the verify verdict onto the latest live task so - // headless goal runs land in unified task history too. + const verifyCmd = process.env.MOSS_GOAL_VERIFY_CMD; + if (process.env.MOSS_GOAL_VERIFY_LOOP === '1' && verifyCmd) { const mirrorGoalVerdict = async (passed: boolean, exitCode: number, tail: string) => { try { const { findLatestLiveTaskSnapshot, emitAcceptanceLifecycle } = @@ -661,41 +662,95 @@ export async function runOneShot( } }; try { - const shell = acceptanceShell(process.env.MOSS_GOAL_VERIFY_CMD); - const verify = await runProcess(shell.cmd, { + const shell = acceptanceShell(verifyCmd); + // runProcess resolves only on exit 0; non-zero / timeout / spawn reject. + await runProcess(shell.cmd, { args: shell.args, cwd: workspaceDir, timeout: 5 * 60_000, ...(shell.windowsVerbatimArguments ? { windowsVerbatimArguments: true } : {}), }); - if (verify.exitCode !== 0) { - const tail = `${verify.stdout}\n${verify.stderr}`.trim().slice(-2000); - await mirrorGoalVerdict(false, verify.exitCode, tail); + noteGoalVerifyResult(completionTracker, { + command: verifyCmd, + passed: true, + exitCode: 0, + }); + await mirrorGoalVerdict(true, 0, ''); + } catch (err) { + const pe = err instanceof ProcessError ? err : undefined; + const timedOut = pe?.timedOut === true; + const exitCode = pe && typeof pe.exitCode === 'number' ? pe.exitCode : 1; + const tail = pe ? `${pe.stdout}\n${pe.stderr}`.trim().slice(-2000) : ''; + const detail = pe?.message || (err instanceof Error ? err.message : String(err)); + // Infrastructure failure (timeout / could not start): record and stop — no continue. + const infraFailure = timedOut || !pe; + if (infraFailure) { + noteGoalVerifyResult(completionTracker, { + command: verifyCmd, + passed: false, + exitCode, + error: detail, + timedOut, + spawnFailed: !timedOut, + }); + process.stderr.write(`[goal-verify] ${detail}\n`); + } else { + // Ordinary non-zero exit: record, continue once, re-verify after. + noteGoalVerifyResult(completionTracker, { + command: verifyCmd, + passed: false, + exitCode, + ...(tail ? { error: tail.slice(0, 400) } : { error: detail }), + }); + await mirrorGoalVerdict(false, exitCode, tail || detail); + if (verifyDepth < 1) { + process.stderr.write( + `[goal-verify] command failed (exit ${exitCode}); running one continuation turn\n` + ); + return runOneShot( + agent, + [ + `[verification failed] The verification command exited ${exitCode}.`, + 'Fix the underlying cause — do not work around or edit the verification command.', + '', + 'Last output (tail):', + tail || detail || '(no output)', + '', + 'Complete the original task so the verification passes.', + ].join('\n'), + { ...options, goalVerifyDepth: verifyDepth + 1 } + ); + } process.stderr.write( - `[goal-verify] command failed (exit ${verify.exitCode}); running one continuation turn\n` - ); - return runOneShot( - agent, - [ - `[verification failed] The verification command exited ${verify.exitCode}.`, - 'Fix the underlying cause — do not work around or edit the verification command.', - '', - 'Last output (tail):', - tail || '(no output)', - '', - 'Complete the original task so the verification passes.', - ].join('\n'), - { ...options, goalVerifyDepth: verifyDepth + 1 } + `[goal-verify] command still failing after continuation (exit ${exitCode})\n` ); } - await mirrorGoalVerdict(true, 0, ''); - } catch { - /* verification infrastructure failures never break the run */ } } + // Stop lifecycle hook: fires after every completed run. A blocking non-zero + // exit vetoes the stop — the REPL turns that into one forced continuation + // turn; headless also exits non-zero (O98). + let stopResult: StopHookResult = { blocked: false }; + try { + stopResult = await runStopHooks({ + sessionKey, + ...(finalResult?.subtype !== undefined ? { stopReason: String(finalResult.subtype) } : {}), + ...(typeof finalResult?.result === 'string' ? { response: finalResult.result } : {}), + }); + if (stopResult.blocked) { + process.stderr.write( + `[hooks] ${stopResult.reason ?? 'Stop hook blocked the run ending.'}${ + options.headless ? '' : ' (interactive mode reports; host may force-continue)' + }\n` + ); + } + } catch { + stopResult = { blocked: false }; + } + // O98: never end a text `--print` run on a false success. Resolve a stable - // RunOutcome (completed | incomplete | unverified); O99 reads the same API. + // RunOutcome; always print the conclusion line (完成 / 未完成 / 未验证 / …). if (outputFormat === 'text') { const lang = isZhLocale() ? 'zh' : 'en'; const outcome = resolveRunOutcome({ @@ -706,29 +761,20 @@ export async function runOneShot( ? { runError: finalResult.error ?? String(finalResult.subtype) } : {}), ...(typeof process.exitCode === 'number' ? { errorExitCode: process.exitCode } : {}), + ...(stopResult.blocked + ? { blocked: true, ...(stopResult.reason ? { blockedReason: stopResult.reason } : {}) } + : {}), }); - if (outcome.status !== 'completed') { + const line = formatOutcomeLine(outcome, lang); + if (line) stdout.write(`\n${line}\n`); + if (outcome.exitCode !== 0) { process.exitCode = outcome.exitCode; - stdout.write(`\n${formatOutcomeLine(outcome, lang)}\n`); + } else if (options.headless && stopResult.blocked) { + process.exitCode = ExitCode.INCOMPLETE; } + } else if (options.headless && stopResult.blocked) { + process.exitCode = ExitCode.INCOMPLETE; } - // Stop lifecycle hook: fires after every completed run. A blocking non-zero - // exit vetoes the stop — the REPL turns that into one forced continuation - // turn; headless reports it on stderr. - try { - const stop = await runStopHooks({ - sessionKey, - ...(finalResult?.subtype !== undefined ? { stopReason: String(finalResult.subtype) } : {}), - ...(typeof finalResult?.result === 'string' ? { response: finalResult.result } : {}), - }); - if (stop.blocked) { - process.stderr.write( - `[hooks] ${stop.reason ?? 'Stop hook blocked the run ending.'} (headless mode reports but does not force-continue)\n` - ); - } - return stop; - } catch { - return { blocked: false }; - } + return stopResult; } diff --git a/src/cli/run-completion.ts b/src/cli/run-completion.ts index 27d0615a..0dce2c33 100644 --- a/src/cli/run-completion.ts +++ b/src/cli/run-completion.ts @@ -30,6 +30,16 @@ export interface CheckRun { passed: boolean; } +export interface GoalVerifyResult { + command: string; + passed: boolean; + exitCode?: number; + /** Short error / stdout+stderr tail for the outcome line. */ + error?: string; + timedOut?: boolean; + spawnFailed?: boolean; +} + export interface RunCompletionTracker { /** Latest run of each check kind in this run (later runs overwrite earlier ones). */ checks: Map; @@ -39,6 +49,8 @@ export interface RunCompletionTracker { lastFailedStep?: string; /** Last tool step that started, e.g. `multi_edit: src/cli/hooks.ts`. */ lastStep?: string; + /** Latest MOSS_GOAL_VERIFY_CMD result (O98 — must feed the tracker). */ + goalVerify?: GoalVerifyResult; /** @internal pending tool_start inputs keyed by toolCallId. */ pending: Map }>; } @@ -148,6 +160,28 @@ export function noteRunCompletionEvent(tracker: RunCompletionTracker, event: Mos } } +/** + * Record a MOSS_GOAL_VERIFY_CMD result on the tracker and mirror it into the + * check map so claim detection sees the real verification outcome. + */ +export function noteGoalVerifyResult( + tracker: RunCompletionTracker, + result: GoalVerifyResult +): void { + tracker.goalVerify = result; + const step = `VERIFY_CMD: ${result.command.replace(/\s+/g, ' ').trim().slice(0, 120)}`; + tracker.lastStep = step; + if (!result.passed) tracker.lastFailedStep = step; + for (const check of checksForCommand(result.command)) { + tracker.checks.set(check, { command: result.command, passed: result.passed }); + } + // A bare `npm run check`-style verify still covers the umbrella "check" claim + // even when checksForCommand finds nothing (e.g. custom scripts). + if (!checksForCommand(result.command).length) { + tracker.checks.set('check', { command: result.command, passed: result.passed }); + } +} + const CLAIM_PATTERNS: Array<{ re: RegExp; requires: VerificationCheck[] }> = [ { re: /\ball\s+(?:the\s+)?checks?\s+(?:are\s+|is\s+)?(?:green|pass(?:ed|ing)?|clean|ok)\b|(?:全部|所有)(?:的)?检查(?:都|均|全部)?(?:已)?(?:通过|是绿的|绿了|变绿)|\bnpm\s+run\s+check\b[^.\n。]*\b(?:pass(?:es|ed)?|green|succeed(?:s|ed)?|clean)\b|npm\s+run\s+check[^。\n]*(?:通过|绿)/i, @@ -233,11 +267,7 @@ export function assessRunCompletion(input: AssessRunCompletionInput): RunComplet ); } if (input.request && requestExpectsFileChanges(input.request) && !tracker.wroteFiles) { - reasons.push( - zh - ? '任务需要改动文件，但本次运行没有修改任何文件' - : 'the task needed file changes, but no file was modified' - ); + reasons.push(zh ? '没有改动任何文件' : 'no files were modified'); } if (reasons.length === 0) { diff --git a/src/cli/run-outcome.ts b/src/cli/run-outcome.ts index 6a7f1614..a07ed5ad 100644 --- a/src/cli/run-outcome.ts +++ b/src/cli/run-outcome.ts @@ -4,34 +4,44 @@ * Tracker / claim detection stay in {@link ./run-completion.js}; this module * maps them to a single bilingual outcome O99 can read without re-deriving: * - * { status: 'completed' | 'incomplete' | 'unverified', reason, reasonZh, exitCode } + * { status: 'completed' | 'incomplete' | 'verify_failed' | 'blocked' | 'unverified', + * reason, reasonZh, exitCode } * - * - `completed` — honest finish - * - `incomplete` — unfinished work (no edits when expected, run errors, …) - * - `unverified` — final answer claimed checks passed without a real pass - * (kept distinct from `incomplete` so O99 / UX can treat false-green specially) + * - `completed` — honest finish (formatOutcomeLine always returns「完成：…」) + * - `incomplete` — unfinished work (no edits when expected, run errors, …) → non-0 + * - `verify_failed` — MOSS_GOAL_VERIFY_CMD failed / timed out / would not start → non-0 + * - `blocked` — Stop hook blocked the run ending → non-0 + * - `unverified` — final answer claimed checks passed without a real pass, and no + * VERIFY_CMD hard failure (exit 0 so pure Q&A is not punished; last line「未验证」) */ import { ExitCode } from './exit-codes.js'; import { createRunCompletionTracker, findUnverifiedCheckClaims, + noteGoalVerifyResult, noteRunCompletionEvent, requestExpectsFileChanges, + type GoalVerifyResult, type RunCompletionTracker, type VerificationCheck, } from './run-completion.js'; -export type { RunCompletionTracker, VerificationCheck }; -export { createRunCompletionTracker, noteRunCompletionEvent }; +export type { GoalVerifyResult, RunCompletionTracker, VerificationCheck }; +export { createRunCompletionTracker, noteGoalVerifyResult, noteRunCompletionEvent }; -export type RunOutcomeStatus = 'completed' | 'incomplete' | 'unverified'; +export type RunOutcomeStatus = + | 'completed' + | 'incomplete' + | 'verify_failed' + | 'blocked' + | 'unverified'; export type OutcomeLang = 'zh' | 'en'; export interface RunOutcome { status: RunOutcomeStatus; - /** English reason; empty when completed. */ + /** English reason. */ reason: string; - /** Chinese reason; empty when completed. */ + /** Chinese reason. */ reasonZh: string; exitCode: number; /** Where the run got stuck (English), when known. */ @@ -46,6 +56,9 @@ export interface ResolveRunOutcomeInput { request?: string; runError?: string; errorExitCode?: number; + /** Stop hook blocked the run ending. */ + blocked?: boolean; + blockedReason?: string; } const CHECK_LABEL: Record = { @@ -64,56 +77,146 @@ function joinReasons(parts: string[], zh: boolean): string { .trim(); } +function goalVerifyFailReason( + tracker: RunCompletionTracker +): { en: string; zh: string; stuckEn: string; stuckZh: string } | undefined { + const gv = tracker.goalVerify; + if (!gv || gv.passed) return undefined; + const cmd = gv.command.replace(/\s+/g, ' ').trim().slice(0, 120); + const detail = (gv.error ?? '').replace(/\s+/g, ' ').trim().slice(0, 200); + if (gv.timedOut) { + const en = detail + ? `verification command timed out: ${cmd} (${detail})` + : `verification command timed out: ${cmd}`; + const zh = detail ? `验证命令超时：${cmd}（${detail}）` : `验证命令超时：${cmd}`; + return { en, zh, stuckEn: `VERIFY_CMD timeout: ${cmd}`, stuckZh: `VERIFY_CMD 超时：${cmd}` }; + } + if (gv.spawnFailed) { + const en = detail + ? `verification command failed to start: ${cmd} (${detail})` + : `verification command failed to start: ${cmd}`; + const zh = detail ? `验证命令未能启动：${cmd}（${detail}）` : `验证命令未能启动：${cmd}`; + return { + en, + zh, + stuckEn: `VERIFY_CMD spawn failed: ${cmd}`, + stuckZh: `VERIFY_CMD 启动失败：${cmd}`, + }; + } + const tail = detail ? ` — ${detail}` : ''; + const en = `model claimed checks passed, but verification failed: ${cmd} (exit ${gv.exitCode ?? 'non-zero'})${tail}`; + const zh = `模型声称通过，但验证失败：${cmd}（退出码 ${gv.exitCode ?? '非 0'}）${tail}`; + return { + en, + zh, + stuckEn: `VERIFY_CMD: ${cmd}`, + stuckZh: `VERIFY_CMD：${cmd}`, + }; +} + /** Resolve the stable O98/O99 outcome from a filled tracker + final answer. */ export function resolveRunOutcome(input: ResolveRunOutcomeInput): RunOutcome { const { tracker } = input; const unverifiedClaims = findUnverifiedCheckClaims(input.finalText, tracker); const reasonsEn: string[] = []; const reasonsZh: string[] = []; + let status: RunOutcomeStatus | undefined; + let stuckAt: string | undefined; + let stuckAtZh: string | undefined; + + if (input.blocked) { + status = 'blocked'; + const reason = input.blockedReason?.trim() || 'Stop hook blocked the run ending'; + reasonsEn.push(reason); + reasonsZh.push(reason); + stuckAt = 'Stop hook'; + stuckAtZh = 'Stop 钩子'; + } + + // Authoritative VERIFY_CMD success: the accept loop is the completion signal. + // Do not demote to incomplete just because the continuation turn itself + // edited no files (the model may have fixed things in an earlier turn). + if (!status && tracker.goalVerify?.passed === true && !input.runError) { + return { + status: 'completed', + reason: 'verification passed', + reasonZh: '验证已通过', + exitCode: ExitCode.SUCCESS, + }; + } + + const verifyFail = goalVerifyFailReason(tracker); + if (verifyFail) { + status = status ?? 'verify_failed'; + reasonsEn.push(verifyFail.en); + reasonsZh.push(verifyFail.zh); + stuckAt = stuckAt ?? verifyFail.stuckEn; + stuckAtZh = stuckAtZh ?? verifyFail.stuckZh; + } if (input.runError) { + status = status ?? 'incomplete'; reasonsEn.push(input.runError); reasonsZh.push(input.runError); } - if (unverifiedClaims.length > 0) { + + if (input.request && requestExpectsFileChanges(input.request) && !tracker.wroteFiles) { + status = status ?? 'incomplete'; + reasonsEn.push('no files were modified'); + reasonsZh.push('没有改动任何文件'); + } + + if (unverifiedClaims.length > 0 && status !== 'verify_failed') { const labels = unverifiedClaims.map((c) => CHECK_LABEL[c]).join(', '); const failedRun = unverifiedClaims.some((c) => tracker.checks.get(c)); + // Soft: no VERIFY_CMD hard failure — show「未验证」, exit 0 unless another + // hard failure already selected a non-completed status. + if (!status) status = 'unverified'; reasonsEn.push( - `final answer claims checks passed, but this run did not actually pass: ${labels}` + failedRun + ? `claimed checks passed, but they did not pass in this run: ${labels}` + : `claimed checks passed, but no check was run: ${labels}` ); reasonsZh.push( - `收尾声称检查已通过，但本次运行${failedRun ? '中这些检查未通过或' : ''}未实际跑过：${labels}` + failedRun + ? `声称检查已通过，但本次运行未通过：${labels}` + : `声称检查已通过，但没有跑检查：${labels}` ); - } - if (input.request && requestExpectsFileChanges(input.request) && !tracker.wroteFiles) { - reasonsEn.push('the task needed file changes, but no file was modified'); - reasonsZh.push('任务需要改动文件，但本次运行没有修改任何文件'); + if (!stuckAt) { + stuckAt = `verification (did not run ${labels})`; + stuckAtZh = `验证（未运行 ${labels}）`; + } } - if (reasonsEn.length === 0) { - return { status: 'completed', reason: '', reasonZh: '', exitCode: ExitCode.SUCCESS }; + if (!status) { + const passedVerify = tracker.goalVerify?.passed === true; + const reasonEn = passedVerify ? 'verification passed' : 'run finished'; + const reasonZh = passedVerify ? '验证已通过' : '运行结束'; + return { + status: 'completed', + reason: reasonEn, + reasonZh, + exitCode: ExitCode.SUCCESS, + }; } - // Prefer `unverified` when the distinctive O98 failure (false check claim) is - // present; otherwise `incomplete` (no edits / prior run error / …). - const status: RunOutcomeStatus = unverifiedClaims.length > 0 ? 'unverified' : 'incomplete'; - - let stuckAt: string | undefined; - let stuckAtZh: string | undefined; - if (tracker.lastFailedStep) { + if (tracker.lastFailedStep && !stuckAt) { stuckAt = tracker.lastFailedStep; stuckAtZh = tracker.lastFailedStep; - } else if (unverifiedClaims.length > 0) { - const labels = unverifiedClaims.map((c) => CHECK_LABEL[c]).join(', '); - stuckAt = `verification (did not run ${labels})`; - stuckAtZh = `验证（未运行 ${labels}）`; - } else if (tracker.lastStep) { + } else if (tracker.lastStep && !stuckAt) { stuckAt = tracker.lastStep; stuckAtZh = tracker.lastStep; } - const exitCode = - input.errorExitCode && input.errorExitCode !== 0 ? input.errorExitCode : ExitCode.INCOMPLETE; + // Soft unverified alone stays exit 0; everything else is non-zero. + let exitCode = ExitCode.SUCCESS; + if (status === 'unverified' && !input.runError && !input.blocked && !verifyFail) { + exitCode = ExitCode.SUCCESS; + } else if (input.errorExitCode && input.errorExitCode !== 0) { + exitCode = input.errorExitCode; + } else { + exitCode = ExitCode.INCOMPLETE; + } return { status, @@ -126,16 +229,26 @@ export function resolveRunOutcome(input: ResolveRunOutcomeInput): RunOutcome { } /** - * Mandatory last line for `-p` / O99 when the run is not honestly complete. - * Returns '' for `completed`. + * Mandatory last line for `-p` / TUI / O99. + * Always non-empty: completed →「完成：…」; others → matching prefix. */ export function formatOutcomeLine(outcome: RunOutcome, lang: OutcomeLang): string { - if (outcome.status === 'completed') return ''; const zh = lang === 'zh'; const reason = (zh ? outcome.reasonZh : outcome.reason) || (zh ? '未知原因' : 'unknown reason'); const stuck = (zh ? outcome.stuckAtZh : outcome.stuckAt) ?? (zh ? '未知' : 'unknown'); - if (outcome.status === 'unverified') { - return zh ? `未核实：${reason}；卡在：${stuck}` : `Unverified: ${reason}; stuck at: ${stuck}`; + switch (outcome.status) { + case 'completed': + return zh ? `完成：${reason}` : `Completed: ${reason}`; + case 'unverified': + return zh ? `未验证：${reason}` : `Unverified: ${reason}`; + case 'verify_failed': + return zh + ? `验证失败：${reason}；卡在：${stuck}` + : `Verify failed: ${reason}; stuck at: ${stuck}`; + case 'blocked': + return zh ? `被拦：${reason}；卡在：${stuck}` : `Blocked: ${reason}; stuck at: ${stuck}`; + case 'incomplete': + default: + return zh ? `未完成：${reason}；卡在：${stuck}` : `Incomplete: ${reason}; stuck at: ${stuck}`; } - return zh ? `未完成：${reason}；卡在：${stuck}` : `Incomplete: ${reason}; stuck at: ${stuck}`; } diff --git a/src/cli/tui/app.ts b/src/cli/tui/app.ts index 699537d8..821fc16b 100644 --- a/src/cli/tui/app.ts +++ b/src/cli/tui/app.ts @@ -225,6 +225,8 @@ import { import { installTuiLogSink } from './terminal-io.js'; import { noteFrameHeight } from './frame-invariant.js'; import { needsTestHint, type VerifyHintState } from '../verify-hint.js'; +import { createRunCompletionTracker, noteRunCompletionEvent } from '../run-completion.js'; +import { formatOutcomeLine, resolveRunOutcome } from '../run-outcome.js'; import { createViewport, moveTranscript, @@ -1225,8 +1227,10 @@ export function TuiAppRoot({ handle.notify(); const controller = new AbortController(); abortRef.current = controller; + const completionTracker = createRunCompletionTracker(); try { for await (const event of streamTurn(message, controller.signal)) { + noteRunCompletionEvent(completionTracker, event); if (event.type === 'error' && isInterruptEvent(controller, event.error)) { // D-9: the loop reports the user's own abort as an `error` event // ("This operation was aborted"), which used to land in the @@ -1271,16 +1275,41 @@ export function TuiAppRoot({ runStartedAtRef.current = undefined; const halted = controller.signal.aborted; const unverified = !halted && needsTestHint(store.run as VerifyHintState); + // Capture final answer before endRun clears the live buffer (O98 TUI line). + const finalText = [ + ...store.rows + .filter((row) => row.id >= firstRowId && row.kind === 'assistant') + .map((row) => row.text), + store.run.streamingText, + ] + .join('\n') + .trim(); + const outcome = resolveRunOutcome({ + tracker: completionTracker, + finalText, + request: message, + }); + const outcomeLine = formatOutcomeLine(outcome, isZhLocale() ? 'zh' : 'en'); endRun(store, halted); if (startedAt !== undefined) { // Claude leaves a finalizing status line in the transcript; keep the // same shape (`✻ Worked for 5s`) and add the run's token spend. - const summary = renderRunSummary(Date.now() - startedAt, halted, Number.MAX_SAFE_INTEGER, { - input: store.usage.runTokensIn, - output: store.usage.runTokensOut, - }); - const text = summary.find((l) => l.text.trim())?.text.trim(); - if (text) appendRow(store, 'summary', text); + // O98: conclusion (完成 / 未完成 / 未验证) sits on the next dim line. + const summary = renderRunSummary( + Date.now() - startedAt, + halted, + Number.MAX_SAFE_INTEGER, + { + input: store.usage.runTokensIn, + output: store.usage.runTokensOut, + }, + new Date(), + outcomeLine + ); + for (const entry of summary) { + const text = entry.text.trim(); + if (text) appendRow(store, 'summary', text); + } } if (unverified) { appendRow(store, 'summary', tui('edited JS/TS files but did not run tests')); diff --git a/src/cli/tui/transcript.ts b/src/cli/tui/transcript.ts index 4417d2d6..38ba15ca 100644 --- a/src/cli/tui/transcript.ts +++ b/src/cli/tui/transcript.ts @@ -989,7 +989,9 @@ export function renderRunSummary( halted: boolean, width: number, _tokens?: { input: number; output: number }, - now: Date = new Date() + now: Date = new Date(), + /** O98 conclusion line (完成 / 未完成 / 未验证 / …); shown under the done stamp. */ + outcomeLine?: string ): TuiLine[] { const seconds = Math.max(1, Math.round(elapsedMs / 1000)); const verb = runVerb(seconds); @@ -1005,7 +1007,12 @@ export function renderRunSummary( const text = halted ? tui('✻ {verb} for {seconds}s · interrupted', { verb, seconds }) : tui('✻ worked for {seconds}s{doneAt}', { seconds, doneAt }); - return [line(''), line(clip(text, width), { dim: true })]; + const out: TuiLine[] = [line(''), line(clip(text, width), { dim: true })]; + const conclusion = outcomeLine?.trim(); + if (conclusion && !halted) { + out.push(line(clip(conclusion, width), { dim: true })); + } + return out; } // ─── inline information blocks (slash commands, `?`) ───────────────────── diff --git a/test/run-completion-o98.spec.mjs b/test/run-completion-o98.spec.mjs index 01dbab9b..e1092664 100644 --- a/test/run-completion-o98.spec.mjs +++ b/test/run-completion-o98.spec.mjs @@ -1,11 +1,7 @@ #!/usr/bin/env node /** * O98: a headless `moss -p` run must not end on a false success. - * O14 run2 said "All checks are green" without running format:check (npm run - * check was red); kimi-k3 r2 changed nothing and exited 0. - * - * Stable O99 surface is `run-outcome.ts` (`resolveRunOutcome` / - * `formatOutcomeLine`); tracker logic stays in `run-completion.ts`. + * Spec: /workspace/specs/O98.md (A–E + completed line + VERIFY_CMD). */ import assert from 'node:assert/strict'; import fs from 'node:fs/promises'; @@ -13,16 +9,18 @@ import os from 'node:os'; import path from 'node:path'; import { runOneShot } from '../dist/cli/oneshot.js'; -import { ExitCode } from '../dist/cli/exit-codes.js'; import { createRunCompletionTracker, findUnverifiedCheckClaims, + noteGoalVerifyResult, noteRunCompletionEvent, } from '../dist/cli/run-completion.js'; import { formatOutcomeLine, resolveRunOutcome } from '../dist/cli/run-outcome.js'; import { MossAgent } from '../dist/core/agent/moss-agent.js'; import { InMemorySessionStore } from '../dist/core/session/session.js'; import { createMockTranscriptProvider } from './e2e/mock-transcript-provider.mjs'; +import { setLifecycleHookRunner } from '../dist/cli/hooks.js'; +import { renderRunSummary } from '../dist/cli/tui/transcript.js'; function execEvents(tracker, id, command, result, isError = false) { noteRunCompletionEvent(tracker, { @@ -50,26 +48,75 @@ function execEvents(tracker, id, command, result, isError = false) { assert.deepEqual(findUnverifiedCheckClaims('prettier --check passed', t), ['format']); assert.deepEqual(findUnverifiedCheckClaims('所有检查都通过了', t), ['check']); + // E: no VERIFY_CMD, false claim → unverified, exit 0,「未验证」 const outcome = resolveRunOutcome({ tracker: t, finalText: 'All checks are green.' }); assert.equal(outcome.status, 'unverified'); - assert.equal(outcome.exitCode, ExitCode.INCOMPLETE); - assert.notEqual(outcome.exitCode, 0); - assert.match(outcome.reason, /npm run check/); - assert.match(outcome.reasonZh, /npm run check/); - assert.match(formatOutcomeLine(outcome, 'zh'), /^未核实：.*npm run check.*；卡在：/); - assert.match(formatOutcomeLine(outcome, 'en'), /^Unverified: .*npm run check.*stuck at:/); + assert.equal(outcome.exitCode, 0, 'soft unverified must not force non-zero'); + assert.match(formatOutcomeLine(outcome, 'zh'), /^未验证：/); + assert.match(formatOutcomeLine(outcome, 'en'), /^Unverified: /); + assert.doesNotMatch(formatOutcomeLine(outcome, 'zh'), /未核实/); } + { - // A failed `npm run check` does not back the claim either → unverified. + // Failed npm run check + claim → still soft unverified (no VERIFY_CMD field) const t = createRunCompletionTracker(); execEvents(t, '1', 'npm run check', 'format:check failed\nexit code 1'); - assert.deepEqual(findUnverifiedCheckClaims('npm run check passes.', t), ['check']); const outcome = resolveRunOutcome({ tracker: t, finalText: 'npm run check passes.' }); assert.equal(outcome.status, 'unverified'); - assert.match(formatOutcomeLine(outcome, 'zh'), /卡在：exec: npm run check/); + assert.equal(outcome.exitCode, 0); + assert.match(formatOutcomeLine(outcome, 'zh'), /未验证：/); +} + +{ + // A-unit: VERIFY_CMD failed → verify_failed, non-0, never empty completed line + const t = createRunCompletionTracker(); + noteGoalVerifyResult(t, { + command: 'npm run check', + passed: false, + exitCode: 1, + error: 'format:check failed', + }); + const outcome = resolveRunOutcome({ + tracker: t, + finalText: 'All checks are green.', + }); + assert.equal(outcome.status, 'verify_failed'); + assert.notEqual(outcome.exitCode, 0); + assert.match(formatOutcomeLine(outcome, 'zh'), /^验证失败：/); + assert.match( + formatOutcomeLine(outcome, 'zh'), + /模型声称通过，但验证失败|format:check|npm run check/ + ); +} + +{ + // C-unit: verify timeout / spawn → verify_failed + const timed = createRunCompletionTracker(); + noteGoalVerifyResult(timed, { + command: 'sleep 999', + passed: false, + timedOut: true, + error: 'timed out after 1000ms', + }); + const to = resolveRunOutcome({ tracker: timed, finalText: 'done' }); + assert.equal(to.status, 'verify_failed'); + assert.notEqual(to.exitCode, 0); + assert.match(formatOutcomeLine(to, 'zh'), /超时/); + + const spawn = createRunCompletionTracker(); + noteGoalVerifyResult(spawn, { + command: 'no-such-verify-bin', + passed: false, + spawnFailed: true, + error: 'ENOENT', + }); + const sp = resolveRunOutcome({ tracker: spawn, finalText: 'done' }); + assert.equal(sp.status, 'verify_failed'); + assert.match(formatOutcomeLine(sp, 'zh'), /未能启动|ENOENT/); } + { - // Really ran and passed: completed, exit 0, empty line. + // Truly completed: line is「完成：…」, never empty const t = createRunCompletionTracker(); execEvents(t, '1', 'npm run check', 'exit code 0'); noteRunCompletionEvent(t, { @@ -92,28 +139,22 @@ function execEvents(tracker, id, command, result, isError = false) { }); assert.equal(outcome.status, 'completed'); assert.equal(outcome.exitCode, 0); - assert.equal(formatOutcomeLine(outcome, 'zh'), ''); - assert.deepEqual( - findUnverifiedCheckClaims( - 'I did not run tests; checks not green yet.', - createRunCompletionTracker() - ), - [] - ); + assert.match(formatOutcomeLine(outcome, 'zh'), /^完成：/); + assert.match(formatOutcomeLine(outcome, 'en'), /^Completed: /); + assert.notEqual(formatOutcomeLine(outcome, 'zh'), ''); } + { - // Edit task, zero files changed → incomplete (not unverified). + // D: edit task, zero files → incomplete non-0「未完成：没有改动任何文件」 const outcome = resolveRunOutcome({ tracker: createRunCompletionTracker(), finalText: 'Done.', request: '修复 src/cli/hooks.ts 的超时问题', }); assert.equal(outcome.status, 'incomplete'); - assert.match( - formatOutcomeLine(outcome, 'zh'), - /^未完成：任务需要改动文件，但本次运行没有修改任何文件/ - ); - assert.match(formatOutcomeLine(outcome, 'en'), /^Incomplete: the task needed file changes/); + assert.notEqual(outcome.exitCode, 0); + assert.match(formatOutcomeLine(outcome, 'zh'), /^未完成：没有改动任何文件/); + assert.match(formatOutcomeLine(outcome, 'en'), /^Incomplete: no files were modified/); assert.equal( resolveRunOutcome({ tracker: createRunCompletionTracker(), @@ -124,8 +165,38 @@ function execEvents(tracker, id, command, result, isError = false) { ); } +{ + // Blocked Stop hook + const outcome = resolveRunOutcome({ + tracker: createRunCompletionTracker(), + finalText: 'done', + blocked: true, + blockedReason: 'Stop hook wants more work', + }); + assert.equal(outcome.status, 'blocked'); + assert.notEqual(outcome.exitCode, 0); + assert.match(formatOutcomeLine(outcome, 'zh'), /^被拦：/); +} + +{ + // TUI: outcome sits under the done stamp + const lines = renderRunSummary( + 5000, + false, + 80, + undefined, + new Date('2026-10-01T09:05:00'), + '完成：运行结束' + ); + const texts = lines.map((l) => l.text.trim()).filter(Boolean); + assert.ok(texts.some((t) => /done|完成于/.test(t))); + assert.ok(texts.some((t) => t === '完成：运行结束')); + const halted = renderRunSummary(5000, true, 80, undefined, new Date(), '完成：运行结束'); + assert.ok(!halted.some((l) => l.text.includes('完成：运行结束')), 'halted runs omit outcome'); +} + // --- Integration: runOneShot text mode (what `moss -p` uses) -------------- -async function runPrint(name, turns, prompt, registerTools) { +async function runPrint(name, turns, prompt, registerTools, env = {}) { const ws = await fs.mkdtemp(path.join(os.tmpdir(), `moss-o98-${name}-`)); const agent = new MossAgent({ llmProvider: createMockTranscriptProvider(name, 'O98', turns), @@ -136,19 +207,32 @@ async function runPrint(name, turns, prompt, registerTools) { domainPrompt: false, includeAgentBehaviorPrompt: false, enableSteering: false, - maxAgentTurns: 6, + maxAgentTurns: 8, }); registerTools?.(agent); let out = ''; const previous = process.exitCode; + const saved = {}; + for (const [k, v] of Object.entries(env)) { + saved[k] = process.env[k]; + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } process.exitCode = undefined; - await runOneShot(agent, prompt, { - sessionKey: `o98-${name}`, - outputFormat: 'text', - headless: true, - cwd: ws, - stdout: { write: (s) => ((out += s), true) }, - }); + try { + await runOneShot(agent, prompt, { + sessionKey: `o98-${name}`, + outputFormat: 'text', + headless: true, + cwd: ws, + stdout: { write: (s) => ((out += s), true) }, + }); + } finally { + for (const [k, v] of Object.entries(saved)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + } const code = process.exitCode; process.exitCode = previous; await fs.rm(ws, { recursive: true, force: true }); @@ -158,9 +242,9 @@ async function runPrint(name, turns, prompt, registerTools) { const prevLang = process.env.MOSS_LANG; process.env.MOSS_LANG = 'zh'; try { - // O14 run2 shape: lint ran, format:check never ran, answer says "All checks are green". - const r = await runPrint( - 'false-green', + // E integration: no VERIFY_CMD, claim green without running check → 未验证, exit 0 + const soft = await runPrint( + 'soft-unverified', [ { toolCalls: [{ name: 'shell_probe', input: { command: 'npm run lint' } }] }, { text: 'Updated the hook. All checks are green.' }, @@ -177,28 +261,119 @@ try { }, }) ); - assert.notEqual(r.code ?? 0, 0, `false "all green" must not exit 0 (got ${r.code})`); - const last = r.out.trimEnd().split('\n').at(-1); - assert.match(last, /^(未核实：|Unverified: )/, `last line must say unverified, got: ${last}`); - assert.match(last, /npm run check/); + assert.equal(soft.code ?? 0, 0, 'soft unverified stays exit 0'); + assert.match(soft.out.trimEnd().split('\n').at(-1), /^(未验证：|Unverified: )/); - // kimi-k3 r2 shape: edit task, no tool calls, "done" → non-zero + 未完成 line. + // D integration: kimi-k3 r2 — edit task, no tools → non-0 + 未完成 const k = await runPrint( 'no-edit', [{ text: 'Done, the fix is in place.' }], 'Fix the timeout clamp in src/cli/hooks.ts' ); assert.notEqual(k.code ?? 0, 0, 'edit task with zero changes must not exit 0'); - assert.match(k.out.trimEnd().split('\n').at(-1), /^(未完成：|Incomplete: )/); + assert.match( + k.out.trimEnd().split('\n').at(-1), + /^(未完成：没有改动任何文件|Incomplete: no files were modified)/ + ); - // Honest answer to a question: unchanged, exit 0, no incomplete/unverified line. + // Honest Q&A: completed line, exit 0 const ok = await runPrint( 'honest', [{ text: 'Lint was not run in this session.' }], 'What does npm run check do?' ); assert.equal(ok.code ?? 0, 0); - assert.doesNotMatch(ok.out, /未完成：|未核实：|Incomplete: |Unverified: /); + assert.match(ok.out.trimEnd().split('\n').at(-1), /^(完成：|Completed: )/); + + // A: VERIFY_CMD fails + model says green → verify_failed non-0 + // depth starts 0 → one continuation; mock has a second turn that still claims green; + // VERIFY_CMD stays failing → re-verify → verify_failed. + const failScript = path.join(os.tmpdir(), `moss-o98-fail-${Date.now()}.mjs`); + await fs.writeFile(failScript, 'process.exit(1);\n'); + const a = await runPrint( + 'verify-fail', + [ + { text: 'All checks are green.' }, + // continuation turn after first verify failure + { text: 'Fixed. All checks are green.' }, + ], + 'Ship the fix.', + undefined, + { + MOSS_GOAL_VERIFY_LOOP: '1', + MOSS_GOAL_VERIFY_CMD: `node ${failScript}`, + } + ); + assert.notEqual(a.code ?? 0, 0, 'A: VERIFY_CMD failure must exit non-zero'); + assert.match(a.out.trimEnd().split('\n').at(-1), /^(验证失败：|Verify failed: )/); + await fs.rm(failScript, { force: true }); + + // B: first VERIFY fails, continuation succeeds → exit 0 + 完成 + const flipScript = path.join(os.tmpdir(), `moss-o98-flip-${Date.now()}.mjs`); + const flipState = path.join(os.tmpdir(), `moss-o98-flip-state-${Date.now()}.txt`); + await fs.writeFile( + flipScript, + ` +import fs from 'node:fs'; +const p = ${JSON.stringify(flipState)}; +if (!fs.existsSync(p)) { fs.writeFileSync(p, '1'); process.exit(1); } +process.exit(0); +` + ); + const b = await runPrint( + 'verify-flip', + [{ text: 'Working on it.' }, { text: 'Done now.' }], + 'Ship the fix.', + undefined, + { + MOSS_GOAL_VERIFY_LOOP: '1', + MOSS_GOAL_VERIFY_CMD: `node ${flipScript}`, + } + ); + assert.equal(b.code ?? 0, 0, 'B: re-verify success must exit 0'); + assert.match(b.out.trimEnd().split('\n').at(-1), /^(完成：|Completed: )/); + await fs.rm(flipScript, { force: true }); + await fs.rm(flipState, { force: true }); + + // B-fail: continue still fails → non-0 + const fail2 = path.join(os.tmpdir(), `moss-o98-fail2-${Date.now()}.mjs`); + await fs.writeFile(fail2, 'process.exit(1);\n'); + const bFail = await runPrint( + 'verify-still-fail', + [{ text: 'try 1' }, { text: 'try 2 still broken' }], + 'Ship the fix.', + undefined, + { + MOSS_GOAL_VERIFY_LOOP: '1', + MOSS_GOAL_VERIFY_CMD: `node ${fail2}`, + } + ); + assert.notEqual(bFail.code ?? 0, 0, 'B: re-verify still failing → non-0'); + assert.match(bFail.out.trimEnd().split('\n').at(-1), /^(验证失败：|Verify failed: )/); + await fs.rm(fail2, { force: true }); + + // C: VERIFY_CMD does not exist → non-0, reason visible + const c = await runPrint('verify-missing', [{ text: 'All done.' }], 'Ship the fix.', undefined, { + MOSS_GOAL_VERIFY_LOOP: '1', + MOSS_GOAL_VERIFY_CMD: path.join(os.tmpdir(), `moss-o98-no-such-${Date.now()}`), + }); + assert.notEqual(c.code ?? 0, 0, 'C: missing VERIFY_CMD must exit non-zero'); + assert.match(c.out.trimEnd().split('\n').at(-1), /^(验证失败：|Verify failed: )/); + + // Stop hook blocked in headless → non-0 + 被拦 + setLifecycleHookRunner({ + async runStop() { + return { blocked: true, reason: 'tests still red' }; + }, + async runSubagentStop() {}, + }); + try { + const blocked = await runPrint('stop-blocked', [{ text: 'Done.' }], 'What is 2+2?'); + assert.notEqual(blocked.code ?? 0, 0, 'Stop hook blocked must exit non-zero in headless'); + assert.match(blocked.out.trimEnd().split('\n').at(-1), /^(被拦：|Blocked: )/); + } finally { + setLifecycleHookRunner(undefined); + } } finally { if (prevLang === undefined) delete process.env.MOSS_LANG; else process.env.MOSS_LANG = prevLang;