Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/cli-main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ import {
} from './cli/model-catalog.js';
import { createModelInfoTool } from './cli/model-info-tool.js';
import { runOneShotWithCliCancellation } from './cli/run-cancellation.js';
import { armExitGuard } from './cli/exit-guard.js';
import { stopAllBackgroundProcesses } from './core/tools/background-process-registry.js';
import { runInteractive } from './cli/repl.js';
import { resolveCliSession } from './cli/session.js';
import {
Expand Down Expand Up @@ -1794,6 +1796,9 @@ async function main() {
}
} finally {
try {
// O70: exec_start children are detached with piped stdout/stderr; left
// running they hold the event loop open and `moss -p` never returns.
stopAllBackgroundProcesses();
await agent.close();
await disconnectAllDevices();
} finally {
Expand All @@ -1813,6 +1818,8 @@ async function main() {
} catch {
/* shutdown hooks never block exit */
}
// O70: whatever still lingers after cleanup must not hang a one-shot run.
if (parsedArgs.print || oneShotMessage) armExitGuard();
}
}
}
Expand Down
1 change: 1 addition & 0 deletions src/cli/config-commands.ts
Original file line number Diff line number Diff line change
Expand Up @@ -399,6 +399,7 @@ export const MOSS_ENV_REFERENCE: ReadonlyArray<{ group: string; vars: readonly s
'MOSS_PI_AI_INTER_EVENT_TIMEOUT_MS',
'MOSS_PI_AI_TOOL_CHOICE',
'MOSS_TRACE_PI_AI_STREAM',
'MOSS_DEBUG_EXIT',
],
},
{
Expand Down
37 changes: 37 additions & 0 deletions src/cli/exit-guard.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
/**
* O70: `moss -p` must exit within 2 s after the answer is printed.
*
* Normal exit relies on the event loop draining. Any handle left behind (a
* background exec child whose stdout/stderr pipes are still open, an MCP
* child that ignores SIGTERM, a stray ref'd timer or socket) keeps the
* process alive. The guard is an unref'd timer: when the loop drains on its
* own it never fires; when something lingers it forces the exit, always prints a
* stderr warning with the number of open handles, and with MOSS_DEBUG_EXIT=1
* also names them.
*/
export const EXIT_GUARD_MS = 2000;

export function describeActiveHandles(): string[] {
const getter = (process as unknown as { getActiveResourcesInfo?: () => string[] })
.getActiveResourcesInfo;
return typeof getter === 'function' ? getter.call(process) : [];
}

export function armExitGuard(
ms: number = EXIT_GUARD_MS,
log: (line: string) => void = (line) => process.stderr.write(`${line}\n`)
): NodeJS.Timeout {
const timer = setTimeout(() => {
// Always warn so the force-exit never hides a leak (O70 acceptance).
const handles = describeActiveHandles();
log(
`[moss] warning: forcing exit after ${ms} ms; ${handles.length} handle(s) still open` +
(process.env.MOSS_DEBUG_EXIT === '1'
? `: ${handles.join(', ')}`
: ' (set MOSS_DEBUG_EXIT=1 for details)')
);
process.exit(typeof process.exitCode === 'number' ? process.exitCode : 0);
}, ms);
timer.unref();
return timer;
}
5 changes: 5 additions & 0 deletions src/core/agent/moss-agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { buildExperienceBlock, experienceEnabled } from '../experience/experienc

const log = getRootLogger().child('agent');
import { ToolRegistry } from '../tools/tool-registry.js';
import { stopAllBackgroundProcesses } from '../tools/background-process-registry.js';
import { filterToolsForRun } from '../tools/tool-filter.js';
import { DeferredToolOffer, TOOL_SEARCH_NAME } from '../tools/deferred-tool-offer.js';
import {
Expand Down Expand Up @@ -220,6 +221,10 @@ export class MossAgent {
this.rootAbortController.abort(
new MossError({ code: ErrorCode.USER_ABORTED, message: 'Agent closed' })
);
// O70: background exec_start children hold stdout/stderr pipes open; stop
// them whenever the agent closes so hosts that skip the CLI finally path
// still drain the event loop.
stopAllBackgroundProcesses();
this.pendingToolAborts.dispose();
const asyncTasks = this.ownsAsyncTaskRegistry
? (this.asyncTasks.stopAll?.('parent_aborted').then(() => {}) ?? Promise.resolve())
Expand Down
14 changes: 12 additions & 2 deletions src/core/loop/agent-loop-stream-helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -505,7 +505,7 @@ export async function runAgentLoopLlmTurn(
}
if (firstChunkTimedOut && !abortSignal.aborted) {
throw new LlmFirstChunkTimeoutError(
`LLM produced no streaming output (including thinking) within ${Math.round(firstChunkBudgetMs / 1000)}s. Check network/proxy, Base URL, API Key and model availability; or try disabling extended thinking or switching models.`
`No response from the model within ${Math.round(firstChunkBudgetMs / 1000)}s (no streaming output, including thinking). Check network/proxy, Base URL, API Key, and that the gateway is reachable.`
);
}
throw streamErr;
Expand All @@ -521,7 +521,17 @@ export async function runAgentLoopLlmTurn(
label: 'llm-call',
shouldRetry: (err, attempt) => {
if (abortSignal.aborted) return false;
if (err instanceof LlmFirstChunkTimeoutError && attempt >= 2) return false;
if (err instanceof LlmFirstChunkTimeoutError) {
// O67: a blackhole HTTPS_PROXY yields zero bytes forever — retrying
// only multiplies the hang. One recovery retry is enough without a proxy.
const proxySet = Boolean(
process.env.HTTPS_PROXY ||
process.env.HTTP_PROXY ||
process.env.https_proxy ||
process.env.http_proxy
);
if (proxySet || attempt >= 2) return false;
}
return classifyLlmError(err).retryable;
},
retryDelayMs: (err, attempt, _computed) => {
Expand Down
11 changes: 11 additions & 0 deletions src/provider/connection-error.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,17 @@ function classifyConnectionHint(
);
}
if (causeCode === 'ECONNREFUSED') {
if (hasProxyEnv) {
const proxy =
process.env.HTTPS_PROXY ||
process.env.HTTP_PROXY ||
process.env.https_proxy ||
process.env.http_proxy ||
'HTTP_PROXY/HTTPS_PROXY';
return hint(
`Proxy connection refused while reaching ${host} — could not connect to the proxy at ${proxy}. Check HTTPS_PROXY/HTTP_PROXY (host, port, and that the proxy is running).`
);
}
return hint(
`Connection refused by ${host} — the server is not running or the port is wrong. Check that the gateway/service is up and the baseUrl port is correct.`
);
Expand Down
20 changes: 14 additions & 6 deletions src/provider/error-classify.ts
Original file line number Diff line number Diff line change
Expand Up @@ -449,6 +449,10 @@ const COPY = {
'连接被拒绝。这个地址的主机或端口没有服务在听。请改地址,然后重试。',
'Connection refused. Nothing is listening at that base URL (wrong host or port). Fix the base URL, then retry.',
],
refusedProxy: [
'代理连接被拒绝。请检查 HTTPS_PROXY/HTTP_PROXY(主机、端口,以及代理是否在运行)。',
'Proxy connection refused. Check HTTPS_PROXY/HTTP_PROXY (host, port, and that the proxy is running).',
],
connect: [
'网络连接失败。请检查地址和网络,然后重试。',
'Could not connect. Check the base URL and your network, then retry.',
Expand Down Expand Up @@ -585,13 +589,17 @@ function classifyProviderErrorNow(input: ProviderErrorInput): ProviderErrorSurfa
}
if (isRateLimitFailure(raw, status)) return hit('rate_limit', 'rate', [RETRY], true);
if (isConnectionError(raw)) {
return hit(
'network',
/econnrefused|connection refused/i.test(raw) ? 'refused' : 'connect',
RETRY_SETTINGS,
true,
'setup'
const refused = /econnrefused|connection refused/i.test(raw);
const proxySet = Boolean(
process.env.HTTPS_PROXY ||
process.env.HTTP_PROXY ||
process.env.https_proxy ||
process.env.http_proxy
);
// O67: when a proxy is configured, a refused connect is almost always the
// proxy port — do not blame the model base URL.
const id = refused ? (proxySet ? 'refusedProxy' : 'refused') : 'connect';
return hit('network', id, RETRY_SETTINGS, true, 'setup');
}
if (isTlsProtocolMismatch(raw, code))
return hit('network', 'httpNotTls', SETTINGS_RETRY, false, 'body');
Expand Down
10 changes: 9 additions & 1 deletion src/provider/llm-error-classifier.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,15 @@ export function classifyLlmError(error: unknown): LlmErrorClassification {
// mention API keys and otherwise look like a non-retryable auth failure.
// Checking the name avoids a dependency cycle with the stream helper.
if (error instanceof Error && error.name === 'LlmFirstChunkTimeoutError') {
return { category: 'timeout', retryable: true, message };
// O67: with a proxy configured, a silent first-chunk stall is almost always
// a dead/blackhole proxy — do not burn 3× the budget retrying.
const proxySet = Boolean(
process.env.HTTPS_PROXY ||
process.env.HTTP_PROXY ||
process.env.https_proxy ||
process.env.http_proxy
);
return { category: 'timeout', retryable: !proxySet, message };
}
if (isAbortFailure(message)) {
return { category: 'user_abort', retryable: false, message };
Expand Down
23 changes: 22 additions & 1 deletion test/connection-error-proxy-hint.spec.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,27 @@ try {
globalThis.fetch = originalFetch;
}

// ─── 5. ECONNREFUSED with proxy env → blame the proxy, not the base URL ─────
process.env.HTTP_PROXY = 'http://127.0.0.1:9';
process.env.HTTPS_PROXY = 'http://127.0.0.1:9';
globalThis.fetch = async () => {
const e = new TypeError('fetch failed');
e.cause = { code: 'ECONNREFUSED', message: 'connect ECONNREFUSED 127.0.0.1:9' };
throw e;
};
try {
await fetchWithConnectionContext('https://api.example.com/v1/x', { method: 'GET' });
assert.fail('should have thrown');
} catch (err) {
assert.ok(err instanceof MossError);
assert.match(err.hint, /[Pp]roxy/, `proxy-named hint, got: ${err.hint}`);
assert.doesNotMatch(err.hint, /baseUrl port is correct/);
} finally {
globalThis.fetch = originalFetch;
delete process.env.HTTP_PROXY;
delete process.env.HTTPS_PROXY;
}

console.log(
' [PASS] connection-error: proxy-tunnel refusal hint + DNS + TLS CA + no-false-positive'
' [PASS] connection-error: proxy-tunnel refusal hint + DNS + TLS CA + no-false-positive + refused-proxy'
);
2 changes: 1 addition & 1 deletion test/loop-first-chunk-hard-timeout.spec.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ try {
firstTokenMs: null,
logDebug() {},
}),
/no streaming output.*within 0s/i
/No response from the model within 0s.*no streaming output/i
);
assert.equal(calls, 2, 'a silent first-chunk timeout gets at most one recovery retry');
assert.ok(
Expand Down
56 changes: 56 additions & 0 deletions test/o67-refused-proxy-copy.spec.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
#!/usr/bin/env node
import assert from 'node:assert/strict';
import { test } from 'node:test';
import { classifyProviderError } from '../dist/provider/error-classify.js';

test('refused connect with HTTPS_PROXY blames the proxy, not the base URL', () => {
const prev = {
HTTPS_PROXY: process.env.HTTPS_PROXY,
HTTP_PROXY: process.env.HTTP_PROXY,
https_proxy: process.env.https_proxy,
http_proxy: process.env.http_proxy,
};
process.env.HTTPS_PROXY = 'http://127.0.0.1:9';
delete process.env.HTTP_PROXY;
delete process.env.https_proxy;
delete process.env.http_proxy;
try {
const r = classifyProviderError({
errorMessage: 'connect ECONNREFUSED 127.0.0.1:9',
locale: 'en',
});
assert.equal(r.category, 'network');
assert.match(r.userMessage, /[Pp]roxy/);
assert.doesNotMatch(r.userMessage, /Nothing is listening at that base URL/);
} finally {
for (const [k, v] of Object.entries(prev)) {
if (v === undefined) delete process.env[k];
else process.env[k] = v;
}
}
});

test('refused connect without proxy still blames the base URL', () => {
const prev = {
HTTPS_PROXY: process.env.HTTPS_PROXY,
HTTP_PROXY: process.env.HTTP_PROXY,
https_proxy: process.env.https_proxy,
http_proxy: process.env.http_proxy,
};
delete process.env.HTTPS_PROXY;
delete process.env.HTTP_PROXY;
delete process.env.https_proxy;
delete process.env.http_proxy;
try {
const r = classifyProviderError({
errorMessage: 'connect ECONNREFUSED 127.0.0.1:9',
locale: 'en',
});
assert.match(r.userMessage, /Fix the base URL|Nothing is listening at that base URL/);
} finally {
for (const [k, v] of Object.entries(prev)) {
if (v === undefined) delete process.env[k];
else process.env[k] = v;
}
}
});
97 changes: 97 additions & 0 deletions test/o70-exit-hang.spec.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
#!/usr/bin/env node
// O70: a one-shot run must exit within 2 s even when a handle lingers.
import test from 'node:test';
import assert from 'node:assert/strict';
import { spawn } from 'node:child_process';
import { fileURLToPath } from 'node:url';

// Uses POSIX `sleep` and process-group kill (kill(-pid)); no equivalent on win32.
const posixOnly =
process.platform === 'win32'
? 'O70 spec relies on POSIX sleep and process-group kill(-pid); Windows covered by O70 follow-up'
: false;

const guard = fileURLToPath(new URL('../dist/cli/exit-guard.js', import.meta.url));
const registry = fileURLToPath(
new URL('../dist/core/tools/background-process-registry.js', import.meta.url)
);

function runChild(source, debug = false) {
return new Promise((resolve) => {
const started = Date.now();
const child = spawn(process.execPath, ['--input-type=module', '-e', source], {
stdio: ['ignore', 'pipe', 'pipe'],
env: { ...process.env, MOSS_DEBUG_EXIT: debug ? '1' : '' },
});
let stderr = '';
child.stderr.on('data', (c) => (stderr += c));
const kill = setTimeout(() => child.kill('SIGKILL'), 8000);
child.on('close', (code, signal) => {
clearTimeout(kill);
resolve({ code, signal, ms: Date.now() - started, stderr });
});
});
}

test(
'baseline: a lingering detached child with pipes keeps node alive (the O70 hang)',
{ skip: posixOnly },
async () => {
const r = await runChild(`
import { spawn } from 'node:child_process';
spawn('sleep', ['30'], { stdio: ['ignore', 'pipe', 'pipe'], detached: true });
`);
assert.equal(r.signal, 'SIGKILL', 'without the guard the process should still be alive at 8 s');
}
);

test(
'exit guard forces exit within ~2 s and keeps the exit code',
{ skip: posixOnly },
async () => {
const r = await runChild(`
import { spawn } from 'node:child_process';
import { armExitGuard } from ${JSON.stringify(guard)};
const c = spawn('sleep', ['30'], { stdio: ['ignore', 'pipe', 'pipe'], detached: true });
setInterval(() => {}, 1000);
process.exitCode = 3;
armExitGuard();
process.on('exit', () => { try { process.kill(-c.pid, 'SIGKILL'); } catch {} });
`);
assert.equal(r.code, 3);
assert.ok(r.ms < 4000, `took ${r.ms} ms`);
// Warning is printed by default (no debug flag) and reports the open-handle count.
assert.match(
r.stderr,
/\[moss\] warning: forcing exit after 2000 ms; [1-9]\d* handle\(s\) still open/
);
}
);

test('MOSS_DEBUG_EXIT=1 adds the handle names to the warning', { skip: posixOnly }, async () => {
const r = await runChild(
`
import { armExitGuard } from ${JSON.stringify(guard)};
setInterval(() => {}, 1000);
armExitGuard();
`,
true
);
assert.equal(r.code, 0);
assert.match(r.stderr, /handle\(s\) still open: .+/);
});

test('exit guard does not delay a clean exit', async () => {
const r = await runChild(`
import { armExitGuard } from ${JSON.stringify(guard)};
armExitGuard();
`);
assert.equal(r.code, 0);
assert.ok(r.ms < 1500, `took ${r.ms} ms`);
assert.doesNotMatch(r.stderr, /forcing exit/);
});

test('stopAllBackgroundProcesses is exported for CLI shutdown', async () => {
const mod = await import(registry);
assert.equal(typeof mod.stopAllBackgroundProcesses, 'function');
});
Loading
Loading